diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index 16e87c77..00000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM ubuntu:22.04 - -# Install needed packages and setup non-root user. Use a separate RUN statement to add your own dependencies. -ARG USERNAME=vscode -ARG USER_UID=1000 -ARG USER_GID=$USER_UID - -# Create the user -RUN groupadd --gid $USER_GID $USERNAME \ - && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ - && apt-get update \ - && apt-get install -y sudo curl \ - && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ - && chmod 0440 /etc/sudoers.d/$USERNAME - -# Install Node.js 22.x from NodeSource -RUN curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - \ - && apt-get install -y nodejs - -# [Optional] Clean up -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# [Optional] Set the default user. Omit if you want to keep the default as root. -USER $USERNAME diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9d52dbb2..d41b017c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,22 +1,18 @@ { "name": "devicecode-lua", - "build": { - "dockerfile": "Dockerfile" - }, + "image": "ghcr.io/jangala-dev/mini-lua-image:latest", "remoteUser": "vscode", "postCreateCommand": "bash .devcontainer/postCreateCommand.sh", "forwardPorts": [ - 8081 + 8081, + 2222, + 2242 ], "customizations": { "vscode": { "extensions": [ - "ms-azuretools.vscode-docker", - "ms-vscode-remote.remote-containers", - "waderyan.gitblame", "sumneko.lua", "rog2.luacheck", - "tomblind.local-lua-debugger-vscode", "bierner.markdown-mermaid" ] } diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh old mode 100755 new mode 100644 index c0c6b447..c77f5a96 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -1,46 +1,62 @@ -#!/bin/sh - -sudo apt update -y - -sudo apt install -y apt-utils unzip curl wget git build-essential libreadline-dev dialog libssl-dev m4 openssl - -# install core lua packages - -sudo apt install -y lua5.1 liblua5.1-dev luarocks lua-dkjson - -cd /tmp -sudo rm -rf LuaJIT -git clone -b v2.1.ROLLING https://github.com/LuaJIT/LuaJIT.git -cd LuaJIT -make -j$(nproc) -sudo make install -# Find the latest installed luajit binary under /usr/local/bin -latest=$(ls -1 /usr/local/bin/luajit-2.1.* | sort | tail -n 1) -# Symlink it to /usr/local/bin/luajit -sudo ln -sf "$latest" /usr/local/bin/luajit -sudo ldconfig - -# install luarocks packages -sudo luarocks install bit32 -sudo luarocks install cqueues -sudo luarocks install http -sudo luarocks install luaposix -sudo luarocks install luacheck -sudo luarocks install lua-cjson -sudo luarocks install dkjson -sudo luarocks install luaunit - -# install cffi-lua -sudo apt install -y meson pkg-config cmake libffi-dev - -cd /tmp -sudo rm -rf cffi-lua -git clone https://github.com/q66/cffi-lua -mkdir cffi-lua/build -cd cffi-lua/build -sudo meson .. -Dlua_version=5.1 --buildtype=release -sudo ninja all -sudo ninja test -sudo cp cffi.so /usr/local/lib/lua/5.1/cffi.so +#!/usr/bin/env sh +set -eu + +echo "[devcontainer] preparing OpenWrt VM test tools" + +if command -v apt-get >/dev/null 2>&1; then + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + gzip \ + openssh-client \ + rsync \ + socat \ + qemu-system-x86 \ + qemu-system-arm \ + qemu-utils \ + ovmf \ + qemu-efi-aarch64 \ + make \ + iproute2 \ + iputils-ping \ + busybox \ + dnsutils \ + wget \ + tcpdump \ + cloud-image-utils \ + genisoimage +elif command -v apk >/dev/null 2>&1; then + sudo apk add --no-cache \ + curl \ + ca-certificates \ + gzip \ + openssh-client \ + rsync \ + socat \ + qemu-system-x86_64 \ + qemu-system-aarch64 \ + qemu-img \ + ovmf \ + edk2-aarch64 \ + make \ + iproute2 \ + iputils \ + busybox \ + bind-tools \ + wget \ + tcpdump \ + xorriso +else + echo "[devcontainer] unsupported base image: no apt-get or apk found" >&2 + exit 1 +fi + +# Privileged OpenWrt dataplane tests are run inside the optional network-lab VM. +# The top-level devcontainer is intentionally unprivileged so normal development +# and fast CI jobs do not inherit bridge/tap/netns permissions. +mkdir -p tests/integration/openwrt_vm/{images,work,scripts,tests} + +echo "[devcontainer] OpenWrt VM tools ready" exit 0 diff --git a/.env b/.env index f72da50b..0f94e0e5 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -FIBERS_VER=43a04d1 -TRIE_VER=28b3572 -BUS_VER=89af71a -UI_VER=f424ac2 +FIBERS_VER=30353346ca58c3662b9942f62ff4a9610f8ab8cb +TRIE_VER=v0.3 +BUS_VER=v0.8.2 +UI_VER=c4451f4f386f5e47a13e8689a0bbf587031e34b2 diff --git a/.gitignore b/.gitignore index 6d6def6a..6665c841 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ scratchpad src/services/modem-* docs/ build/ +.env.secret +local-ui/ +src/services/ui/www/ diff --git a/.gitmodules b/.gitmodules index 1a8ed5f6..ed1843b6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,9 @@ -[submodule "src/lua-fibers"] - path = src/lua-fibers +[submodule "vendor/lua-fibers"] + path = vendor/lua-fibers url = https://github.com/jangala-dev/lua-fibers.git -[submodule "src/lua-trie"] - path = src/lua-trie +[submodule "vendor/lua-trie"] + path = vendor/lua-trie url = https://github.com/jangala-dev/lua-trie.git -[submodule "src/lua-bus"] - path = src/lua-bus +[submodule "vendor/lua-bus"] + path = vendor/lua-bus url = https://github.com/jangala-dev/lua-bus.git -[submodule "src/services/ui/local-ui"] - path = src/services/ui/local-ui - url = https://github.com/jangala-dev/local-ui.git diff --git a/.luarc.json b/.luarc.json new file mode 100644 index 00000000..6a4925e2 --- /dev/null +++ b/.luarc.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://raw.githubusercontent.com/LuaLS/vscode-lua/master/setting/schema.json", + + "runtime.version": "LuaJIT", + + "runtime.path": [ + "src/?.lua", + "src/?/init.lua", + + "vendor/lua-fibers/src/?.lua", + "vendor/lua-fibers/src/?/init.lua", + + "vendor/lua-bus/src/?.lua", + "vendor/lua-bus/src/?/init.lua", + + "vendor/lua-trie/src/?.lua", + "vendor/lua-trie/src/?/init.lua" + ], + + "workspace.library": [ + "src", + "vendor/lua-fibers/src", + "vendor/lua-bus/src", + "vendor/lua-trie/src" + ], + + "workspace.checkThirdParty": false, + "workspace.maxPreload": 10000, + + "diagnostics.enable": true, + "diagnostics.workspaceDelay": -1 +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..796c6d08 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Debug Lua Interpreter", + "type": "lua-local", + "request": "launch", + "program": { + "lua": "lua", + "file": "${file}" + } + }, + { + "name": "Debug Custom Lua Environment", + "type": "lua-local", + "request": "launch", + "program": { + "command": "command" + }, + "args": [] + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 5e578f27..dcc7a766 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -14,10 +14,7 @@ "[lua]": { "editor.tabSize": 4 }, - "Lua.workspace.checkThirdParty": false, - "Lua.workspace.maxPreload": 0, "Lua.diagnostics.workspaceDelay": -1, - "Lua.workspace.library": [], "Lua.diagnostics.enable": true, "cSpell.words": [ "autopin", diff --git a/Makefile b/Makefile index 45fcf056..6f6b553e 100644 --- a/Makefile +++ b/Makefile @@ -1,110 +1,148 @@ # Variables include .env -SRC_DIR := src -BUILD_DIR := build -TEST_DIR := tests -LINTER := luacheck +SRC_DIR := src +VENDOR_DIR := vendor +BUILD_DIR := build +TEST_DIR := tests +LINTER := luacheck +UI_REPO ?= https://github.com/jangala-dev/local-ui.git +UI_DIR ?= local-ui +UI_WWW_DIR := $(SRC_DIR)/services/ui/www # Default target .PHONY: all -all: env test-all build lint +all: env build-all test-all lint -# Build: Moves source files and removes and testing from submodules +# Build: Copy source files into the build directory .PHONY: build build: - @echo "Building the project..." + @echo "Building source..." @rm -rf $(BUILD_DIR) @mkdir -p $(BUILD_DIR) @cp -r $(SRC_DIR)/* $(BUILD_DIR)/ - @cp $(BUILD_DIR)/lua-bus/src/* $(BUILD_DIR) - @rm -rf $(BUILD_DIR)/lua-bus - @cp -r $(BUILD_DIR)/lua-fibers/fibers $(BUILD_DIR) - @rm -rf $(BUILD_DIR)/lua-fibers - @cp $(BUILD_DIR)/lua-trie/src/* $(BUILD_DIR) - @rm -rf $(BUILD_DIR)/lua-trie @rm -rf $(BUILD_DIR)/services/ui/local-ui @echo "Build complete." -# Build-All: Builds the project and all submodules including ui +# Build-Vendor: Copy vendor library source into build/lib +.PHONY: build-vendor +build-vendor: + @echo "Building vendor libs..." + @mkdir -p $(BUILD_DIR)/lib + @cp -r $(VENDOR_DIR)/lua-fibers/src/. $(BUILD_DIR)/lib/ + @cp -r $(VENDOR_DIR)/lua-trie/src/. $(BUILD_DIR)/lib/ + @cp -r $(VENDOR_DIR)/lua-bus/src/. $(BUILD_DIR)/lib/ + @echo "Vendor build complete." + +# Build-UI: Build local-ui and stage static files for the UI service +.PHONY: build-ui +build-ui: + @echo "Building local UI..." + @if [ ! -d "$(UI_DIR)/client" ]; then \ + echo "local-ui checkout missing; run 'make env' first." >&2; \ + exit 1; \ + fi + @$(MAKE) -C $(UI_DIR) build + @rm -rf $(UI_WWW_DIR) + @mkdir -p $(UI_WWW_DIR) + @cp -R $(UI_DIR)/build/dist/. $(UI_WWW_DIR)/ + @echo "Local UI build complete." + +# Build-All: Build UI, source, and vendor libs, then substitute secrets .PHONY: build-all -build-all: - @make build - @cd $(SRC_DIR)/services/ui/local-ui && make build - @mkdir -p $(BUILD_DIR)/www/ui - @cp -r $(SRC_DIR)/services/ui/local-ui/build/dist $(BUILD_DIR)/www/ui - @echo "Build complete." +build-all: build-ui build build-vendor substitute-secrets + +# Substitute-Secrets: Replace $VAR placeholders in build configs using .env.secret +.PHONY: substitute-secrets +substitute-secrets: + @echo "Substituting secrets in build configs..." + @if [ ! -f .env.secret ]; then \ + echo "WARNING: .env.secret not found — \$$VAR placeholders in build configs will not be replaced."; \ + else \ + while IFS='=' read -r key value; do \ + [ -z "$$key" ] && continue; \ + find $(BUILD_DIR)/configs -name "*.json" | while read -r f; do \ + sed -i 's|\$$'"$$key"'|'"$$value"'|g' "$$f"; \ + done; \ + done < .env.secret; \ + fi + @echo "Secret substitution complete." + +# Env: Pin each vendor submodule to the revision specified in .env +.PHONY: env +env: + @echo "Initialising vendor submodules..." + @git submodule update --init --recursive + @cd $(VENDOR_DIR)/lua-fibers && git fetch && git checkout $(FIBERS_VER) + @cd $(VENDOR_DIR)/lua-trie && git fetch && git checkout $(TRIE_VER) + @cd $(VENDOR_DIR)/lua-bus && git fetch && git checkout $(BUS_VER) + @if [ ! -d "$(UI_DIR)/.git" ]; then \ + if [ -e "$(UI_DIR)" ]; then \ + echo "$(UI_DIR) exists but is not a git checkout." >&2; \ + exit 1; \ + fi; \ + git clone $(UI_REPO) $(UI_DIR); \ + fi + @cd $(UI_DIR) && git fetch && git checkout --detach $(UI_VER) + @cd $(UI_DIR)/client && npm install + @echo "Submodules pinned." -# Test: Run the project's test suite +# Test: Run the devicecode test suite .PHONY: test test: - @echo "Running project tests..." - @cd $(TEST_DIR) && luajit test.lua - @echo "Tests completed." + @echo "Running devicecode tests..." + @cd $(TEST_DIR) && luajit run.lua + @echo "Tests complete." -# Test-All: Run the project's and submodule's test suite +# Test-All: Run devicecode and vendor test suites .PHONY: test-all test-all: @echo "Running all tests..." # Devicecode tests - @cd $(TEST_DIR) && luajit test.lua -# Fiber tests - @cd $(SRC_DIR)/lua-fibers/tests && luajit test.lua + @cd $(TEST_DIR) && luajit run.lua +# Fibers tests + @cd $(VENDOR_DIR)/lua-fibers/tests && luajit test.lua # Trie tests - @cd $(SRC_DIR)/lua-trie/tests && luajit test.lua -# Bus tests (require movement of fiber and trie then cleanup) - @cp -r $(SRC_DIR)/lua-fibers/fibers $(SRC_DIR)/lua-bus/src - @cp -r $(SRC_DIR)/lua-trie/src/* $(SRC_DIR)/lua-bus/src - @cd $(SRC_DIR)/lua-bus/tests && luajit test.lua - @rm -rf $(SRC_DIR)/lua-bus/src/fibers - @rm -rf $(SRC_DIR)/lua-bus/src/trie.lua -# UI tests - @cd $(SRC_DIR)/services/ui/local-ui && make unit-test - @echo "Tests completed." - -# Env: Initialize environment and update git submodules -.PHONY: env -env: - @echo "Updating git submodules..." - @git submodule update --init --recursive - @cd $(SRC_DIR)/lua-fibers && git checkout main && git pull && git checkout $(FIBERS_VER) - @cd $(SRC_DIR)/lua-trie && git checkout main && git pull && git checkout $(TRIE_VER) - @cd $(SRC_DIR)/lua-bus && git checkout main && git pull && git checkout $(BUS_VER) - @echo "Git submodules updated." - -# Includes ui -.PHONY: env-all -env-all: - @echo "Updating git submodules..." - @git submodule update --init --recursive - @cd $(SRC_DIR)/lua-fibers && git checkout main && git pull && git checkout $(FIBERS_VER) - @cd $(SRC_DIR)/lua-trie && git checkout main && git pull && git checkout $(TRIE_VER) - @cd $(SRC_DIR)/lua-bus && git checkout main && git pull && git checkout $(BUS_VER) - @cd $(SRC_DIR)/services/ui/local-ui && git checkout main && git fetch && git checkout $(UI_VER) \ - && cd client && npm install - @echo "Git submodules updated." + @cd $(VENDOR_DIR)/lua-trie/tests && luajit test.lua +# Bus tests — fibers and trie must be present on the require path; stage then clean up + @cp -r $(VENDOR_DIR)/lua-fibers/src/fibers $(VENDOR_DIR)/lua-bus/src/ + @cp $(VENDOR_DIR)/lua-fibers/src/fibers.lua $(VENDOR_DIR)/lua-bus/src/ + @cp $(VENDOR_DIR)/lua-fibers/src/coxpcall.lua $(VENDOR_DIR)/lua-bus/src/ + @cp $(VENDOR_DIR)/lua-trie/src/trie.lua $(VENDOR_DIR)/lua-bus/src/ + @cd $(VENDOR_DIR)/lua-bus/tests && luajit test.lua + @rm -rf $(VENDOR_DIR)/lua-bus/src/fibers + @rm -f $(VENDOR_DIR)/lua-bus/src/fibers.lua + @rm -f $(VENDOR_DIR)/lua-bus/src/coxpcall.lua + @rm -f $(VENDOR_DIR)/lua-bus/src/trie.lua + @echo "All tests complete." -# Lint: Run the linter to check code quality +# Lint: Static analysis on source and test directories .PHONY: lint lint: @echo "Running linter..." @$(LINTER) $(SRC_DIR) $(TEST_DIR) @echo "Linting complete." -# Clean: Remove build artifacts +# Clean: Remove the build directory .PHONY: clean clean: @echo "Cleaning build artifacts..." @rm -rf $(BUILD_DIR) @echo "Clean complete." -# Help: Display available targets +# Help: Show available targets +.PHONY: help help: - @echo "Available targets:" - @echo "all - Build, test, lint, and update submodules" - @echo "build - Build the project (removes testing code and documentation)" - @echo "test - Run project test suite" - @echo "test-all - Run project and submodules test suite" - @echo "env - Update and initialize git submodules" - @echo "lint - Run the code linter" - @echo "clean - Remove build artifacts" - @echo "help - Display this help message" + @echo "Usage: make " + @echo "" + @echo "Targets:" + @echo " build Copy source files into $(BUILD_DIR)/" + @echo " build-vendor Copy vendor library sources into $(BUILD_DIR)/lib/" + @echo " build-ui Build local-ui into $(UI_WWW_DIR)/" + @echo " substitute-secrets Replace \$$VAR placeholders in $(BUILD_DIR)/configs/ using .env.secret" + @echo " build-all Run build-ui, build, build-vendor, and substitute-secrets (default)" + @echo " env Pin vendor submodules to revisions in .env" + @echo " test Run the devicecode test suite" + @echo " test-all Run devicecode and all vendor test suites" + @echo " lint Run luacheck on $(SRC_DIR)/ and $(TEST_DIR)/" + @echo " clean Remove $(BUILD_DIR)/" + @echo " help Show this message" diff --git a/README.md b/README.md index c1da79a9..3ca8228a 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,57 @@ This repository contains the Lua version of [Jangala's](https://www.janga.la) `devicecode`, the program that powers our Big Box and Get Box devices. -# Using make script -SRC_DIR, TEST_DIR and BUILD_DIR are all optional values +## Run devicecode (current runtime) -## Initialise dev environment -To ensure you have all the required dependencies, use the devcontainer (see [.devcontainer/devcontainer.json](.devcontainer/devcontainer.json)). +Use the devcontainer for dependencies (see [.devcontainer/devcontainer.json](.devcontainer/devcontainer.json)). -Before testing or building the code all submodules need to be loaded in. You can select -the version of each submodule in `.env`. -``` -make env SRC_DIR= -``` +Runtime entrypoint is `src/main.lua`. -## Build devicecode -Building devicecode creates a folder with only required source files and restructures the submodules -to make a simpler file structure. -``` -make build SRC_DIR= BUILD_DIR= -``` +For on-box runs, copy both `src/` and `vendor/` to the target, then run devicecode from inside `src/`. -## Test devicecode -Devicecode and it's submodules can be tested, to test devicecode only -``` -make test TEST_DIR= -``` -to test devicecode and the submodules -``` -make test-all TEST_DIR= SRC_DIR= -``` +### Required environment variables + +- `DEVICECODE_SERVICES`: Comma-separated services to start (example: `hal,config,monitor`). +- `DEVICECODE_CONFIG_DIR`: Directory that contains `.json` used at startup by HAL. +- `CONFIG_TARGET`: Base filename (without `.json`) for the config blob consumed by the config service. + +### Optional environment variables + +- `DEVICECODE_ENV`: `dev` or `prod` (defaults to `dev`). + +### Run command + +Run from `src/` so Lua module paths resolve correctly: + +```bash +cd src +DEVICECODE_ENV=dev \ +DEVICECODE_SERVICES='hal,config,monitor' \ +DEVICECODE_CONFIG_DIR=/path/to/config/dir \ +CONFIG_TARGET=config \ +luajit main.lua +``` +## Build + +Run `make build-all` to copy sources into `build/` and inject secrets into config files. + +### Secret substitution + +Config files under `src/configs/` may contain `$VAR` placeholders for sensitive values (credentials, URLs, etc.). During `make build-all`, these are replaced in the **build output only** — `src/` is never modified. + +Create `.env.secret` in the repository root with the following keys: -## Linter -To run the linter ``` -make lint SRC_DIR= TEST_DIR= +SWITCH_USERNAME= +SWITCH_PASSWORD= +UNIFI_ADDRESS= +CLOUD_URL= ``` + +`.env.secret` is listed in `.gitignore` and must never be committed. If the file is absent, `make build-all` will print a warning and leave placeholders unreplaced. + +## Vendor Versions + +`lua-fibers`: v0.8.1 +`lua-trie`: v0.3 +`lua-bus`: v0.4 diff --git a/docs/devhost/local-ui-demo.md b/docs/devhost/local-ui-demo.md new file mode 100644 index 00000000..f08e0010 --- /dev/null +++ b/docs/devhost/local-ui-demo.md @@ -0,0 +1,38 @@ +# Local UI devhost demo + +This demo is for the initial local UI port. It starts the real HTTP, UI and GSM +services, and uses a fake HAL control-store capability for GSM APN persistence. +No hardware is required. + +Run from the repository root: + +```sh +scripts/devhost-local-ui-demo --port 18089 +``` + +Then open: + +```text +http://127.0.0.1:18089/ +``` + +Useful checks: + +```sh +curl -fsS http://127.0.0.1:18089/api/local-ui/bootstrap +curl -fsS http://127.0.0.1:18089/api/gsm/apns/custom +curl -fsS http://127.0.0.1:18089/api/diagnostics +``` + +Update the prototype APN store: + +```sh +curl -fsS -X PUT \ + -H 'Content-Type: application/json' \ + -d '{"records":[{"carrier":"Demo Carrier","mcc":"234","mnc":"10","apn":"demo.internet"}]}' \ + http://127.0.0.1:18089/api/gsm/apns/custom +``` + +The fake control-store is in-memory and lasts only for the lifetime of the demo +process. This is deliberate: the demo proves the service boundaries and UI HTTP +paths without writing host state. diff --git a/docs/naming.md b/docs/naming.md new file mode 100644 index 00000000..caadf5c5 --- /dev/null +++ b/docs/naming.md @@ -0,0 +1,1102 @@ +# Devicecode control-plane naming standard + +## Status + +Draft for team review. + +## 0. Why this scheme exists + +Devicecode has now grown beyond a simple in-process service runtime. + +It has become a local control plane for: + +* appliance composition +* imported internal members +* raw provider-native capabilities +* stable public manager interfaces +* retained workflow records +* canonical retained domain truth +* operator-facing local UI +* compatibility publication during migration + +Earlier naming was developed while we were still conceiving the system. It's revealed not be be adequate once the system must clearly distinguish: + +* raw provenance-bearing truth from curated public truth +* appliance composition from domain policy +* stable public interfaces from provider-native interfaces +* workflow managers from retained workflow instances +* intended behaviour from observed state +* canonical surfaces from temporary compatibility aliases + +This scheme exists to make those distinctions **structural**, not merely conventional. + +It is intended to ensure that public contracts remain stable even when: + +* implementation placement changes +* transport changes +* a domain is split across multiple services +* a provider is replaced +* a raw source moves behind a compatibility seam +* or a service is renamed, decomposed or merged + +The central change in this scheme is away from “whatever topic tree the current service happens to publish” and towards a control plane with explicit abstraction planes, explicit ownership and explicit provenance. One day this might become graph view projections over canonical objects, but not today! + +## 1. Purpose + +This specification defines the canonical control-plane naming model for Devicecode. + +It gives stable answers to these questions: + +* how are service lifecycle and intended configuration named? +* how is raw provenance-bearing truth separated from canonical public truth? +* how is appliance composition separated from domain policy state? +* how are stable public interfaces separated from raw provider-native interfaces? +* how are workflow manager interfaces separated from retained workflow instance records? +* how are durable desired state, retained workflow truth and immediate controls kept distinct? +* how is canonical public ownership declared? +* how are compatibility aliases handled during migration? +* how do public contracts survive changes in implementation placement, transport, decomposition and provider choice? + +This specification governs control-plane naming, ownership and abstraction boundaries. It absolutely doesn't redefine fibers, bus semantics, scope semantics or service lifecycle rules. + +## 2. Architectural position + +This model preserves the non-negotiable Devicecode rules: + +* services are implementation boundaries +* HAL is the only OS and hardware boundary +* services depend on interfaces, not on each other’s internals +* each service owns its own lifecycle and scoped connection + +The following architectural rules are explicit. + +> `device` is the authoritative appliance composition service. + +`device` composes appliance-local truth from raw local, internal, imported and software-defined parts. + +> Raw provenance-bearing surfaces (eg. hardware modem capability path) and curated public interfaces (eg. modem-1) MUST remain structurally segregated. + +Metadata alone is not sufficient to preserve this distinction. + +> Canonical retained domain truth MAY exist outside `device` where a domain service genuinely owns it. + +Not all public retained truth is appliance composition truth. + +> A service is an implementation boundary. A service is not, by itself, a public naming boundary. + +A service MAY own one or more public naming families, but those families are justified by semantic ownership, not by the service name alone. + +> `state/` names a semantic public domain, not the implementing service. + +A domain family MUST NOT be introduced merely by copying the name of the service that currently publishes it. + +> Durable intended behaviour SHOULD normally be expressed declaratively and reconciled by the owning service. + +Imperative controls are the exception, not the default. For example, a modem is disabled by changing config, not by flipping a switch. + +> Compatibility seams MAY exist at controlled branching points during migration. + +A compatibility seam MUST preserve canonical ownership and MUST NOT become a second public architecture. + +## 3. Normative language + +The key words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT** and **MAY** are to be interpreted as normative requirements. + +## 3.1 Canonical, summary and compatibility surfaces + +The following distinctions are normative. + +* A **canonical surface** is the one authoritative public surface for an abstraction role. +* A **summary surface** is a deliberately reduced public projection of richer canonical truth. +* A **compatibility alias** is a temporary additional surface retained only to preserve compatibility during migration. + +Compatibility aliases MUST: + +* declare which canonical surface they mirror +* preserve the abstraction role of the canonical surface +* be implemented as one-way projections from canonical truth +* not become the dependency target of new work +* be sunsetted as soon as practicable + +Summary surfaces MUST: + +* clearly differ in abstraction role from the canonical surface they summarise +* not pretend to be the authoritative home of richer truth + +## 4. Canonical top-level roots + +The canonical top-level roots are: + +* `svc` +* `cfg` +* `raw` +* `state` +* `cap` +* `obs` + +No new top-level root SHOULD be introduced without strong justification. + +`obs` is versioned below the root. The canonical initial observability plane is `obs/v1/...`. + +`state` is the root for canonical retained state families. + +The initial required canonical `state/...` families are: + +* `state/device/...` +* `state/workflow/...` + +Common additional domain families include: + +* `state/update/...` +* `state/fabric/...` +* `state/gsm/...` +* `state/net/...` +* `state/wifi/...` +* `state/time/...` +* `state/site/...` + +A narrow `state/ui/...` family MAY exist where the UI service owns genuine canonical retained UI-domain truth such as operator-facing session or model summaries. It MUST remain narrow and MUST NOT become the naming boundary for workflows owned elsewhere. + +Any `state//...` family MUST have one clearly identified owning service. + +## 5. Canonical token form + +Topics are represented in code as dense token arrays. + +Slash form MAY be used in documentation. + +Example: + +* documentation: `cap/update-manager/main/rpc/create-job` +* code: `{ 'cap', 'update-manager', 'main', 'rpc', 'create-job' }` + +### 5.1 Token rules + +* tokens MUST be lowercase +* multiword tokens MUST use kebab-case +* method names MUST use verbs in kebab-case +* curated public identifiers SHOULD be semantic and stable for the intended audience +* raw identifiers MAY be concrete, topology-bearing, provider-facing or transport-bearing where provenance requires it + +## 6. Identifier scope and stability + +Every identifier used in control-plane naming MUST have an intended scope and stability class. + +At minimum, the following distinctions apply: + +* **service ids** are runtime-local implementation identifiers +* **source ids** identify concrete provenance boundaries under `raw/...` and MAY be topology-bearing +* **component ids** identify appliance-level parts and SHOULD remain stable across routine reprovisioning, transport changes and provider replacement +* **domain ids** identify semantic domain-owned public instances, roles or summaries and SHOULD be stable for the intended domain audience +* **curated interface ids** identify stable local public interfaces under `cap/...` +* **raw ids** identify provider-native concrete interfaces and MAY be provider-facing or topology-bearing +* **workflow instance ids** identify retained operation records and MUST be unique within their workflow family + +An identifier MUST NOT be reused casually across scopes with different meanings. + +### 6.1 Service ids and domain ids + +Service ids and domain ids are distinct. + +* A **service id** identifies a runtime-local implementation boundary. +* A **domain id** identifies a stable semantic public audience or meaning. + +A service id MUST NOT be assumed to define the correct `state/` family name. + +A service MAY own canonical truth for a domain whose name differs from the service id. + +A domain family SHOULD remain meaningful even if: + +* the implementing service is renamed +* implementation is split across multiple services +* implementation is merged into another service +* the backing provider changes + +### 6.2 Component ids and role ids + +Component ids and role ids are distinct. + +A component id identifies an appliance part, for example: + +* `modem-1` +* `wifi-radio-2` +* `ethernet-port-1` + +A role or semantic domain id identifies a domain-owned public meaning, for example: + +* `primary` +* `secondary` +* `wan` +* `guest` +* `mesh` + +A component id MUST NOT implicitly carry a domain role meaning. + +A domain role mapping SHOULD be published explicitly by the domain service that owns that role. + +## 7. Core address planes + +## 7.1 `svc`: service lifecycle + +Use `svc` for service lifecycle only. + +Topics: + +* `svc//status` +* `svc//meta` + +This plane answers: + +* is the service running? +* is it degraded? +* why did it fail? +* what build or version is it? + +No business state, workflow state or configuration intent belongs under `svc`. + +`svc/...` SHOULD remain intentionally boring. + +## 7.2 `cfg`: intended service configuration and declarative policy input + +Use `cfg` for the intended configuration and declarative policy input currently supplied to a service. + +Topics: + +* `cfg/` + +This plane answers: + +* what configuration is this implementation boundary expected to reconcile against? +* what durable intended behaviour is this service being asked to achieve? + +`cfg` is not persistence itself. Persistence MAY be provided by a capability, but `cfg/` is the canonical intended-configuration plane. + +`cfg/` is not the general plane for “effective configuration currently in force”. If a service needs to distinguish between: + +* intended configuration +* currently effective configuration +* last successfully applied or reconciled state + +that distinction MUST be represented explicitly in canonical retained state rather than left ambiguous in `cfg/`. + +A configuration change MUST NOT be treated as a hidden imperative shortcut without the owning service’s reconciliation semantics. + +## 7.3 `raw`: provenance-bearing truth and raw source-scoped interfaces + +Use `raw` for concrete provenance-bearing truth and raw provider-native interfaces. + +Topics: + +* `raw///meta` +* `raw///status` +* `raw///state/...` +* `raw///cap///meta` +* `raw///cap///status` +* `raw///cap///state/` +* `raw///cap///event/` +* `raw///cap///rpc/` + +This plane answers: + +* what concrete source exists? +* what did it actually publish or expose? +* what provider-native interfaces does it expose? +* what imported remote surfaces exist locally as provenance-bearing sources? + +At minimum, each source MUST publish: + +* `raw///meta` +* `raw///status` + +### 7.3.1 Source kind semantics + +`` identifies what sort of source this is. + +Initial common values include: + +* `host` +* `member` +* `peer` +* `software` + +This vocabulary SHOULD remain small and stable. + +Relation, placement and trust qualifiers such as `local`, `internal`, `federated`, `derived`, `managed` or `owner` SHOULD normally live in metadata, not in the path, unless there is strong justification for path-level distinction. + +### 7.3.2 Raw source-wide state + +`raw///state/...` is for source-owned raw facts that are not naturally attached to a specific provider-native interface. + +It MUST remain narrow. + +It MUST NOT become a miscellaneous compatibility tree or a dumping ground for inconvenient facts. + +Where a fact belongs to a raw provider-native interface, it SHOULD live under: + +* `raw///cap///state/` + +not under the source-wide state tree. + +Legitimate source-wide raw state typically includes: + +* source incarnation or generation +* session or link state +* source-wide health +* transport state +* source trust or assurance state +* source boot state +* source-wide diagnostics not naturally attached to a particular interface + +### 7.3.3 Imported remote interfaces + +Imported remote interfaces, even if curated at their point of origin, are locally treated as provenance-bearing source-scoped surfaces under `raw/...` unless and until deliberately re-curated into a local public alias. + +### 7.3.4 Source ids + +A source id under `raw///...` MUST identify a concrete provenance boundary as observed locally. + +A source id: + +* MAY be topology-bearing +* MAY be transport-bearing where provenance requires it +* SHOULD remain stable for the local runtime while that source identity remains meaningful +* SHOULD NOT be re-used for a different concrete source merely because it occupies the same transport position later + +Imported remote sources SHOULD normally be assigned a stable local source id by the importing service rather than exposing remote naming or transport details directly as the canonical local source id. + +## 7.4 `state`: canonical retained state families + +Use `state/...` for canonical retained state families. + +This plane is for canonical public retained truth, not raw provider-native truth and not merely interface availability. + +The initial required families are: + +* `state/device/...` +* `state/workflow/...` + +Additional domain families MAY be introduced where a service owns canonical public retained truth for its domain. + +## 7.4.1 `state/device`: composed appliance truth + +Use `state/device/...` for composed appliance-local truth. + +This is the canonical place to ask: + +* what appliance is this? +* what components does it have? +* what is their current state? +* what appliance-level summaries are true? + +Only `device` publishes `state/device/...`. + +Typical topics include: + +* `state/device/identity` +* `state/device/components` +* `state/device/component/` +* `state/device/component//software` +* `state/device/component//update` + +`state/device/...` is for appliance-facing composed truth. It is not a dumping ground for raw provider state, workflow state or unrelated domain policy. + +## 7.4.2 `state/workflow`: retained workflow truth + +Use `state/workflow/...` for retained workflow instance records. + +Examples: + +* `state/workflow/update-job/` +* `state/workflow/update-job//timeline` +* `state/workflow/artifact-ingest/` + +Workflow state is public truth about concrete operations, not manager interfaces. + +Not every action needs a workflow instance. This family SHOULD be used where the retained record is justified by the criteria in section 11. + +## 7.4.3 `state/`: canonical retained domain truth + +Use `state//...` where a service owns canonical public retained truth for a semantic domain. + +`state/` is **not** a synonym for `state/`. + +A domain family names the public domain audience and meaning of the truth, not the current implementation module or service boundary. + +Therefore: + +* `state/` MUST NOT be introduced merely because a service of the same name exists +* renaming, splitting or merging services SHOULD NOT by itself require renaming a `state/` family +* if the only justification for a family name is “this service publishes it”, that family name is not yet justified + +Examples may include: + +* `state/gsm/role/primary` +* `state/gsm/modem/modem-1` +* `state/net/interface/wan` +* `state/net/dns` +* `state/wifi/ap/main` +* `state/site/member/` +* `state/update/summary` +* `state/update/component/mcu` +* `state/fabric/link/` + +A `state//...` family MUST: + +* have one clear owning service +* represent canonical public retained truth for that domain +* not merely restate raw provider-native truth +* not merely duplicate appliance composition truth +* not merely act as interface availability +* not merely be a workflow instance record + +## 7.5 `cap`: stable public interfaces + +Use `cap/...` for stable local public callable and inspectable interfaces intended for ordinary consumers. + +Topics: + +* `cap///meta` +* `cap///status` +* `cap///rpc/` +* optionally `cap///event/` +* narrowly and by exception, optionally `cap///state/` + +This plane answers: + +* what stable local public interfaces exist? +* what methods can callers invoke? +* what broad interface-level condition applies? +* where justified, what small amount of interface-scoped retained state is part of the public contract? + +Every address under `cap/...` MUST be intended as a stable public contract. + +`cap/...` is the canonical local public interface plane. It includes both: + +* stable feature interfaces +* stable manager interfaces + +Examples include: + +* `cap/modem/modem-1/rpc/connect` +* `cap/update-manager/main/rpc/create-job` +* `cap/artifact-ingest/main/rpc/commit` + +Provider-native or topology-bearing concrete interfaces MUST NOT appear directly under `cap/...`. + +`cap/...` is not a general retained state plane. + +### 7.5.1 Ownership of `cap` + +Any service MAY publish under `cap/...` where it owns a stable semantic public interface. + +`device` MAY publish curated aliases for appliance-defining features. + +Non-device services MAY publish stable public interfaces they own, including manager interfaces and domain feature interfaces. + +`device` is authoritative for appliance composition. It is not the mandatory owner of every public interface. + +### 7.5.2 Capability status + +`cap///status` is for interface availability and broad operational condition only. + +It MAY describe, for example: + +* available or unavailable +* degraded +* broad reason or mode +* interface version or compatibility summary where useful + +It MUST NOT be used as a substitute for canonical retained state. + +In particular, `cap/.../status` MUST NOT quietly become a general state tree under another name. + +`cap/.../status` SHOULD usually be summary-level and low-cardinality. + +### 7.5.3 Capability events + +`cap///event/` MAY be used where a curated interface has meaningful interface-level events. + +Events are optional. + +They MUST NOT be used as authoritative retained state. + +### 7.5.4 Narrow interface-scoped retained state + +Some stable public interfaces MAY, by exception, publish a small amount of retained interface-scoped state under: + +* `cap///state/` + +This is appropriate only where all of the following are true: + +* the state is genuinely part of the interface contract rather than broad domain truth +* the state is naturally scoped to that one interface +* publishing it under `cap/...` materially improves inspectability or usability +* the interface-scoped state is not being used as the hidden canonical home of a larger retained state family + +Examples that MAY be justified include: + +* current ingest progress for a narrow public ingest interface +* current broad mode of a stable public manager interface +* a small interface-scoped summary that is meaningful to ordinary callers + +If a richer retained state model exists, that richer canonical truth SHOULD usually live under: + +* `state/device/...` +* `state/workflow/...` +* `state//...` + +and `cap/.../state/...` SHOULD remain summary-level. + +If `cap/.../state/...` grows beyond a small interface adjunct, the design SHOULD be reconsidered. + +### 7.5.5 Curated aliases and provenance + +Where a curated public interface is backed by raw sources or raw provider-native interfaces, its `meta` SHOULD include references to those backing sources and interfaces. + +Curated aliases MUST NOT erase provenance completely. + +### 7.5.6 Common public interface patterns + +The following patterns are recommended. + +* Manager interfaces SHOULD use semantic manager classes such as: + + * `cap/update-manager//...` + * `cap/artifact-ingest//...` + * `cap/transfer-manager//...` + +* Curated appliance-defining feature aliases published by `device` SHOULD normally use: + + * `cap/component//...` + +* Provider-native interfaces MUST NOT be exposed directly under these curated classes. Where a provider-native interface is intentionally re-curated as a stable public alias, the alias `meta` SHOULD reference the backing raw source and raw interface. + +* Stable utility interfaces MAY exist under `cap/...` only where they are intentionally public and are not merely convenient wrappers around raw provider-native utilities. + +## 8. Declarative state, workflows and immediate controls + +The normal Devicecode control-plane pattern is: + +* durable intended behaviour enters through configuration +* services reconcile that intended behaviour into observed state +* structured operations are managed through stable public manager interfaces +* narrow immediate controls exist only where they are clearly justified + +### 8.1 Durable desired state + +Durable intended behaviour SHOULD normally be represented declaratively in service configuration or other canonical retained state owned by the appropriate service. + +Examples include: + +* modem preference and role policy +* APN policy +* network role assignment +* Wi-Fi configuration +* PoE policy +* update policy +* telemetry export policy + +Public systems MUST NOT accumulate ad hoc imperative command trees for things that are really durable policy changes. + +For update systems, durable desired behaviour such as bundled-image preference, approval policy or automatic reconcile policy SHOULD normally live in `cfg/update`. + +`state/update/...` is for canonical retained update-domain truth such as current summary state, component-level update summaries or reconcile outcomes. It is not the canonical home of desired intent. + +### 8.2 Workflow-managed operations + +Workflow-managed operations are represented through stable public manager interfaces under `cap/...` and retained workflow instance records under `state/workflow/...`. + +These are appropriate for operations that are at least one of: + +* asynchronous +* multi-step +* reboot-spanning +* auditable +* operator-visible +* worth retaining after the initiating call completes + +Examples include: + +* uploaded artefact ingest +* update job creation +* update job approval +* update job commit +* support bundle generation +* onboarding and bootstrap flows + +Workflow managers are not substitutes for durable policy. Durable policy belongs in configuration or canonical retained domain state, depending on ownership. + +### 8.3 Immediate controls + +Immediate controls are narrow, explicit actions that do not fit durable desired state and do not necessarily require a retained workflow instance. + +Examples include: + +* reboot the device +* power-cycle a member +* commit a prepared update job +* force a rescan +* acknowledge or clear a transient condition + +An immediate control MUST NOT silently create durable policy unless that is its explicit and documented purpose. + +### 8.4 Operational overrides + +A service MAY define retained operational overrides where temporary intended behaviour needs to be represented declaratively without becoming long-lived user configuration. + +Such overrides MUST be: + +* explicitly modelled +* clearly owned by one service +* distinguishable from durable configuration +* distinguishable from immediate controls + +## 9. Curated and raw segregation rules + +The following rules are normative. + +1. All stable public callable interfaces MUST live under `cap/...`. +2. All provider-native callable or inspectable interfaces MUST live under `raw/.../cap/...`. +3. No provider-native interface MAY also appear directly under `cap/...` without deliberate curation into a distinct alias. +4. Ordinary business services SHOULD depend on `cap/...`, not on `raw/...`. +5. `device` and other composing or orchestrating services MAY depend on `raw/...` where required. +6. Structural segregation MUST be preserved even where metadata could in principle distinguish the surfaces. + +## 10. Device composition rules + +The central composition rule is: + +> Raw sources publish raw truth. +> Raw providers publish raw source-scoped interfaces. +> `device` composes appliance truth. +> Curated public interfaces are stable semantic contracts, not mirrors of raw topology. + +### 10.1 What `device` publishes + +`device` publishes: + +* composed appliance truth under `state/device/...` +* explicit references to backing sources and interfaces where appropriate +* a curated subset of appliance-defining aliases under `cap/...` + +### 10.2 What `device` MUST NOT do + +`device` MUST NOT: + +* mirror every raw source-scoped interface under a curated alias +* erase provenance completely +* absorb unrelated domain policy merely because it composes appliance state +* turn component ids into domain role ids + +`device` is the authoritative appliance composition service. It is not the universal owner of all policy domains or all public interfaces. + +## 11. Workflow rules + +A workflow-managing public interface is a stable public manager interface under `cap/...`. + +Examples include: + +* `cap/update-manager/main/rpc/create-job` +* `cap/update-manager/main/rpc/commit-job` +* `cap/artifact-ingest/main/rpc/create` +* `cap/artifact-ingest/main/rpc/append` +* `cap/artifact-ingest/main/rpc/commit` +* `cap/artifact-ingest/main/rpc/abort` + +A retained workflow record lives under: + +* `state/workflow/...` + +The manager is not the instance. + +Not every action needs a workflow instance. Workflow records SHOULD be used for operations that are at least one of: + +* asynchronous +* multi-step +* reboot-spanning +* auditable +* operator-visible +* worth retaining after the initiating call completes + +Public workflows MUST NOT be exposed through ad hoc service-named command trees such as: + +* `cmd/update/...` +* `cmd/ui/upload/...` + +## 12. Canonical ownership rule + +A fact MUST have one canonical public owner within its abstraction plane. + +In particular: + +* `raw/...` owns raw provenance-bearing truth +* `state/device/...` owns composed appliance truth +* `state/workflow/...` owns retained workflow instance truth +* `state//...` owns canonical retained domain truth +* `cap/...` owns stable public interface contracts and broad interface status + +If related truth appears in more than one plane, each occurrence MUST be justified by a distinct abstraction role. + +For example: + +* raw provider truth under `raw/...` +* composed appliance truth under `state/device/...` +* retained workflow instance truth under `state/workflow/...` +* retained domain truth under `state//...` + +may all legitimately describe the same real-world situation at different levels. + +What is prohibited is casual mirroring of the same abstraction-role fact across multiple planes for convenience. + +If equivalent truth appears in more than one place within the same abstraction role, one surface MUST be explicitly canonical and the others MUST be summaries, projections or compatibility aliases. + +## 13. Minimal publication requirements + +The following rules are normative. + +1. Every service publishes: + + * `svc//status` + * `svc//meta` + * observability under `obs/v1//...` + +2. Intended service configuration is published under: + + * `cfg/` + +3. Only `device` publishes: + + * `state/device/...` + +4. Raw provenance-bearing truth is published under: + + * `raw/...` + +5. Every source publishes: + + * `raw///meta` + * `raw///status` + +6. All stable public callable interfaces use: + + * `cap///rpc/` + +7. Every stable public curated interface publishes: + + * `cap///meta` + * `cap///status` + +8. All provider-native callable interfaces use: + + * `raw///cap///rpc/` + +9. Retained workflow truth lives under: + + * `state/workflow/...` + +10. Curated and raw capability surfaces MUST remain structurally segregated. + +## 14. Decision guide + +Use this quick test. + +* Is this service lifecycle or build state? + Use `svc/...` + +* Is this intended configuration or declarative policy input for one service? + Use `cfg/...` + +* Is this raw provenance-bearing truth from a concrete source? + Use `raw/...` + +* Is this a raw provider-native callable or inspectable interface? + Use `raw/.../cap/...` + +* Is this composed appliance truth? + Use `state/device/...` + +* Is this canonical retained truth for a semantic public domain? + Use `state//...` + +* Is this a retained record of a specific long-running or operator-visible operation? + Use `state/workflow/...` + +* Is this a stable local public interface? + Use `cap/...` + +* Is this an immediate control that is neither durable desired state nor a retained workflow? + Use a narrow stable public interface under `cap/...` + +* Is this logs, audit or observability rather than operational truth? + Use `obs/v1/...` + +### 14.1 `state/` versus service name + +Before choosing ``, ask: + +* would this family still have the same public meaning if the implementing service were renamed? +* would this family still have the same public meaning if implementation were split across multiple services? +* is the audience for this truth a domain audience rather than an implementation audience? + +If the answer is no, this is probably not a `state/` family. + +### 14.2 Public interface versus canonical retained state + +When choosing between `cap/...` and `state/...`, apply this additional test. + +Use `cap/...` when the thing you are naming is primarily: + +* a callable method surface +* an inspectable interface contract +* a manager for creating or controlling operations +* a stable local alias for a semantic feature + +Use `state//...` or `state/device/...` when the thing you are naming is primarily: + +* canonical retained truth for a domain or appliance audience +* a durable public summary of current reality +* a role mapping, selection result or composed view +* a state model that should remain meaningful even if the backing interface changes + +If both exist, be explicit about which is canonical for which purpose: + +* `cap/...` is canonical for invoking and inspecting the interface contract +* `state/...` is canonical for retained public truth + +## 15. Compatibility publication + +Compatibility aliases MAY be published temporarily during migration. + +The following rules are normative. + +1. A compatibility alias MUST name its canonical source in metadata or in the owning service documentation. +2. A compatibility alias MUST be implemented as a one-way projection from canonical truth. +3. New consumers MUST depend on canonical surfaces, not on compatibility aliases. +4. Compatibility aliases MUST NOT be used to bypass raw-versus-curated segregation. +5. Compatibility aliases SHOULD be removed once dependent consumers have migrated. + +Compatibility publication is a migration tool, not a long-term architecture pattern. + +## 16. Migration rule + +Older public `cap/...` surfaces MAY remain as legacy-compatible public interface names where already shipped. + +New work SHOULD adopt this model. + +Compatibility aliases MAY exist temporarily, but canonical ownership MUST still be declared. + +Raw-versus-curated segregation SHOULD improve over time and MUST NOT be broken for convenience. + +## 17. Where we are now + +Devicecode is now in the middle of a concrete migration to this model. + +The current state after the `update-migration` PR stack establishes: + +- `raw/...` as the home of provenance-bearing imported member and host/provider surfaces +- `state/device/...` as the canonical appliance composition plane +- `state/workflow/...` as the retained workflow plane +- `state/update/...` and `state/fabric/...` as domain-level retained summary planes +- stable public managers such as: + - `cap/update-manager/...` + - `cap/artifact-ingest/...` + - `cap/transfer-manager/...` +- curated component interfaces under: + - `cap/component//...` +- compatibility seams at controlled branch points to project older surfaces from canonical truth during migration + +This means the model is being landed through a stacked set of implementation PRs. + +The main remaining work is not conceptual discovery but: + +* tightening canonical ownership +* reducing compatibility publication +* and continuing migration of older surfaces and consumers + +## 18. Next steps + +The next steps are: + +1. make canonical ownership explicit for all major published families +2. ensure new work depends only on canonical surfaces +3. narrow compatibility seams so that they remain migration tools rather than parallel architectures +4. retire legacy publication from raw/provider-facing areas before summary or observability aliases +5. strengthen tests so that they assert primarily on canonical surfaces +6. continue reviewing `cap/.../state/...` and `raw/.../state/...` for misuse +7. continue reviewing candidate `state/` families to ensure they are truly domain families, not service-local trees under another name + +In practical terms, the near-term focus is: + +* remove dependence on old `cmd/...` trees +* reduce legacy mirroring of provider-native capabilities +* make summary versus canonical versus compatibility surfaces explicit +* and keep `device` focused on appliance composition rather than raw mirroring or domain ownership creep + +## 19. Migration appendix: worked examples + +This appendix is non-normative, but strongly recommended. + +### 19.1 Current migrated update manager + +Manager interfaces: + +* `cap/update-manager/main/rpc/create-job` +* `cap/update-manager/main/rpc/start-job` +* `cap/update-manager/main/rpc/commit-job` +* `cap/update-manager/main/rpc/cancel-job` +* `cap/update-manager/main/rpc/retry-job` +* `cap/update-manager/main/rpc/discard-job` + +Retained workflow truth: + +* `state/workflow/update-job/` +* `state/workflow/update-job//timeline` + +Retained domain summaries: + +* `state/update/summary` +* `state/update/component/mcu` + +Durable desired bundled behaviour belongs in: + +* `cfg/update` + +not in retained workflow state. + +### 19.2 Current migrated artifact ingest + +A public ingest flow is modelled as: + +* `cap/artifact-ingest/main/rpc/create` +* `cap/artifact-ingest/main/rpc/append` +* `cap/artifact-ingest/main/rpc/commit` +* `cap/artifact-ingest/main/rpc/abort` + +Retained ingest truth lives under: + +* `state/workflow/artifact-ingest/` + +The UI service may transport operator input to this manager, but the UI service is not the public naming boundary of the ingest workflow. + +### 19.3 Current migrated fabric and device + +Raw imported member truth: + +* `raw/member/mcu/meta` +* `raw/member/mcu/status` +* `raw/member/mcu/state/...` +* `raw/member/mcu/cap/updater/main/...` + +Fabric domain summaries: + +* `state/fabric/link/` +* `state/fabric/transfer/` +* `cap/transfer-manager/main/rpc/send-blob` + +Device-composed appliance truth: + +* `state/device/identity` +* `state/device/component/mcu` +* `state/device/component/mcu/software` +* `state/device/component/mcu/update` + +Curated device-facing control: + +* `cap/component/mcu/rpc/prepare-update` +* `cap/component/mcu/rpc/stage-update` +* `cap/component/mcu/rpc/commit-update` + +### 19.4 HAL host/provider utilities + +Raw host/provider utility surfaces may include: + +* `raw/host/platform/cap/artifact-store/main/...` +* `raw/host/platform/cap/control-store/update/...` +* `raw/host/platform/cap/updater/cm5/...` + +Curated public manager interfaces may include: + +* `cap/artifact-ingest/main/...` +* `cap/control-store/update/...` + +Canonical retained truth remains under: + +* `state/update/...` +* `state/workflow/...` + +### 19.5 Ryan's GSM tree + +Provider-native host modem interfaces exposed directly as public capabilities SHOULD be reviewed and split into: + +* raw provider-native interfaces under `raw/host//cap/modem//...` +* canonical GSM domain truth under `state/gsm/...` +* stable public semantic interfaces under `cap/...` where deliberate curation is intended (for example, `device` owning the replublishing of the raw `hal` provided capability into `modem-1`/`modem-2`) + +Examples of canonical GSM retained state may include: + +* `state/gsm/role/primary` +* `state/gsm/modem/modem-1` +* `state/gsm/apn/selected` +* `state/gsm/uplink/primary` + +This allows the GSM service to remain the canonical owner of GSM domain truth while preserving raw provider provenance under `raw/...`. + +## 20. Conclusion + +The canonical Devicecode naming model is: + +* `svc/...` for service lifecycle +* `cfg/...` for intended service configuration and declarative policy input +* `raw/...` for raw provenance-bearing truth and raw source-scoped interfaces +* `state/device/...` for composed appliance truth +* `state/workflow/...` for retained workflow records +* `state//...` for canonical retained domain truth +* `cap/...` for stable public interfaces +* `obs/v1/...` for observability + +Within this model: + +* services remain implementation boundaries +* HAL remains the only OS and hardware boundary +* `device` is the authoritative appliance composition service +* raw sources preserve provenance +* curated and raw interfaces remain structurally segregated +* workflows are managed through stable public interfaces and recorded as workflow instances +* canonical retained domain truth is allowed where a domain service genuinely owns it +* `state/` names semantic public domains, not service ids +* durable intended behaviour is expressed declaratively by default +* immediate controls remain narrow and explicit +* component ids remain distinct from role ids +* identifier scope and stability are explicit rather than assumed +* canonical, summary and compatibility surfaces remain distinct +* and the distinction between interface contracts and canonical retained truth remains explicit + +## Wired provider naming + +Read-only or writable wired hardware providers publish observations under +`raw/host/wired/provider//...` when provided by the local +HAL. Device publishes the product physical assembly under `state/device/assembly`; +Wired combines assembly with observations and publishes the appliance-level wired +view under `state/wired/...`. A future switch-fabric member running devicecode +may import equivalent raw facts under `raw/member//...`, but the public +wired contract remains `state/wired/...`. + +The appliance-level wired view should use stable product surface identifiers such as `cm5-eth0`, `switch-uplink-cm5`, +`lan-1` and `lan-2`. + +## GSM uplink state consumed by NET + +GSM publishes curated semantic uplink state under: + +```text +state/gsm/uplink/ +``` + +`` is stable product identity, for example `primary` or `secondary`. It +must not be a volatile Linux device name. The Linux interface name observed from +ModemManager or the kernel is payload data, normally `linux.ifname`, and may +change between boots. + +NET may consume this state as its GSM source contract. NET should not consume +raw modem or HAL provider topics for WAN source identity. + diff --git a/docs/service-architecture.md b/docs/service-architecture.md new file mode 100644 index 00000000..32ebbef1 --- /dev/null +++ b/docs/service-architecture.md @@ -0,0 +1,1202 @@ +# Devicecode service architecture guide + +> **Basis of this note** +> +> This architecture follows the `fibers` doctrine. +> +> It relies on the observed runtime contracts that: +> +> * `fibers.perform` is scope-aware; +> * `fibers.spawn` spawns into the current scope; +> * `perform_raw` bypasses scope-aware cancellation and belongs to infrastructure; +> * scope cancellation is downward only; +> * scope cancellation is cooperative; +> * finalisers cannot yield, only `perform` with an `:or_else()` immediate fallback is permitted; +> * parentage is the structural ownership mechanism; +> * attached child scopes are joined by their parent’s join/finalisation path; +> * `join_op()` is active finalisation, not passive observation; +> * non-parent joining must be an explicit service discipline; +> * `run_scope_op` is useful for deliberate whole-operation waits, but not for strict coordinator branches; +> * `named_choice`, `choice`, and `first_ready` select readiness, not semantic priority; +> * Lua table order must not be used as a priority mechanism; +> * where semantic priority is **genuinely** required, Device uses `devicecode/support/priority_event.lua`; +> * finalisers must release owned resources without waiting. + +## Purpose + +This guide is the canonical architecture guide for Devicecode services built on `fibers` and `bus`. + +It defines the expected shape and programming style for Fabric, Device, Update, HTTP, UI and future services. Individual service notes should describe only the service-specific application of this guide. + +The core rule is: + +```text +Ops describe possible waits. +Scopes own lifetimes. +Coordinators decide. +Workers wait. +Finalisers terminate. +Models expose observable state. +Callbacks stay at the edge. +``` + +Devicecode services are written as explicit state machines with visible waiting points. The aim is not to avoid concurrency. The aim is to make concurrency inspectable. + +## Core abstractions + +```text +Op = a first-class possible wait +Scope = a lifetime, cancellation, cleanup, joining and failure boundary +Model = observable state, not a worker +Bus = local control-plane transport +Handle = an owned resource with immediate termination +``` + +Ordinary service code waits with: + +```lua +local ev = fibers.perform(next_event_op(state)) +``` + +This means: + +```text +wait for the next service event while the current scope remains healthy +``` + +Do not use `perform_raw` in service code. It is infrastructure. + +## Canonical service shape + +```mermaid +flowchart TD + Main["devicecode main"] --> ServiceScope["service scope"] + + ServiceScope --> Coordinator["service coordinator"] + ServiceScope --> Model["service-owned model"] + ServiceScope --> Publisher["state/cap/svc publisher"] + ServiceScope --> Generation["current generation scope"] + ServiceScope --> Durable["optional service-owned durable runtime"] + ServiceScope --> Active["optional service-owned active runtime"] + + Generation --> Observers["observer scopes"] + Generation --> Endpoints["public endpoint bindings"] + Generation --> Requests["request scopes"] + Generation --> Components["component scopes"] + + Requests --> Workers["blocking workers"] + Components --> Workers + + Publisher --> Bus["bus retained state and capabilities"] +``` + +A service usually has: + +```text +service scope + coordinator + service-owned model + service-owned bus connection + publication cleanup + current generation scope + request scopes + worker scopes +``` + +Introduce a scope when work has any of: + +```text +identity +owned resources +cleanup +child work +blocking I/O +timeout policy +retry policy +diagnostics +caller-visible result +durable side effects +ownership transfer +``` + +A fibre runs code. A scope owns lifetime. + +## Service entry contract + +A service launched by `main.lua` has a foreground entry point: + +```text +start(conn, opts) + creates service lifecycle state + enters the long-lived service coordinator + must not return while the service is healthy +``` + +The lower-level coordinator body is: + +```text +run(scope, params) + owns the service event loop + normally blocks until cancellation or failure +``` + +A helper that returns a local handle must use an explicit name such as +`open_handle`, `new_handle`, or `open_component`. It must not be called +`start`. If `start()` or `run()` returns in a healthy path, the service should +publish a stopped/failed lifecycle state and raise an error. + +## Programming style + +Devicecode service code should be direct, explicit and state-machine shaped. + +Prefer: + +```text +small modules with clear ownership +plain tables for event records and model snapshots +explicit topic helper functions +explicit request owner objects +explicit resource ownership wrappers +explicit _op functions for waits +explicit terminate(reason) methods for finaliser-safe cleanup +``` + +Avoid: + +```text +callbacks as service control flow +hidden suspension +implicit global state +backend objects escaping into public APIs +long call chains that both decide and wait +boolean flags whose ownership is unclear +fire-and-forget fibres with no scope identity +ad hoc topic literals scattered across service logic +``` + +A good service reads as a sequence of ownership and event transitions: + +```lua +local ev = fibers.perform(next_event_op(state)) +local decision = reduce_event(state, ev) +dispatch_immediate_effects(state, decision.effects) +``` + +The service should make it clear: + +```text +who owns the resource +who may cancel the work +where the result is stored +who reports completion +what happens if reporting fails +what happens if the operation loses a choice +``` + +## Avoid callbacks in service logic + +Callbacks are allowed only at infrastructure edges, such as: + +```text +transport driver callbacks +backend wake-up hooks +foreign event-loop integration +low-level command-loop plumbing +test fakes modelling callback-driven backends +``` + +Callbacks must not be used as ordinary service control flow. + +A callback must not: + +```text +call a service reducer +mutate service coordinator state +perform a Fibers Op +call fibers.perform or perform_raw +run a bus RPC +publish canonical service state +resolve public requests directly unless it owns the request object by construction +``` + +A callback may: + +```text +update backend-local state +mark a backend command complete +wake an Op-facing boundary +terminate a backend-local resource +enqueue an already-shaped service event through a narrow immediate port +``` + +The preferred pattern is: + +```mermaid +flowchart LR + Backend["backend callback"] --> Boundary["driver-local boundary"] + Boundary --> Op["Fibers Op becomes ready"] + Op --> Worker["worker/request scope"] + Worker --> Event["service event"] + Event --> Coordinator["coordinator reduction"] +``` + +Do not write: + +```lua +backend.on_event(function(raw) + state.model:update(raw) -- bad: callback mutates service state + conn:retain(topic, raw) -- bad: callback publishes canonical truth +end) +``` + +Prefer: + +```lua +local ev = fibers.perform(driver:event_op()) +port:emit({ + kind = "backend_event", + generation = generation, + payload = ev, +}) +``` + +Then let the coordinator decide. + +## Do not hide suspension + +These functions must not yield: + +```text +wrap functions +guard builders +or_else fallback thunks +abort handlers +finalisers +waitable steps +model update functions +coordinator branches +immediate publication helpers +validation functions +topic helper functions +``` + +`wrap` transforms a result. It must not perform another wait. + +Good: + +```lua +stream:read_line_op():wrap(parse_line) +``` + +Bad: + +```lua +stream:read_line_op():wrap(function(line) + return fibers.perform(read_body_op(stream, line)) +end) +``` + +If a sequence needs multiple waits, put it in a worker or operation scope: + +```lua +local function read_request_op(stream) + return fibers.run_scope_op(function(scope) + local line = fibers.perform(stream:read_line_op()) + local body = fibers.perform(read_body_op(stream, line)) + return { line = line, body = body } + end) +end +``` + +## Coordinator discipline + +A coordinator should have one suspending control point: + +```lua +while true do + local ev = fibers.perform(next_event_op(state)) + + local decision = reduce_event(state, ev) + dispatch_immediate_effects(state, decision.effects) +end +``` + +Coordinator branches may: + +```text +mutate coordinator-owned state +record pending work +start scoped work +cancel child scopes +ignore stale completions +resolve already-owned in-memory requests +publish through immediate publication helpers +update models +terminate local handles immediately +``` + +Coordinator branches must not: + +```text +perform waits +sleep +join +call join_op +do stream I/O +make blocking bus calls +call backends directly +run protocol work inline +block on queue capacity +``` + +Blocking work belongs in workers, request scopes, I/O owners, operation scopes or service-owned runtimes. + +## Reducers and decisions + +Reducers should be boring. + +A reducer should normally: + +```text +check event identity +ignore stale events +update coordinator-owned records +decide what immediate effects are required +record state before effects are dispatched +``` + +A reducer should not: + +```text +perform work +call an Op +publish to the bus directly +open handles +close protocols gracefully +call HAL or HTTP transport +call another service directly +``` + +A useful split is: + +```lua +local decision = reduce_event(state, ev) + +for _, effect in ipairs(decision.effects) do + dispatch_effect(state, effect) +end +``` + +Effects should be immediate. If an effect would wait, dispatch it by starting scoped work. + +## Event records + +Use explicit event records rather than positional tuples once events cross a service boundary. + +Good: + +```lua +{ + kind = "component_done", + component = "reader", + link_id = link_id, + generation = generation, + status = "ok", + result = result, +} +``` + +Avoid: + +```lua +{ "done", link_id, true, result } +``` + +Event records should include enough information to make stale handling simple: + +```text +kind +service-owned id +generation +status +result or reason +primary failure where relevant +``` + +## Completion records + +Completion events must carry enough identity to be stale-safe. + +```lua +{ + kind = "operation_done", + operation_id = operation_id, + generation = generation, + + status = "ok", + report = report, + + result = result, + primary = primary, +} +``` + +Coordinator handling should be defensive: + +```lua +local rec = state.operations[ev.operation_id] +if not rec or rec.generation ~= ev.generation then + return +end +``` + +Child failure is data until the owning coordinator decides policy. + +## Ops and reusable blocking code + +Reusable blocking helpers should expose `_op` forms. + +Good: + +```lua +local function read_message_op(stream) + return stream:read_line_op():wrap(parse_message) +end +``` + +Bad: + +```lua +local function read_message(stream) + return fibers.perform(stream:read_line_op()) +end +``` + +The caller chooses the waiting policy: + +```lua +local which, value = fibers.perform(fibers.named_choice { + message = read_message_op(stream), + timeout = sleep.sleep_op(2), +}) +``` + +Avoid APIs where the reusable helper accepts opaque timeout parameters and hides a race internally. Return an Op and let the caller compose it. + +### Caller-composed timeouts + +Reusable SDKs and client helpers should normally return Ops with no hidden bus timeout policy. + +Good: + +```lua +local which, result, err = fibers.perform(fibers.named_choice { + reply = ref:call_op(payload), + timeout = sleep.sleep_op(5):wrap(function () + return nil, "timeout" + end), +}) +``` + +Bad: + +```lua +-- Bad: a reusable client silently installs a default 5s bus timeout. +return conn:call_op(topic, payload, { timeout = opts.timeout or 5 }) +``` + +A reusable client may expose an explicit timeout option for convenience, but its default should be caller-composed waiting, usually by passing `timeout = false` to bus calls. Service-protection timeouts are still valid where the service owns the policy; they should live in the admitted operation scope, not be smuggled through a general SDK helper. + +## Choice is readiness, not priority + +`choice`, `named_choice` and `first_ready` do not express semantic priority. + +Where priority matters, use an explicit selector: + +```text +try ready sources in priority order +if none are ready, block once on all relevant Ops +after waking, re-run the priority selector +``` + +The waking event is a hint. It need not be the event processed next. + +Use the shared priority helper rather than relying on table order. + +## Shared service support modules + +The shared support modules encode service discipline. Prefer them over service-local reinvention. + +| Module | Role | +|---|---| +| `devicecode.support.scoped_work` | Starts identity-bearing child work, stores completion before reporting, separates body-ended from authorised reaping, and accepts `cancel_op` for external cancellation sources. | +| `devicecode.support.service_events` | Stamps events with identity and generation before sending them to coordinators. | +| `devicecode.support.priority_event` | Implements explicit semantic priority without relying on `choice` ordering. | +| `devicecode.support.resource` | Provides immediate ownership, handoff and termination helpers. | +| `devicecode.support.request_owner` | Ensures at-most-once reply, fail or abandon for bus/HTTP requests, and provides `caller_cancel_op()` for bus request abandonment. | +| `devicecode.support.queue` | Provides public try-now helpers built from `or_else`. | +| `devicecode.support.config_watch` | Standard retained `cfg/` watching. | +| `devicecode.support.bus_cleanup` | Finaliser-safe bus cleanup wrappers. | +| Service models | Pulse-backed observable state with snapshots, versions and `changed_op`. | + +Use the shared helper unless the service has a clear reason not to. + +### Retained service configuration + +Modern service shells consume intended configuration through +`devicecode.support.config_watch`. + +The helper opens a normal `lua-bus` subscription to `cfg/`. In +`lua-bus`, subscription creation replays matching retained messages before +subsequent live publications. That retained replay is the bootstrap mechanism: +a service which starts after its configuration has already been retained must +observe that configuration as a normal `config_changed` event. + +Do not build service-specific retained configuration paths using an independent +retained view plus a live subscription. That split can reintroduce the timing +race retained configuration was introduced to remove. Service code should own +validation and generation policy, but the subscription/replay mechanics belong +in the shared helper. + +Direct `cfg/` subscriptions in modern services are architecture debt. +Bridge code may still subscribe to arbitrary bus topics supplied by Fabric +routing rules, but service shell configuration should go through +`config_watch.open`. + +## Request ownership + +Every request-like object needs one owner. + +This includes: + +```text +bus Request +HTTP accepted context +HTTP response writer +artifact ingest session +upload body source +transfer source +worker completion waiter +``` + +Use a request owner to ensure at-most-once resolution: + +```text +reply once +fail once +abandon once +finalise unresolved once +``` + +A finaliser may abandon or fail an unresolved in-memory request owner. It must not wait for graceful completion. + +## Bus request cancellation + +A public bus request may outlive the caller's interest in the result. If the caller races a bus SDK Op against another Op and the bus Op loses, the bus `Request` is abandoned. Service code must treat that abandonment as a cancellation source for any admitted caller-owned work. + +The canonical admission pattern is: + +```lua +local owner = request_owner.new(req) + +local handle, err = scoped_work.start { + lifetime_scope = scope, + reaper_scope = scope, + report_scope = scope, + identity = identity, + + setup = function (work_scope) + work_scope:finally(function (_, status, primary) + owner:finalise_unresolved(primary or status or "request_closed") + end) + + return { + request_owner = owner, + cancel_owned_now = function (reason) + owner:abandon_unresolved(reason or "caller_abandoned") + return true + end, + } + end, + + run = run_request_work, + report = report_completion, + cancel_op = owner:caller_cancel_op(), +} +``` + +Use this for bus requests that admit meaningful scoped work, including device actions, update manager requests, artifact ingest, Fabric transfer requests, Fabric local-to-remote bridge calls, HTTP operations and long HAL capability operations. + +Do not apply `caller_cancel_op()` to service-owned background components such as observers, publishers, generation reconcilers, retained watchers or listener runtimes unless they were directly admitted from a caller-owned bus `Request`. + +The intended semantics are: + +```text +caller abandons or times out the SDK Op + -> bus Request becomes abandoned + -> request_owner observes caller cancellation + -> scoped_work cancels the admitted child scope + -> owned resources terminate through the scope's normal cleanup path + -> late completion is stale or resolves only local cleanup +``` + +Caller abandonment is not a service fault. Log it as cancellation or abandonment, not as a backend failure. + +## Resource ownership + +Every owned resource should have one clear owner at any point in time. + +A resource may be: + +```text +owned by caller +owned by request scope +owned by generation +owned by service +handed off to receiver +borrowed but not owned +``` + +Ownership transfer must be visible. + +```text +before admission: + caller owns cleanup + +after successful admission: + admitted scope owns cleanup + +if admission fails: + original owner still owns cleanup +``` + +For handoff: + +```text +receiver installs cleanup first +sender then detaches cleanup +sender must not return a resource that its own finaliser will immediately close +``` + +## Handles and public wrappers + +Do not return backend objects as public handles. + +A public handle should expose only the safe public operations: + +```text +read_op +write_op +send_op +receive_op +close_op where graceful close is intended +terminate(reason) +public id or metadata +``` + +It should not expose: + +```text +backend driver object +raw file descriptor +private command mailbox +service coordinator state +scope join authority +private cleanup function +``` + +Public wrappers keep service boundaries enforceable. + +## Join authority + +`join_op()` is active finalisation. It may: + +```text +close admission +wait for fibres +join child scopes +run finalisers +record outcome +detach the joined scope from its parent +``` + +Therefore: + +```text +parent finalisation is the default structural reaper +early reaping must be explicit +non-parent reaping must be delegated +reporters must not join arbitrary work +``` + +Use `scoped_work` for identity-bearing child work that needs completion reporting. + +## Finalisers + +Finalisers terminate owned resources. They do not wait. + +Allowed: + +```text +terminate a handle +detach a registration +release a local reservation +mark closed +cancel a child scope without waiting +resolve or abandon an in-memory request owner +``` + +Not allowed: + +```text +perform on a waiting Op +perform_raw +sleep +join +close_op +stream read/write/flush +protocol handshake +retry loop +blocking queue send +backend call that may wait +``` + +Use this vocabulary consistently: + +```text +terminate(reason) immediate, idempotent, finaliser-safe +close_op() graceful protocol work, may wait +``` + +Any owned resource must have an immediate termination path. + +## Models + +Models are observable state. They are not workers. + +A model should provide: + +```text +snapshot() +version() +changed_op(seen) +set_snapshot(...) +terminate(reason) +``` + +Rules: + +```text +snapshot returns copies +updates are non-yielding +updates signal only on material change +changed_op is an Op for observers +models do not perform Ops +models do not call services or backends +models do not publish unless explicitly designed as publishers +``` + +Expensive projection belongs in scoped work, not in model update logic. + +## Capability dependencies + +Capability availability is coordinator input, not service start-up ordering. + +Modern service shells may start concurrently, bind safe public endpoints and open +configuration watches before their dependencies are ready. Dependent blocking +work must be admitted only after the coordinator has observed the required +capabilities as effectively available. + +Use `devicecode.support.capability_dependencies` for repeated capability waits. +The helper owns status subscriptions, status normalisation, effective +availability, route-missing inference and copied snapshots. It does not start +work, retry work, degrade services or make policy decisions. + +Use `devicecode.support.dependency_slot` for the repetitive owner-side mechanics +of replacing, terminating and projecting one dependency manager field. It owns +boilerplate only; the coordinator still owns admission policy. + +Use `devicecode.support.dependency_failure` at service edges to normalise messy +transport or backend failures into a canonical dependency failure. Core helpers +consume clean facts; they do not recursively search reports, child failures or +stringified diagnostics. + +A service coordinator should use it like this: + +```lua +local deps = assert(cap_deps.open(conn, { + { key = 'job_store', class = 'control-store', id = 'update', required = true }, +})) + +-- In the coordinator event set: +-- deps:event_source() +-- or, for model-style observers, deps:changed_op(seen) + +if state.pending_start and deps:available('job_store') then + start_job_runtime(state, 'job_store_available') +end +``` + +If a capability call returns `no_route`, do not convert that directly into a +backend failure. The edge that made the call should normalise it into a clean +shape such as: + +```lua +{ kind = 'dependency_failure', err = 'no_route', dependency_key = key } +``` + +Record route-missing on the dependency and let the coordinator return the +affected work to a pending or waiting state. A later retained `available` +status clears route-missing and wakes the coordinator. + +The intended split is: + +```text +capability dependency helper = facts and wakeups +service coordinator = policy and admission +worker/request scope = blocking capability calls +``` + +## Publication + +Publication should be an immediate effect over already-computed state. + +A publisher may: + +```text +retain canonical state +retain capability metadata +retain capability status +publish observability metrics +unretain records during finalisation through immediate cleanup helpers +record cleanup failures explicitly +``` + +A publisher should not: + +```text +perform blocking reads +query other services +call HAL +do expensive projection +hide retries inside model update +silently drop publication failures +``` + +If publication can fail, the service should decide what that means: + +```text +mark degraded +fail the observing scope +record cleanup failure +retry through explicit worker +continue with documented loss +``` + +## Topic style + +Use topic helper functions. + +Good: + +```lua +topics.update_manager_rpc("create-job") +topics.workflow_update_job(job_id) +topics.raw_member_cap_rpc("mcu", "update", "main", "stage") +``` + +Avoid scattered literals: + +```lua +{ "cap", "update-manager", "main", "rpc", "create-job" } +``` + +Topic helpers should: + +```text +return fresh tables +validate identifiers where useful +use kebab-case public tokens +make canonical ownership obvious +reduce accidental compatibility roots +``` + +## Control-plane publication + +Devicecode uses these planes: + +```text +svc/... service lifecycle only +cfg/... intended service configuration +raw/... provenance-bearing source truth and raw provider interfaces +state/... canonical retained public truth +cap/... stable public callable/inspectable interfaces +obs/v1/... observability +``` + +Canonical roots are deliberately limited. Public call surfaces belong under `cap/...`, raw provider-native call surfaces under `raw/.../cap/...`, and retained workflow records under `state/workflow/...`. Public workflows should not be exposed through ad hoc `cmd/...` trees. :contentReference[oaicite:0]{index=0} :contentReference[oaicite:1]{index=1} + +General rules: + +```text +svc//status and svc//meta are lifecycle only. +cfg/ is intended service configuration. +raw/... is for provenance-bearing facts and raw provider-native capabilities. +state/device/... is composed appliance truth owned by Device. +state/workflow/... is retained workflow instance truth. +state//... is canonical domain retained truth. +cap/... is the public interface plane. +obs/v1/... is for metrics, logs and diagnostics, not operational truth. +``` + +A service id is not automatically a domain id. `state/` names public semantic truth, not an implementation module. + +## Typical publication pattern + +```mermaid +flowchart LR + Raw["raw/..."] --> Observer["observer scope"] + Cfg["cfg/"] --> Coordinator["coordinator"] + Observer --> Model["service model"] + Coordinator --> Model + Model --> Publisher["publisher"] + Publisher --> State["state//..."] + Publisher --> Cap["cap///..."] + Publisher --> Svc["svc//..."] + Publisher --> Obs["obs/v1//..."] +``` + +## Observability + +Observability is not canonical operational truth. + +Use `obs/v1/...` for: + +```text +metrics +diagnostics +logs +counters +service-local debugging +performance or queue statistics +``` + +Do not use `obs/v1/...` for: + +```text +public workflow records +component state +appliance composition +stable interface availability +durable configuration +``` + +Where a statistic is useful for interface inspection, a narrow retained adjunct under `cap/.../state/...` may also be justified. Keep that narrow and summary-level. + +## Backpressure + +Backpressure is policy, not an accident of queue length. + +A service should explicitly choose one of: + +```text +reject the request +terminate the client or watch +drop low-priority telemetry +mark degraded +cancel scoped work +fail the observing scope +slow a worker or I/O owner +store and retry completion reporting +``` + +Completion events for healthy observing scopes should not be silently dropped. + +## Error style + +Prefer explicit status values and public-safe errors. + +Good public result: + +```lua +{ + ok = false, + error = "unknown-component", + message = "component is not configured", +} +``` + +Good internal completion: + +```lua +{ + status = "failed", + primary = "durable-save-failed", + report = report, +} +``` + +Avoid exposing: + +```text +private resource objects +stack traces in public replies +ownership internals +mailbox objects +scope handles +backend errors without classification +``` + +Internal diagnostics can retain richer reports under service-owned observability. + +## Validation style + +Validate at the boundary. + +Examples: + +```text +bus RPC payload enters service +HTTP request becomes UI operation +config is loaded from cfg/ +raw provider capability is accepted into HAL +public handle is returned to another service +``` + +Validation functions should be pure and non-yielding. They should not repair ambiguous public input silently where that would create a second contract. + +## Test style + +Happy-path tests are insufficient. + +Tests should include: + +```text +operation loses a choice +timeout wins +caller abandonment crosses the bus boundary into admitted scoped work +queued bus requests abandoned before admission are skipped or resolved locally +scope cancellation interrupts waits +finalisers do not wait +try-now helpers do not suspend +callback registration unregisters on abort +queue overflow follows documented policy +join authority is respected +stale completions are ignored +stored completion exists before reporting +ownership transfer disables old cleanup +terminate is idempotent +close_op is not called from finalisers +models wake observers on termination +priority selectors re-check after wake-up +``` + +Integration tests should drive public seams: + +```text +cfg/ +cap/.../rpc/... +raw/... retained facts +HTTP public capability handles +state/... retained truth +obs/v1/... where observability is the subject +``` + +They should not inspect private coordinator tables unless the test is explicitly a unit test for that module. + +## Service review checklist + +A service review should check: + +```text +No service code uses perform_raw. +Reusable waits expose _op APIs. +Coordinators have one visible wait point. +Coordinator branches do not block. +Callbacks do not mutate service state directly. +Identity-bearing work has a scope. +Completions carry identity and generation. +Stale completions are ignored. +Stored completion exists before reporting. +Finalisers terminate only. +Owned resources have terminate(reason). +Public handles do not expose backend objects. +Public calls are under cap/... +Workflow records are under state/workflow/... +Canonical retained truth is under state/... +Raw provider truth remains under raw/... +Observability is under obs/v1/... +Backpressure policy is explicit. +Topic helpers are used instead of scattered literals. +Boundary validation is pure and non-yielding. +Tests cover losing paths. +Bus-admitted scoped work uses owner:caller_cancel_op(). +Reusable SDK/client helpers do not install hidden bus timeouts by default. +``` + +## Signs that code is drifting + +Look closely if a service introduces: + +```text +a callback that updates service state +a helper without _op that can block +a reusable SDK/client helper with a hidden default bus timeout +a bus request that starts scoped_work without owner:caller_cancel_op() +a finaliser calling close_op +a coordinator branch calling fibers.perform +a queue send with no overflow policy +a child fibre with no identity or completion path +a public reply containing private handles +a state/ tree justified only by module name +a cap/.../state subtree growing into a domain model +a raw provider interface mirrored directly under cap/... +a cmd/... public path +a test that depends on private coordinator tables for an integration behaviour +``` + +Any one of these may be justified in a narrow case. Several together usually mean the service boundary is losing shape. + + +### Capability dependency admission for modern services + +Modern services should start their shell before all dependencies are available. +The shell may open config watches, dependency watches, retained projections and +safe public endpoints. It must not start dependency-backed work merely because +configuration has arrived. + +Use `devicecode.support.capability_dependencies` as a coordinator-facing model: + +```text +capability dependency helper = facts and wakeups +service coordinator = policy and admission +worker/request scope = blocking capability calls +``` + +For admission decisions, use `deps:available(key)`, not `deps:status(key)`. +`status()` is diagnostic: an observed capability may say `running` while carrying +`available=false`, or may be locally overridden by `route_missing` after a +`no_route` call. + +Classify dependency failures before applying service policy: + +```text +no_route / capability absent / available=false + => admission failure; record waiting or reject the individual request + +accepted operation returns a domain error + => domain failure; degrade, fail, or record a failed job/action according to + the owning service's policy +``` + +The current modern service expectations are: + +```text +HTTP exposes cap/http status; non-status requests require backend ready. +UI waits for cap/http before starting configured listener work. +Fabric waits for HAL/raw transport capabilities before starting link generation. +Device gates explicitly dependency-annotated actions; observations remain tolerant. +Wired projects provider dependency state as model facts, not startup blockers. +NET gates network-config and network-state work. +Update gates job and artifact runtime work. +``` diff --git a/docs/services/hal.md b/docs/services/hal.md new file mode 100644 index 00000000..03e64a48 --- /dev/null +++ b/docs/services/hal.md @@ -0,0 +1,124 @@ +# HAL +## File Structure +- hal.lua +- hal + + - backends <-- portability layer: choose a provider implementation per device family + - uart + - provider.lua <-- selects UART provider (fibers, ubus, etc) + - providers + - fibers_core + - init.lua + + - modem + - provider.lua <-- selects modem provider (OpenWrt ModemManager, testing, etc) + - iface.lua <-- the modem-backend interface contract (documented + validated) + - mode + - qmi.lua + - mbim.lua + - model + - quectel.lua + - fibcocom.lua + - providers + - openwrt_mm + - init.lua <-- constructs a backend that implements modem/iface.lua + - mmcli.lua <-- executor: mmcli bindings + - at.lua <-- ModemManager-specific AT bridge (if needed) + - some_other_software + - init.lua + + - drivers <-- domain logic + stable capability surface + - modem.lua <-- specific modem functionality and HAL interaction + - modem + - any helper stuff here + + - managers <-- owns lifecycles, supervision, (re)announce capabilities + - modemcard.lua + + - types <-- structured tables + validation + - core.lua + - capabilities.lua + - external.lua + - modem.lua + + - resources.lua <-- loads managers based on hardware target (declarative wiring) + - resources + - bigbox_v1_cm.lua + - bigbox_ss.lua + +## HAl monitor acticheture + +```mermaid +sequenceDiagram + participant Config + participant Bus + participant Core + participant Monitors + Core->>Core: Init storage driver and cap (for config loading) + Config->>Core: Request config blob + Core->>StoreCap: Parrot Request + StoreCap->>Core: Return blob + Core->>Config: Response + Config->>Core: Load config + Core->>Core: Create monitors + Core->>Monitors: op choice over monitors (and bus etc) + Monitors->>Core: (UART) use config to build uart driver, return driver in connected event + Core->>Core: Init driver and get capabilities + Core->>Bus: Broadcast uart device and capability +``` + +Monitors are scope protected modules that provide a event based op which returns when a device is either connected or disconnected. +```lua +local device_event = perform(uart_monitor:monitor_op()) +if device_event.connected then + -- somthing +else + --somthing else +end +``` + +Monitors can scale for complexity, for example the simplest case of a monitor would be uart, where the monitor would recieve a config and build drivers immediately based on that config, the monitor_op would iterate over the drivers and then block forever +```lua +function monitor:monitor_op() + if self._progress >= #self._drivers then + return op.never() + end + return op.always( + -- device connected event goes here + ) +end +``` + +Some devices are not defined at config time, and appear dynamically. These monitors can return a scope_op which listens to an underlying command to build device events +```lua +function monitor.new() + return setmetatable({ + scope = -- child scope to protect HAL from montior based failure + cmd = -- monitoring command here + }, monitor) +end + +-- A monitor op that is dependant on command line output +function monitor:monitor_op() + return scope:scope_op(function () + return self.cmd:read_line_op():wrap(parser) + end) +end +``` + +Both of these examples are fiberless + +Monitors may also spawn fibers in the background, although the majority won't need to, to mediate command output between drivers + +## HAL Authoring Docs + +For manager, driver, capability, and backend implementation guidance, use this document set: + +- [HAL Authoring Guide Index](./hal/authoring-index.md) +- [HAL Architecture Standards](./hal/architecture-standards.md) +- [HAL Implementation Cookbook](./hal/implementation-cookbook.md) +- [HAL Interfaces Reference](./hal/interfaces-reference.md) +- [HAL Backend Authoring Guide](./hal/backend-authoring.md) + + + diff --git a/docs/services/hal/architecture-standards.md b/docs/services/hal/architecture-standards.md new file mode 100644 index 00000000..a4ed2fbf --- /dev/null +++ b/docs/services/hal/architecture-standards.md @@ -0,0 +1,173 @@ +# HAL Architecture Standards + +This document defines the required architecture for new HAL device classes. + +## Layer Model + +Every HAL device class MUST follow this layering model unless explicitly justified in design review. + +```mermaid +flowchart LR + M[Manager] --> D[Driver] + D --> C[Capability] + D --> B[Backend Composite] + C --> Bus[HAL Bus] +``` + +### Manager + +Responsibilities: + +- Detect add/remove events for a device class. +- Construct, initialize, start, and stop drivers. +- Emit device lifecycle events. + +Standards: + +- A manager MUST own device discovery and lifecycle orchestration for one device class. +- A manager MUST emit `DeviceEvent("added", ...)` only after the driver is initialized, capabilities are built, and the driver starts successfully. +- A manager MUST emit `DeviceEvent("removed", ...)` when a driver is removed or faults. +- A manager MUST expose `apply_config(config)` as part of the manager interface. +- `apply_config(config)` MAY be a no-op for managers that do not support runtime config. +- A manager SHOULD isolate failures by running detector/manager loops in a child scope. + +Manager channel contract: + +- A manager MUST consume hardware detection/removal from class-specific inputs. +- A manager MUST produce `DeviceEvent` messages on `dev_ev_ch`. +- A manager MUST pass `cap_emit_ch` into `driver:capabilities(cap_emit_ch)`. +- A manager SHOULD own internal channels for detection, removal, and initialized-driver handoff. + +Canonical reference: `src/services/hal/managers/modemcard.lua` + +## Capability RPC cancellation + +HAL capability RPCs are bus requests. If a caller abandons a request after HAL admission, the abandonment should be visible to the manager or driver as a cancellation source, not discovered only as a failed late reply. + +Managers that convert capability requests into scoped work should pass the request cancellation Op into that work: + +```lua +scoped_work.start { + ..., + cancel_op = request.cancel_op, +} +``` + +Drivers that own long-running operation scopes should treat `request.cancel_op` as an immediate cancellation source. Immediate handlers may simply avoid blocking reply delivery after abandonment. + +### Driver + +Responsibilities: + +- Represent one hardware instance. +- Expose capability verbs through control channels. +- Emit state/event/meta updates. +- Delegate platform-specific operations to backend when a backend exists. + +Standards: + +- A driver MUST implement `init()`, `start()`, `stop(timeout)` and `capabilities(emit_ch)`. +- A driver MUST validate option types for each verb. +- A driver MUST construct `Reply` objects for control requests. +- A driver SHOULD keep long-running loops in fibers supervised by a scope. + +Driver channel contract: + +- A driver MUST expose one or more capability `control_ch` channels used to receive `ControlRequest` messages. +- A driver MUST publish emits through `cap_emit_ch` using `hal_types.new.Emit(...)`. +- A driver SHOULD use dedicated internal channels for asynchronous signals (for example SIM state changes) when needed. + +Canonical references: + +- `src/services/hal/drivers/modem.lua` +- `src/services/hal/drivers/filesystem.lua` + +### Capability + +Responsibilities: + +- Publish callable offerings for a class/id pair. +- Route control requests to driver logic through `control_ch`. + +Standards: + +- Capabilities MUST be created using `capabilities.new.Capability(...)` or a typed constructor from `types/capabilities.lua`. +- Offerings MUST include only verbs implemented by the driver. +- Capability class/id combinations MUST be stable and unique per device instance. +- Offerings SHOULD be static at startup by default. + +Canonical reference: `src/services/hal/types/capabilities.lua` + +### Backend + +Responsibilities: + +- Implement platform-specific operations (system commands, provider APIs, parsing). +- Compose a stable function surface consumed by drivers. + +Standards: + +- A backend MAY be omitted when platform-specific abstraction adds no value (filesystem reference). +- If a backend exists, it MUST satisfy the class contract validator. +- Backends MUST return `(result, error_string)` semantics consistently. + +Backend composition standard: + +- A backend MAY be composed from multiple sources before validation. +- The base source is typically OS/software-provider implementation. +- Additional layers (for example mode/model/device-specific contributors) MAY augment behavior. +- The final composed backend MUST fully satisfy the target backend contract before it is returned. + +Canonical references: + +- `src/services/hal/backends/modem/provider.lua` +- `src/services/hal/backends/modem/contract.lua` + +## Lifecycle Standard + +```mermaid +sequenceDiagram + participant HW as Hardware + participant M as Manager + participant D as Driver + participant CC as Capability control_ch + participant CE as cap_emit_ch + participant DE as dev_ev_ch + participant H as HAL + + HW->>M: detection event (added) + M->>D: new(address) + M->>D: init() + D-->>M: init ok + M->>D: capabilities(cap_emit_ch) + M->>D: start() + M->>DE: DeviceEvent added + DE->>H: publish lifecycle + + CC->>D: ControlRequest(verb, opts, reply_ch) + D-->>CC: Reply(ok, reason_or_value, code) + D->>CE: Emit(class, id, mode, key, data) + CE->>H: publish capability update + + HW->>M: detection event (removed) + M->>D: stop(timeout) + M->>DE: DeviceEvent removed + DE->>H: publish lifecycle +``` + +Standards: + +- Managers MUST NOT announce `added` before `start()` returns success. +- Managers MUST handle driver scope faults as removal events. +- Drivers MUST stop within timeout or return an explicit timeout error. + +## Backend Optionality Decision Rule + +Use this decision process for new classes: + +1. If implementation requires platform-specific command execution, parsing, or provider detection, a backend SHOULD be used. +2. If logic is local and portable with no provider split, backend MAY be skipped. +3. If future provider variance is likely, backend SHOULD be introduced early. + +Filesystem is the skip-backend reference. +Modem is the use-backend reference. diff --git a/docs/services/hal/authoring-index.md b/docs/services/hal/authoring-index.md new file mode 100644 index 00000000..502e93d6 --- /dev/null +++ b/docs/services/hal/authoring-index.md @@ -0,0 +1,34 @@ +# HAL Authoring Guide Index + +This index is the entry point for engineers implementing or extending HAL managers, drivers, capabilities, and backends. + +## Read Order + +1. [Architecture Standards](./architecture-standards.md) +2. [Implementation Cookbook](./implementation-cookbook.md) +3. [Interfaces Reference](./interfaces-reference.md) +4. [Backend Authoring Guide](./backend-authoring.md) + +## Scope + +- Primary example stack: modem +- Secondary/simple example stack: filesystem +- Intended for both onboarding engineers and maintainers +- Forward-looking guidance for adding new device classes + +## Standards Policy + +- Rules labeled MUST are compatibility and correctness requirements. +- Rules labeled SHOULD are recommended defaults. +- Rules labeled MAY are optional patterns. + +## Canonical Source References + +- `src/services/hal/managers/modemcard.lua` +- `src/services/hal/drivers/modem.lua` +- `src/services/hal/drivers/filesystem.lua` +- `src/services/hal/backends/modem/provider.lua` +- `src/services/hal/backends/modem/contract.lua` +- `src/services/hal/types/core.lua` +- `src/services/hal/types/capabilities.lua` +- `src/services/hal/types/capability_args.lua` diff --git a/docs/services/hal/backend-authoring.md b/docs/services/hal/backend-authoring.md new file mode 100644 index 00000000..cb03648c --- /dev/null +++ b/docs/services/hal/backend-authoring.md @@ -0,0 +1,127 @@ +# HAL Backend Authoring Guide + +This guide covers authoring backend providers and implementations for HAL device classes, using modem as the canonical pattern. + +Mode/model in modem are modem-specific names for a broader composition idea: a backend can be built from multiple contributors, as long as the final composed backend satisfies the contract. + +## Backend Composition Model + +```mermaid +flowchart LR + P[provider.lua] --> S[select supported provider] + S --> I[base source: OS or software package] + I --> C1[contributor layer 1] + C1 --> C2[contributor layer N] + C2 --> V[contract.validate] +``` + +Standards: + +- Provider assembly MUST call contract validation before returning backend. +- Provider selection MUST choose first supported provider in configured order. +- Contributor augmentation MUST happen before contract validation when contributor methods are required contract members. + +Composition guidance: + +- The base source is usually an OS/software-provider implementation. +- Additional contributors MAY be mode/model/device/board/vendor specific. +- Contributor names are device-class specific and are not globally fixed. +- Modem `mode` and `model` are one concrete specialization of this pattern. + +## Provider Contract + +A provider module SHOULD expose: + +- `is_supported()` +- `backend` table with `new(address)` constructor +- `new_monitor()` when discovery monitoring is supported + +Standards: + +- `is_supported()` MUST be fast and safe to call at startup. +- `new_monitor()` is required only for backends that support dynamic runtime detection. +- Backends using static config/manager wiring MAY omit `new_monitor()`. +- When provided, `new_monitor()` MUST return a monitor satisfying the monitor contract for that device class. + +Canonical reference: + +- `src/services/hal/backends/modem/providers/linux_mm/init.lua` + +## Backend Function Contract + +Contract is enforced in: + +- `src/services/hal/backends/modem/contract.lua` + +Standards: + +- Backend implementations MUST implement all required functions in the contract list. +- Backend implementations MUST NOT expose extra function-valued keys not listed in the contract. +- Functions MUST return `(result, error_string)` and use empty string on success where applicable. + +## Monitor Contract + +Monitor contracts are per device class. + +Modem monitor contract currently requires: + +- `next_event_op` + +Standards: + +- Monitors MUST NOT expose extra function-valued keys beyond required monitor contract. +- Monitor event parser SHOULD treat unknown lines as non-fatal and continue. +- Each device class MUST define and validate its own monitor contract when monitors are used. + +## Contributor Augmentation Rules + +Contributor augmentations apply class-specific behavior during backend construction. + +For modem this appears as mode and model augmentations. + +Mode augmentations: + +- Apply mode-specific behavior based on detected transport, for example qmi or mbim. +- Keep overrides localized to mode-specific concerns. + +Model augmentations: + +- Apply vendor/model-specific workarounds. +- Use narrow conditional matching (manufacturer + model + optional variant). + +Standards: + +- Overrides MUST preserve expected call signatures. +- Overrides MUST preserve return semantics and error conventions. +- Overrides SHOULD avoid global side effects outside backend instance mutation. +- The fully composed backend MUST satisfy the class contract before return. + +Canonical references: + +- `src/services/hal/backends/modem/modes/qmi.lua` +- `src/services/hal/backends/modem/models/quectel.lua` + +## Authoring Checklist + +1. Add provider entry and support detection logic. +2. Implement backend constructor and required contract functions. +3. Add monitor implementation only if class discovery needs dynamic monitor support. +4. Add optional contributor modules (for example mode/model/device-specific layers). +5. Run through contract validation path from provider assembly. + +## Common Errors and Fixes + +1. Missing required backend function. +Fix: implement function named exactly as contract key. + +2. Unsupported extra backend function. +Fix: remove extra function key or add it to contract only if required by driver architecture. + +3. Broken driver integration due to signature mismatch. +Fix: match function arguments expected by driver call sites. + +4. Inconsistent success/error tuple returns. +Fix: normalize return to `(value_or_ok, error_string)`. + +5. Monitor shape validation failure. +Fix: expose only required monitor methods. diff --git a/docs/services/hal/implementation-cookbook.md b/docs/services/hal/implementation-cookbook.md new file mode 100644 index 00000000..c8732da4 --- /dev/null +++ b/docs/services/hal/implementation-cookbook.md @@ -0,0 +1,244 @@ +# HAL Implementation Cookbook + +This document gives template-first implementation patterns for adding or extending HAL components. + +## Pattern 1: Add a Capability Verb + +Use this when adding a new action to an existing capability class. + +### Required changes + +1. Add a typed options constructor in `types/capability_args.lua`. +2. Add a driver verb method in the relevant driver. +3. Add the verb to capability offerings in `types/capabilities.lua`. +4. If driver delegates to backend, add backend function and contract entry. + +### Template + +```lua +-- types/capability_args.lua +local ModemFooOpts = {} +ModemFooOpts.__index = ModemFooOpts + +function new.ModemFooOpts(value) + if type(value) ~= "string" or value == "" then + return nil, "invalid value" + end + return setmetatable({ value = value }, ModemFooOpts), "" +end +``` + +```lua +-- drivers/modem.lua +function Modem:foo(opts) + if opts == nil or getmetatable(opts) ~= capability_args.ModemFooOpts then + return false, "invalid options", 1 + end + local ok, err = self.backend:foo(opts.value) + if not ok then + return false, err, 1 + end + self:_emit_event("foo_done", { value = opts.value }) + return true +end +``` + +### Standards + +- New verbs MUST validate metatable-typed opts. +- Offerings MUST match real driver method names. +- Driver verb responses MUST return HAL reply shape: `ok, reason_or_value, code`. + +### Return-value semantics + +Capability endpoints are not limited to boolean-only outcomes. + +- State-changing verbs often return boolean success, for example `enable`, `disable`, `connect`. +- Query verbs SHOULD return the requested value on success. + +Canonical example (`modem:get` pattern): + +```lua +function Modem:get(opts) + if opts == nil or getmetatable(opts) ~= capability_args.ModemGetOpts then + return false, "invalid options", 1 + end + local value, err = self.backend[opts.field](self.backend, opts.timescale) + if err ~= "" then + return false, err, 1 + end + return true, value +end +``` + +In the control loop, this value is carried in `Reply.reason` when `ok == true`. + +## Pattern 2: Add a New Manager + Driver Pair + +Use this for a new device class. + +### Manager template responsibilities + +1. Own detector loop. +2. Spawn and initialize driver on add. +3. Start driver after capability creation. +4. Emit added/removed `DeviceEvent`. +5. Expose `apply_config(config)` even when configuration is optional. + +### Driver template responsibilities + +1. Implement `init/start/stop/capabilities`. +2. Implement control manager loop over `control_ch`. +3. Emit state/event/meta updates via `Emit`. + +### Sequence template + +```mermaid +sequenceDiagram + participant Det as Detector + participant M as Manager + participant D as Driver + participant Bus as HAL + + Det->>M: address added + M->>D: new(address) + M->>D: init() + M->>D: capabilities(cap_emit_ch) + M->>D: start() + M->>Bus: DeviceEvent added +``` + +### Standards + +- Manager MUST keep map from address to active driver. +- Manager MUST remove map entry on removal/fault. +- Manager MUST implement `apply_config(namespaces)`; if unused it SHOULD return success as a no-op. +- Driver MUST reject `start()` when not initialized or capabilities not applied. + +### Required channel wiring + +Manager-side channels: + +1. `dev_ev_ch`: manager MUST publish add/remove lifecycle events. +2. `cap_emit_ch`: manager MUST pass to driver capability setup for emit forwarding. +3. Internal manager channels: detection/removal/driver-ready channels SHOULD isolate detector logic from lifecycle logic. + +Driver-side channels: + +1. Capability `control_ch` per exposed capability endpoint MUST receive `ControlRequest`. +2. `request.reply_ch` MUST receive `Reply` from driver control manager. +3. `cap_emit_ch` MUST be used for `Emit` messages (event/state/meta/log). + +## Pattern 3: Add a Backend Provider + +Use this when a device class needs platform-specific implementations. + +### Required changes + +1. Add provider module with `is_supported()` and backend constructor. +2. Add provider name to provider selection table. +3. Ensure backend passes contract validation. +4. Optionally add monitor implementation and monitor contract compliance. +5. Optionally compose backend with extra contributors (mode/model/device-specific layers) before validation. + +### Template + +```lua +-- providers/my_platform/init.lua +local function is_supported() + return true +end + +local function new_monitor() + return monitor.new() +end + +return { + is_supported = is_supported, + backend = backend, + new_monitor = new_monitor, +} +``` + +### Standards + +- Providers MUST be side-effect free at module load. +- Provider selection MUST fail fast when no provider is supported. +- `new_monitor()` is required only when dynamic runtime detection is needed. +- Monitor objects MUST satisfy the monitor contract for that device class exactly. + +## Pattern 4: Add Mode or Model Overrides + +Use this when behavior differs by transport mode, model, or other device-specific contributors. + +### Mode override template + +```lua +local function add_mode_funcs(backend) + backend.wait_for_sim_present_op = function(self) + -- mode-specific behavior + end +end + +return { add_mode_funcs = add_mode_funcs } +``` + +### Model override template + +```lua +local function add_model_funcs(backend, model, variant) + if model == "rm520n" and backend.base == "linux_mm" then + backend.connect = function(self, connection_string) + -- model-specific connect flow + end + end +end + +return { add_model_funcs = add_model_funcs } +``` + +### Standards + +- Overrides MUST preserve function signatures expected by driver callers. +- Overrides MUST continue returning `(result, error_string)` semantics. +- Overrides SHOULD be narrowly conditional (mode/model/variant/base) to avoid global behavior changes. +- The composed backend MUST still pass contract validation after all contributor layers are applied. + +## Pattern 5: Backend-Free Driver (Filesystem Style) + +Use this for classes with purely local logic. + +### Required properties + +1. Driver directly implements verbs. +2. Driver still uses capability/type contracts. +3. Driver still emits events and replies through HAL types. + +### Standards + +- Backend-free drivers MUST still maintain lifecycle and control loop standards. +- Backend-free does NOT relax capability/options validation. + +## Common Failure Modes + +1. Contract validation failure: backend has missing or extra function. +2. Offering mismatch: capability declares verb not implemented in driver. +3. Options mismatch: caller passes non-typed opts table. +4. Scope leak: long-running loop not scope-managed. +5. Emit misuse: emitting `nil` data fails `Emit` constructor validation. + +## Author Checklist + +1. Manager lifecycle order is detection -> init -> capabilities -> start -> added event. +2. Removal and fault both result in driver stop and removed event. +3. Every verb has typed options and explicit error handling. +4. Offerings list and driver methods are aligned. +5. Contract validation passes when backend exists. +6. Manager includes `apply_config(config)` endpoint with either real handling or explicit no-op behavior. + +## Code Templates + +Concrete starter templates are available in source: + +- `src/services/hal/templates/manager_template.lua` +- `src/services/hal/templates/driver_template.lua` diff --git a/docs/services/hal/interfaces-reference.md b/docs/services/hal/interfaces-reference.md new file mode 100644 index 00000000..29ae00d4 --- /dev/null +++ b/docs/services/hal/interfaces-reference.md @@ -0,0 +1,167 @@ +# HAL Interfaces Reference + +This reference defines the core HAL message and capability interfaces used by managers and drivers. + +## ControlRequest + +Constructor: `hal_types.new.ControlRequest(verb, opts, reply_ch)` + +Fields: + +- `verb` string +- `opts` table +- `reply_ch` Channel + +Invariants: + +- `verb` MUST be a non-empty string. +- `opts` MUST be a table. +- `reply_ch` MUST be a fibers channel. + +Source: `src/services/hal/types/core.lua` + +## Reply + +Constructor: `hal_types.new.Reply(ok, reason, code)` + +Fields: + +- `ok` boolean +- `reason` any +- `code` integer optional + +Invariants: + +- `ok` MUST be boolean. +- Drivers SHOULD provide a non-empty reason on failure. + +Outcome semantics: + +- On failure (`ok == false`), `reason` SHOULD contain an error string and optional `code`. +- On success (`ok == true`), `reason` MAY carry a direct endpoint outcome value. +- Query endpoints (for example modem `get`) SHOULD return their value directly rather than only boolean success. + +Source: `src/services/hal/types/core.lua` + +## Emit + +Constructor: `hal_types.new.Emit(class, id, mode, key, data)` + +Fields: + +- `class` capability class +- `id` capability id +- `mode` one of `event`, `state`, `meta`, `log` +- `key` non-empty string +- `data` non-nil value + +Invariants: + +- `mode` MUST be one of the supported values. +- `data` MUST NOT be nil. + +Source: `src/services/hal/types/core.lua` + +## DeviceEvent + +Constructor: `hal_types.new.DeviceEvent(event_type, class, id, meta, capabilities)` + +Fields: + +- `event_type` one of `added`, `removed` +- `class` device class +- `id` device id +- `meta` table +- `capabilities` array of Capability + +Invariants: + +- `event_type` MUST be `added` or `removed`. +- `capabilities` MUST contain only Capability typed objects. + +Source: `src/services/hal/types/core.lua` + +## Capability + +Constructor: `cap_types.new.Capability(class, id, control_ch, offerings)` + +Fields: + +- `class` string +- `id` string or non-negative integer +- `control_ch` Channel +- `offerings` map of supported verbs + +Invariants: + +- Offerings input MUST be array of non-empty strings. +- `control_ch` MUST be a fibers channel. +- Offerings SHOULD be stable for the device lifetime. + +Typed constructors include: + +- `new.ModemCapability(id, control_ch)` +- `new.FilesystemCapability(id, control_ch)` +- `new.UARTCapability(id, control_ch)` + +Source: `src/services/hal/types/capabilities.lua` + +## Capability Args Pattern + +Options objects are typed by metatable and validated by constructor helpers. + +Examples: + +- `capability_args.new.ModemGetOpts(field, timescale)` +- `capability_args.new.ModemConnectOpts(connection_string)` +- `capability_args.new.FilesystemReadOpts(filename)` +- `capability_args.new.FilesystemWriteOpts(filename, data)` + +Invariants: + +- Driver verb handlers MUST validate expected metatable type. +- Callers SHOULD always construct opts using the exported constructor helpers. + +Source: `src/services/hal/types/capability_args.lua` + +## Channel Contracts + +Manager channels: + +- `dev_ev_ch` carries `DeviceEvent` add/remove lifecycle announcements. +- `cap_emit_ch` carries `Emit` messages sourced by drivers. +- Internal detection/removal/driver-ready channels MAY be manager-specific implementation details. + +Manager interface requirement: + +- Every manager MUST expose `start(logger, dev_ev_ch, cap_emit_ch)`, `stop(timeout)`, and `apply_config(config)`. +- `apply_config(config)` MAY be a no-op when runtime configuration is not used, but the endpoint MUST exist. + +Driver channels: + +- Capability `control_ch` receives `ControlRequest`. +- `reply_ch` embedded in each `ControlRequest` receives `Reply`. +- `cap_emit_ch` is the outbound channel for `Emit`. + +Capability channels: + +- Every capability object MUST expose a valid fibers `control_ch`. +- Clients call capabilities by writing `ControlRequest` to capability `control_ch`. + +## Extension Point Index + +Manager extension points: + +- detector loop and event routing in `src/services/hal/managers/modemcard.lua` + +Driver extension points: + +- verb methods and control loop in `src/services/hal/drivers/modem.lua` +- backend-free verb methods in `src/services/hal/drivers/filesystem.lua` + +Backend extension points: + +- provider selection and assembly in `src/services/hal/backends/modem/provider.lua` +- contract validation in `src/services/hal/backends/modem/contract.lua` +- mode augmentation in `src/services/hal/backends/modem/modes/qmi.lua` +- model augmentation in `src/services/hal/backends/modem/models/quectel.lua` diff --git a/docs/services/hal/op-only-drivers.md b/docs/services/hal/op-only-drivers.md new file mode 100644 index 00000000..479533ca --- /dev/null +++ b/docs/services/hal/op-only-drivers.md @@ -0,0 +1,101 @@ +# Op-only driver guide + +## Purpose + +Op-only drivers are edge components that expose backend or provider behaviour as Fibers-native Ops. + +They are used by HAL and transport services to keep backend-specific polling, callbacks and command loops out of ordinary service code. + +An Op-only driver should make this true: + +```text +service code sees Ops and terminate(reason) +driver code owns backend mechanics +``` + +## Boundary shape + +```mermaid +flowchart LR + Service["service or manager"] --> OpAPI["driver Op API"] + OpAPI --> Driver["driver command loop"] + Driver --> Backend["OS / cqueues / lua-http / file / provider"] + Backend --> Driver + Driver --> OpAPI + OpAPI --> Service +``` + +The service should not know about backend polling, raw callbacks or backend object lifetimes. + +## Driver API rules + +Prefer APIs like: + +```lua +driver:start_op(...) +driver:apply_config_op(...) +driver:shutdown_op(...) +driver:fault_op() +handle:read_op(...) +handle:write_op(...) +handle:terminate(reason) +``` + +Avoid APIs that hide blocking work behind ordinary functions. + +Synchronous helpers are acceptable only when they are pure validation, pure projection or immediate state inspection. + +## Termination and graceful closure + +Drivers should distinguish: + +```text +terminate(reason) immediate, idempotent, finaliser-safe +close_op() graceful protocol work, may wait +``` + +Every object handed to a service must have an immediate termination path if it is owned by a scope finaliser. + +## Losing choices + +Every driver Op must define what happens when it loses a choice. + +Expected patterns: + +```text +queued command loses: remove or mark abandoned +active backend job loses: abort/terminate active job +owned backend handle loses: terminate handle +borrowed backend handle loses: leave owner intact +callback registration loses: unregister callback +``` + +A losing Op must not leave a callback, file descriptor, driver command or backend job silently live. + +## Publication rules + +Op-only drivers normally do not publish canonical `state/...` directly. + +Instead: + +```text +driver/provider emits raw facts or capability events to its owning service +HAL publishes raw/... for provider-native truth +Device/Update/Fabric/UI publish canonical state where they own it +HTTP publishes narrow interface stats and obs metrics where it owns the interface +``` + +Drivers should not invent public `cap/...` or `state/...` families unless the driver itself is deliberately the service boundary. + +## Reviewer checklist + +```text +All blocking driver operations are Ops. +No service-facing method hides a wait. +No driver handle lacks terminate(reason). +close_op is not required for finaliser cleanup. +Losing Ops clean up queued and active backend work. +Callbacks do not call service reducers directly. +Backend objects are not returned as public handles. +Raw/provider facts are emitted to the owning service, not published as canonical state by accident. +``` diff --git a/docs/specs/device.md b/docs/specs/device.md new file mode 100644 index 00000000..de62def5 --- /dev/null +++ b/docs/specs/device.md @@ -0,0 +1,141 @@ +# Device service guide + +## Purpose + +Device owns composed appliance truth. + +It observes raw local, member and Fabric-imported facts and publishes stable appliance-facing component state. + +Device is responsible for: + +```text +component catalogue +component state +component software/update projections +component capability metadata/status +curated component actions +raw-source provenance references +``` + +Device is not HAL. Device curates raw facts. It is not a raw RPC mirror. + +## Lifetime shape + +```mermaid +flowchart TD + DeviceScope["device service scope"] --> Coordinator["coordinator"] + DeviceScope --> Model["device model"] + DeviceScope --> Publisher["state/device publisher"] + DeviceScope --> Generation["generation scope"] + + Generation --> Observers["observer scopes"] + Generation --> Actions["action endpoint scopes"] + + Observers --> Raw["raw/host, raw/member, raw/peer"] + Actions --> ComponentCaps["cap/component//rpc/..."] + Actions --> Transfer["cap/transfer-manager/main/rpc/send-blob"] + + Model --> State["state/device/..."] + Publisher --> ComponentMeta["cap/component//meta"] +``` + +The service model outlives generations. Catalogue-dependent observers and action endpoints belong to the generation. + +## Public surfaces + +Device should publish: + +```text +svc/device/status +svc/device/meta +cfg/device + +state/device/identity +state/device/components +state/device/component/ +state/device/component//software +state/device/component//update + +cap/component//meta +cap/component//status +cap/component//rpc/ +``` + +Component capability metadata should include backing raw sources or interfaces where relevant. + +Example: + +```text +cap/component/mcu/meta + backing: + facts: + software: raw/member/mcu/state/software + actions: + stage-update: raw/member/mcu/cap/update/main/rpc/stage +``` + +## Raw versus composed truth + +```mermaid +flowchart LR + RawSoftware["raw/member/mcu/state/software"] --> Observer["device observer"] + Observer --> Model["device model"] + Model --> Component["state/device/component/mcu"] + Model --> Software["state/device/component/mcu/software"] + Model --> CapMeta["cap/component/mcu/meta"] +``` + +Raw provider facts stay under `raw/...`. Device publishes the composed appliance view under `state/device/...`. + +## Action rules + +Component actions are public curated controls: + +```text +cap/component//rpc/prepare-update +cap/component//rpc/stage-update +cap/component//rpc/commit-update +``` + +A non-trivial action should run in a request/action scope. The coordinator admits or rejects. The worker waits. + +Actions admitted from public component bus requests must use the canonical request-owner cancellation path: + +```lua +scoped_work.start { + ..., + cancel_op = owner:caller_cancel_op(), +} +``` + +If the caller abandons the action request, the action scope is cancelled and late completion must not reply to the abandoned bus request. + +Fabric-stage action rules: + +```text +open source inside action scope +install termination before waits +call cap/transfer-manager/main/rpc/send-blob +transfer ownership only after successful admission +sanitize ownership internals from public replies +``` + +## Reviewer checklist + +```text +Only Device publishes state/device/... +Raw facts are not mirrored wholesale. +Component ids are stable appliance ids, not raw topology ids. +Capability metadata preserves provenance. +Actions run in request/action scopes. +Action request scopes observe caller abandonment through owner:caller_cancel_op(). +Fabric-stage receiver topics use raw/.../cap/.../rpc/... +Model updates are non-yielding. +Publication failures are explicit. +Config replacement cancels generation-owned observers and actions. +``` + + +## Action dependency admission + +Component actions may declare explicit capability dependencies. Missing or route-missing dependencies reject the individual action with dependency unavailable; they do not fail the Device service. Observation feeds remain tolerant and are projected as component state. diff --git a/docs/specs/fabric-mcu-jsonl-v1.md b/docs/specs/fabric-mcu-jsonl-v1.md new file mode 100644 index 00000000..b64c34d3 --- /dev/null +++ b/docs/specs/fabric-mcu-jsonl-v1.md @@ -0,0 +1,1186 @@ +# Fabric MCU link contract: `fabric-jsonl/1` + +## 1\. Transport + +The wire transport is newline-delimited JSON. + +``` +one JSON object per line +UTF-8 JSON +line terminator: \n +no length prefix +no binary payloads directly in JSON +``` + +Binary transfer chunks are carried as unpadded base64url strings. + +## 2\. Session handshake + +Every session starts with versioned hello frames. + +```json +{ + "type": "hello", + "proto": "fabric-jsonl/1", + "sid": "cm5-session-id", + "node": "cm5", + "identity": null, + "auth": null +} +``` + +```json +{ + "type": "hello_ack", + "proto": "fabric-jsonl/1", + "sid": "mcu-session-id", + "node": "mcu", + "identity": null, + "auth": null +} +``` + +Required fields: + +| Field | Meaning | +| :---- | :---- | +| `type` | `hello` or `hello_ack` | +| `proto` | must be `fabric-jsonl/1` | +| `sid` | non-empty session id | +| `node` | sender node name | +| `identity` | optional claimed stable fabric identity object, or `null` | +| `auth` | optional authentication negotiation/proof object, or `null` | + +`identity` and `auth` are reserved for stable peer identity and authentication. For `fabric-jsonl/1`, the MCU may omit them, send them as null, and ignore them when received. + +If present, `identity` is only a claim until authenticated. Unauthenticated identity must not be used for trust, authorisation, routing policy, update authority, or durable peer identity. + +`node` remains the short wire speaker label. It is not a stable authenticated fabric identity. `sid` remains an ephemeral session id. It is not a peer identity. A local `link_id`, if used by the CM5 implementation, identifies the configured local link instance only. + +Use `node` on the wire. Lua may call it `node_id` internally. + +If `proto` is missing or unsupported, the peer must not treat the session as established. + +Liveness frames: + +```json +{ "type": "ping", "sid": "..." } +{ "type": "pong", "sid": "..." } +``` + +The MCU should treat a new peer session id as a fresh link session. The CM5 will also treat a new MCU session id as a new link session. + +### Reserved identity and authentication objects + +These objects are reserved in v1 so that later fabric links can distinguish local link instances, wire speaker labels, and authenticated fabric peers. + +The MCU v1 implementation is not required to populate or verify these fields. + +Example future identity object: + +```json +{ + "id": "fabric-peer-id", + "kind": "fabric-peer", + "roles": ["bigbox"] +} +``` + +Example future auth object: + +```json +{ + "scheme": "reserved", + "nonce": "base64url-random", + "proof": null +} +``` + +Rules for v1: + +``` +identity may be omitted or null +auth may be omitted or null +unknown identity fields must be ignored +unknown auth fields must be ignored +identity is not trusted unless authentication succeeds +authentication schemes are reserved and not required for MCU v1 +``` + +## 3\. Frame classes + +The frame `type` determines routing. + +| Class | Frame types | +| :---- | :---- | +| session control | `hello`, `hello_ack`, `ping`, `pong` | +| RPC/state | `pub`, `unretain`, `call`, `reply` | +| transfer control | `xfer_begin`, `xfer_ready`, `xfer_need`, `xfer_commit`, `xfer_done`, `xfer_abort` | +| transfer bulk | `xfer_chunk` | + +## 4\. Topics + +Topics are JSON arrays of strings or numbers. + +Examples: + +```json +["state", "self", "software"] +["event", "self", "power", "charger", "alert"] +["cap", "self", "updater", "main", "rpc", "prepare-update"] +``` + +Remote MCU topics follow this model. + +| MCU wire topic prefix | Meaning on CM5 side | +| :---- | :---- | +| `["state", "self", ...]` | retained MCU facts imported to `raw/member/mcu/state/...` | +| `["event", "self", ...]` | MCU events imported to `raw/member/mcu/cap/telemetry/main/event/...` | +| `["cap", "self", "updater", "main", "rpc", ]` | RPC call to MCU updater capability | + +The MCU must not emit `raw/member/...` topics on the wire. Those are CM5-side provenance topics. + +## 5\. Retained state required from the MCU + +The MCU must publish retained state using `pub` frames with `retain: true`. + +The minimum required retained facts for the update system are: + +``` +state/self/software +state/self/updater +``` + +The broader retained state set is: + +``` +state/self/software +state/self/updater +state/self/health +state/self/power/battery +state/self/power/charger +state/self/power/charger/config +state/self/environment/temperature +state/self/environment/humidity +state/self/runtime/memory +``` + +The MCU should publish the current retained state promptly after session establishment, and again whenever it changes. + +### `state/self/software` + +This is required. It describes the image currently booted and running. + +Example: + +```json +{ + "type": "pub", + "topic": ["state", "self", "software"], + "retain": true, + "payload": { + "version": "1.2.3", + "build_id": "2026-05-04T120000Z", + "image_id": "mcu-image-abc123", + "boot_id": "boot-unique-id-456", + "payload_sha256": "64-lowercase-hex" + } +} +``` + +Required fields: + +| Field | Requirement | +| :---- | :---- | +| `image_id` | stable identity of the currently running firmware image | +| `boot_id` | unique identity for this boot of the MCU | + +Strongly recommended fields: + +| Field | Meaning | +| :---- | :---- | +| `version` | human-readable firmware version | +| `build_id` | build identity | +| `payload_sha256` | SHA-256 of the firmware payload from the `.dcmcu` manifest | + +`boot_id` requirements: + +``` +Generated once per MCU boot. +Stable for the whole boot. +Changes after every successful reboot. +Must not be derived only from image_id. +Must be published before the CM5 is expected to reconcile update success. +``` + +This is part of the update correctness contract. The CM5 update flow captures the pre-commit `boot_id` before committing an update. After commit and reboot, it expects to see the target `image_id` and a different `boot_id`. Without this, the CM5 cannot reliably distinguish “image staged or reported” from “new image actually booted”. + +### `state/self/updater` + +This is required. It describes the MCU updater’s current state. + +Example: + +```json +{ + "type": "pub", + "topic": ["state", "self", "updater"], + "retain": true, + "payload": { + "state": "ready", + "last_error": null, + "pending_version": null, + "pending_image_id": null, + "staged_image_id": null, + "job_id": null + } +} +``` + +Required fields: + +| Field | Requirement | +| :---- | :---- | +| `state` | current updater state | +| `last_error` | last updater error, or `null` | + +Recommended fields: + +| Field | Meaning | +| :---- | :---- | +| `pending_version` | version of staged or pending image, if known | +| `pending_image_id` | image expected to be booted after commit | +| `staged_image_id` | image currently staged | +| `job_id` | current update job id, if any | + +Allowed `state` values for v1: + +``` +ready +preparing +receiving +staged +committing +rebooting +running +failed +rollback_detected +``` + +`ready` and `running` both mean the updater is not currently in a failed state. `failed` and `rollback_detected` are terminal failure states for update reconciliation. + +The MCU should publish updater state changes at these points: + +``` +after boot +after prepare accepted +after transfer begins +after transfer is staged +after commit accepted +before reboot, if possible +after reboot +after any update failure +after rollback detection +``` + +The “before reboot” publication is useful but not guaranteed to be observed. The authoritative result of an update is the post-reboot retained state. + +### `state/self/health` + +Recommended. + +Example: + +```json +{ + "type": "pub", + "topic": ["state", "self", "health"], + "retain": true, + "payload": { + "state": "ok" + } +} +``` + +Acceptable simple payloads: + +```json +"ok" +``` + +or: + +```json +{ "state": "ok" } +``` + +Suggested states: + +``` +ok +degraded +failed +unknown +``` + +### Power, charger and environment facts + +These are device model. They are not required for firmware update correctness, but they are part of the expected MCU telemetry surface. + +#### `state/self/power/battery` + +```json +{ + "pack_mV": 12800, + "per_cell_mV": 3200, + "ibat_mA": -120, + "temp_mC": 24500, + "bsr_uohm_per_cell": 12000, + "seq": 42, + "uptime_ms": 123456 +} +``` + +#### `state/self/power/charger` + +```json +{ + "vin_mV": 24000, + "vsys_mV": 12800, + "iin_mA": 500, + "state_bits": 0, + "status_bits": 0, + "system_bits": 0, + "seq": 43, + "uptime_ms": 123500 +} +``` + +The MCU may also include decoded sub-objects: + +```json +{ + "state": {}, + "status": {}, + "system": {} +} +``` + +#### `state/self/power/charger/config` + +```json +{ + "schema": "charger-config/1", + "source": "mcu", + "alert_mask_bits": 0, + "thresholds": { + "vin_lo_mV": 11000, + "vin_hi_mV": 26000, + "bsr_high_uohm_per_cell": 50000 + }, + "alert_mask": { + "vin_lo": true, + "vin_hi": true, + "bsr_high": true, + "bat_missing": true, + "bat_short": true, + "max_charge_time_fault": true, + "absorb": true, + "equalize": true, + "cccv": true, + "precharge": true, + "iin_limited": true, + "uvcl_active": true, + "cc_phase": true, + "cv_phase": true + }, + "seq": 44, + "uptime_ms": 123600 +} +``` + +#### `state/self/environment/temperature` + +```json +{ + "deci_c": 234, + "seq": 45, + "uptime_ms": 123700 +} +``` + +#### `state/self/environment/humidity` + +```json +{ + "rh_x100": 4512, + "seq": 46, + "uptime_ms": 123800 +} +``` + +#### `state/self/runtime/memory` + +```json +{ + "alloc_bytes": 12345, + "seq": 47, + "uptime_ms": 123900 +} +``` + +## 6\. MCU events + +The event is: + +``` +event/self/power/charger/alert +``` + +Example: + +```json +{ + "type": "pub", + "topic": ["event", "self", "power", "charger", "alert"], + "retain": false, + "payload": { + "kind": "vin_lo", + "severity": "warning", + "source": "charger", + "state_bits": 0, + "status_bits": 0, + "system_bits": 0, + "seq": 48, + "uptime_ms": 124000 + } +} +``` + +Known charger alert kinds: + +``` +vin_lo +vin_hi +bsr_high +bat_missing +bat_short +max_charge_time_fault +absorb +equalize +cccv +precharge +iin_limited +uvcl_active +cc_phase +cv_phase +``` + +Unknown alert kinds should still be forwarded; the CM5 will mark them as not known. + +## 7\. Pub/sub frames + +Retained state: + +```json +{ + "type": "pub", + "topic": ["state", "self", "software"], + "retain": true, + "payload": { + "image_id": "mcu-image-abc123", + "boot_id": "boot-unique-id-456" + } +} +``` + +Non-retained event: + +```json +{ + "type": "pub", + "topic": ["event", "self", "power", "charger", "alert"], + "retain": false, + "payload": { + "kind": "vin_lo", + "severity": "warning" + } +} +``` + +Remove retained state: + +```json +{ + "type": "unretain", + "topic": ["state", "self", "updater"] +} +``` + +The MCU should avoid unretaining required facts during normal operation. If a fact is temporarily unknown, prefer retaining a payload with an explicit state such as `unknown`, `unavailable`, or `last_error`. + +## 8\. RPC frames + +Call: + +```json +{ + "type": "call", + "id": "call-123", + "topic": ["cap", "self", "updater", "main", "rpc", "prepare-update"], + "payload": { + "job_id": "job-1", + "target": "mcu", + "expected_image_id": "image-id", + "metadata": {} + } +} +``` + +Successful reply: + +```json +{ + "type": "reply", + "id": "call-123", + "ok": true, + "payload": { + "ready": true + } +} +``` + +Failed reply: + +```json +{ + "type": "reply", + "id": "call-123", + "ok": false, + "err": "not_ready" +} +``` + +Rules: + +``` +id is chosen by caller. +reply.id must match call.id. +ok is authoritative. +err is a short string when ok is false. +payload is arbitrary JSON when ok is true. +Unknown payload fields should be ignored. +``` + +Required MCU updater RPC methods for v1: + +``` +cap/self/updater/main/rpc/prepare-update +cap/self/updater/main/rpc/commit-update +``` + +The previous idea of `receive` is retained conceptually, but not as a required MCU RPC. In v1, binary staging is the transfer target: + +``` +updater/main +``` + +## 9\. Updater RPC behaviour + +### `cap/self/updater/main/rpc/prepare-update` + +Host payload: + +```json +{ + "job_id": "job-1", + "target": "mcu", + "expected_image_id": "image-id", + "metadata": {} +} +``` + +MCU behaviour: + +``` +Check that updater is available. +Check that no incompatible update is already active. +Clear or supersede stale staging state for the same updater target, if safe. +Record job_id and expected_image_id, if supplied. +Publish state/self/updater with state preparing or ready. +Reply ok only when the MCU can accept the transfer. +``` + +Suggested success reply: + +```json +{ + "ready": true, + "target": "updater/main", + "max_chunk_size": 2048 +} +``` + +Suggested failure strings: + +``` +busy +not_ready +unsupported_target +storage_unavailable +invalid_request +``` + +### `cap/self/updater/main/rpc/commit-update` + +Host payload: + +```json +{ + "job_id": "job-1", + "expected_image_id": "image-id", + "metadata": {} +} +``` + +The commit RPC is a cut-over command. Once the CM5 sends it, both the MCU and CM5 may reboot or lose the link immediately. The MCU should reply before reboot if practical, but the CM5 must not require the reply for update success. + +MCU behaviour: + +``` +Check that a staged image exists. +Check that the staged image matches expected_image_id, if supplied. +Validate the staged .dcmcu container. +Make the selected boot image durable according to MCU safety policy. +Record enough updater state to report the result after reboot. +Publish state/self/updater with state committing or rebooting, if practical. +Reply before reboot if practical. +Trigger or allow the reboot/power-cycle. +After reboot, publish state/self/software with image_id and a new boot_id. +After reboot, publish state/self/updater with the final updater state. +``` + +Suggested success reply, if the reply can be delivered before reboot: + +```json +{ + "accepted": true, + "reboot_required": true +} +``` + +Suggested failure strings: + +``` +no_staged_image +image_id_mismatch +validation_failed +commit_failed +not_ready +``` + +A commit reply is advisory. The authoritative result is retained state observed after both devices have restarted and the fabric link has been re-established. + +## 10\. Transfer protocol + +Transfer is used for firmware staging. + +The sender sends raw bytes. The wire encodes chunks as unpadded base64url. Size, offsets and digests are over the raw bytes, not the encoded JSON strings. + +Digest decisions: + +``` +transfer digest_alg = "xxhash32" +transfer digest = lower-case 8-character hex xxHash32, seed 0, over the complete raw artefact + +chunk_digest = lower-case 8-character hex xxHash32, seed 0, over the decoded raw chunk bytes +``` + +The transfer digest is the authoritative transport integrity check for the complete artefact. + +The per-chunk digest is an early corruption-detection and retry aid. It allows the receiver to reject a bad chunk before writing it or before advancing the expected offset. It is not sufficient to prove that the complete transfer is valid. + +Neither digest replaces `.dcmcu` container validation, payload SHA-256 validation, or signature verification. + +### Begin + +```json +{ + "type": "xfer_begin", + "xfer_id": "xfer-123", + "target": "updater/main", + "size": 123456, + "digest_alg": "xxhash32", + "digest": "1a2b3c4d", + "meta": { + "kind": "firmware", + "component": "mcu", + "job_id": "job-1", + "image_id": "expected-image-id", + "format": "dcmcu-v1" + } +} +``` + +The receiver checks: + +``` +target is supported +size is acceptable +digest_alg is supported +digest is syntactically valid for digest_alg +there is no conflicting active transfer +staging storage can be opened +``` + +For `fabric-jsonl/1`, the only required digest algorithm is: + +``` +xxhash32, seed 0, encoded as 8 lower-case hexadecimal characters +``` + +Then the receiver replies: + +```json +{ + "type": "xfer_ready", + "xfer_id": "xfer-123" +} +``` + +and asks for the first chunk: + +```json +{ + "type": "xfer_need", + "xfer_id": "xfer-123", + "next": 0 +} +``` + +The receiver drives chunk offsets explicitly from offset zero. + +### Chunk + +```json +{ + "type": "xfer_chunk", + "xfer_id": "xfer-123", + "offset": 0, + "data": "base64url-without-padding", + "chunk_digest": "9abcdef0" +} +``` + +Required fields: + +| Field | Meaning | +| :---- | :---- | +| `type` | must be `xfer_chunk` | +| `xfer_id` | transfer id | +| `offset` | raw byte offset of this chunk in the complete artefact | +| `data` | unpadded base64url encoding of the raw chunk bytes | +| `chunk_digest` | xxHash32 seed 0 over the decoded raw chunk bytes, as 8 lower-case hex characters | + +Receiver behaviour: + +``` +Check xfer_id matches the active transfer. +Check offset is the next expected offset. +Decode data from unpadded base64url. +Check chunk_digest over the decoded raw bytes. +Reject the chunk if chunk_digest does not match. +Write raw bytes at offset only after the chunk digest has passed. +Advance the expected offset by the decoded byte count. +Send xfer_need with the next offset. +``` + +Example continuation request: + +```json +{ + "type": "xfer_need", + "xfer_id": "xfer-123", + "next": 2048 +} +``` + +Initial MCU chunk size target: + +``` +2048 bytes +``` + +Lua may configure this, but the first MCU implementation should be safe with 2048-byte raw chunks. + +If the receiver detects a bad chunk before advancing the offset, it may either abort the transfer or ask again for the same offset. The CM5 implementation supports the optional same-offset retry for a digest-mismatched chunk by caching the latest sent chunk until the receiver acknowledges the next offset. This is still a stop-and-wait transfer: only the latest outstanding chunk is retryable, and there is no arbitrary seek, resume, or sliding window. + +Recommended simple failure after retry budget is exhausted: + +```json +{ + "type": "xfer_abort", + "xfer_id": "xfer-123", + "err": "chunk_digest_mismatch" +} +``` + +### Commit + +After all bytes are sent: + +```json +{ + "type": "xfer_commit", + "xfer_id": "xfer-123", + "size": 123456, + "digest_alg": "xxhash32", + "digest": "1a2b3c4d" +} +``` + +Receiver verifies: + +``` +received byte count == size +computed xxhash32 over the complete raw artefact == digest +staged object is valid for target +``` + +Then: + +```json +{ + "type": "xfer_done", + "xfer_id": "xfer-123" +} +``` + +`xfer_done` means the complete object digest has been verified and the transferred object has been accepted by the transfer target. For firmware update, it means the `.dcmcu` container is staged sufficiently for the subsequent updater `commit` RPC. It does not mean the new firmware is running. + +Failure at any point: + +```json +{ + "type": "xfer_abort", + "xfer_id": "xfer-123", + "err": "digest_mismatch" +} +``` + +Suggested transfer failure strings: + +``` +busy +unsupported_target +unsupported_digest_alg +size_too_large +storage_unavailable +unexpected_offset +invalid_chunk_encoding +missing_chunk_digest +invalid_chunk_digest +chunk_digest_mismatch +short_transfer +digest_mismatch +validation_failed +timeout +aborted +``` + +--- + +## 11\. Firmware image artefact + +For the bundled MCU update path, the transferred artefact is the complete `.dcmcu` image container, not only an extracted firmware payload. + +The MCU should validate at least: + +``` +container magic +container format version +manifest JSON +manifest schema +component == "mcu" +target compatibility +payload length +payload SHA-256 +signature, when enabled for the product/release path +``` + +The `.dcmcu` v1 manifest concepts are \- described canonically: + +```json +{ + "schema": 1, + "component": "mcu", + "target": { + "product_family": "bigbox", + "hardware_profile": "bb-v1-cm5-2", + "mcu_board_family": "rp2354a" + }, + "build": { + "version": "1.2.3", + "build_id": "build-id", + "image_id": "image-id" + }, + "payload": { + "format": "raw-bin", + "length": 123456, + "sha256": "64-lowercase-hex" + }, + "signing": { + "key_id": "key-id", + "sig_alg": "ed25519" + } +} +``` + +The CM5 may preflight the image, but the MCU remains the authority for whether an image is safe to stage and commit. + +## 12\. Firmware update flow + +The canonical MCU firmware update flow is: + +``` +prepare RPC +transfer to updater/main +durably record CM5 job state before commit +commit RPC +both MCU and CM5 may reboot or lose the link +post-boot retained-state reconciliation +``` + +### Step 1: current state before update + +Before staging, the CM5 observes: + +``` +state/self/software.image_id +state/self/software.boot_id +state/self/updater.state +``` + +The CM5 stores the pre-commit `boot_id` in the durable update job. + +### Step 2: prepare + +Host calls: + +``` +cap/self/updater/main/rpc/prepare-update +``` + +Payload: + +```json +{ + "job_id": "job-1", + "target": "mcu", + "expected_image_id": "image-id", + "metadata": {} +} +``` + +MCU replies success when it can accept a staged update. + +### Step 3: stage + +Host sends the complete `.dcmcu` container using transfer: + +``` +target = "updater/main" +digest_alg = "xxhash32" +digest = xxhash32 over the complete transmitted container +chunk_digest = xxhash32 over each decoded raw chunk +``` + +The MCU must verify each `xfer_chunk.chunk_digest` before accepting the chunk. + +The MCU must still verify the complete transfer digest at `xfer_commit`. + +The MCU replies `xfer_done` only after: + +``` +all chunks have been accepted +received byte count matches xfer_commit.size +complete raw artefact xxhash32 matches xfer_commit.digest +the container has been durably staged enough for the subsequent commit step +``` + +Per-chunk digests are only an early corruption-detection mechanism. They do not prove that the complete artefact is correct. + +### Step 4: durable pre-commit record on CM5 + +Before sending `cap/self/updater/main/rpc/commit-update`, the CM5 must durably record enough information to reconcile after a forced reboot. + +Minimum durable fields: + +``` +job_id +target = mcu +expected_image_id +pre_commit_boot_id +pre_commit_image_id, if known +transfer digest_alg +transfer digest +transfer size +commit_state = about_to_commit or commit_sent +deadline / retry / reconciliation metadata +``` + +Required ordering: + +``` +durable job state written +then commit RPC sent +then assume lights out +``` + +This ordering is required because the MCU reboot also causes the CM5 to reboot. + +### Step 5: commit + +Host calls: + +``` +cap/self/updater/main/rpc/commit-update +``` + +Payload: + +```json +{ + "job_id": "job-1", + "expected_image_id": "image-id", + "metadata": {} +} +``` + +The commit call is a cut-over command. Once sent, the CM5 must assume that both sides may reboot immediately. + +The MCU should reply success if practical: + +```json +{ + "accepted": true, + "reboot_required": true +} +``` + +However, the CM5 must not treat absence of this reply as failure. The update result is determined after reboot. + +### Step 6: post-boot reconciliation + +After CM5 restart, the CM5 resumes the durable update job, reconnects fabric, and waits for retained MCU state. + +The job succeeds when retained state shows: + +``` +state/self/software.image_id == expected_image_id +state/self/software.boot_id != pre_commit_boot_id +state/self/updater.state is not failed or rollback_detected +``` + +The job fails when retained state shows: + +``` +state/self/updater.state == failed +``` + +or: + +``` +state/self/updater.state == rollback_detected +``` + +or when the post-reboot state proves the old or wrong image is running after the reconciliation window. + +Examples: + +``` +software.image_id != expected_image_id +software.boot_id != pre_commit_boot_id +``` + +means the MCU rebooted, but not into the expected image. + +``` +software.image_id == pre_commit_image_id +software.boot_id == pre_commit_boot_id +``` + +means the CM5 has not yet seen proof of a new MCU boot. The CM5 should keep waiting until the reconciliation deadline before declaring failure. + +If the MCU boots the old image after an attempted update, it should publish one of: + +```json +{ + "state": "rollback_detected", + "last_error": "booted_previous_image" +} +``` + +or: + +```json +{ + "state": "failed", + "last_error": "commit_failed" +} +``` + +on: + +``` +state/self/updater +``` + +## 13\. Required implementation decisions frozen for v1 + +``` +Wire protocol name: fabric-jsonl/1 +Handshake field: proto +Hello may reserve identity and auth objects +node is a wire speaker label, not a stable authenticated identity +Topic format: JSON array +RPC reply model: call/reply by id, no reply topics +MCU state prefix: state/self +MCU event prefix: event/self +MCU updater RPC prefix: cap/self/updater/main/rpc +Required updater RPCs: prepare, commit +Transfer target for firmware: updater/main +Transfer digest field names: digest_alg, digest +Required transfer digest algorithm: xxhash32 +xxhash32 format: 8 lower-case hex characters, seed 0 +Chunk encoding: unpadded base64url +Per-chunk digest field: chunk_digest +Per-chunk digest algorithm: xxhash32, seed 0, over decoded raw chunk bytes +Per-chunk digest is required for xfer_chunk in v1 +Final transfer digest remains required and authoritative +Initial chunk size: 2048 bytes +Firmware artefact: complete .dcmcu container +boot_id is required update-correctness state +Commit is a cut-over command +Commit reply is advisory +Update success requires post-boot image_id and boot_id reconciliation +``` + +## 14\. Compatibility deliberately not carried forward into the MCU contract + +The MCU should not implement these ambiguities as part of the new v1 contract: + +``` +unversioned hello +checksum field +implicit first chunk after xfer_ready +xfer_chunk without chunk_digest +receiver-as-local-bus-topic in transfer metadata +receive RPC as the required staging path +raw/member/... topic names on the wire +update success based only on commit RPC success +update success without boot_id reconciliation +per-chunk digest as a substitute for final transfer digest +transfer digest as a substitute for .dcmcu payload SHA-256 or signature validation +``` + diff --git a/docs/specs/fabric.md b/docs/specs/fabric.md new file mode 100644 index 00000000..9d1ad56b --- /dev/null +++ b/docs/specs/fabric.md @@ -0,0 +1,139 @@ +# Fabric service guide + +## Purpose + +Fabric owns local link/session orchestration and cross-node message and transfer plumbing. + +It is responsible for: + +```text +configured links +link lifecycle +session establishment +bus publish/call bridging +imported retained state cleanup +transfer-manager capability +fabric-domain state publication +``` + +Fabric is not the Device service. It does not publish appliance composition truth. + + +## Wire contracts + +The MCU/CM5 v1 wire contract is documented separately in: + +```text +docs/specs/fabric-mcu-jsonl-v1.md +``` + +That document is the canonical contract for `fabric-jsonl/1` MCU links, including session handshake, topic mapping, updater RPCs, transfer framing, digest rules, and post-boot update reconciliation. + +## Lifetime shape + +```mermaid +flowchart TD + FabricScope["fabric service scope"] --> Shell["service shell"] + FabricScope --> Model["fabric model"] + FabricScope --> Publisher["publisher"] + FabricScope --> TransferCap["cap/transfer-manager/main"] + + Shell --> Generation["generation scope"] + Generation --> LinkA["link scope: link-a"] + Generation --> LinkB["link scope: link-b"] + + LinkA --> Reader["reader component"] + LinkA --> Session["session/session-control"] + LinkA --> Bridge["bus RPC/pub bridge"] + LinkA --> Transfer["transfer manager"] + LinkA --> Writer["writer component"] + + TransferCap --> Transfer +``` + +A generation owns configured links. A link owns its transport, session machinery, bridge and transfer lanes. + +## Public surfaces + +Fabric should publish: + +```text +svc/fabric/status +svc/fabric/meta +cfg/fabric + +state/fabric/link/ +state/fabric/link//component/ + +cap/transfer-manager/main/meta +cap/transfer-manager/main/status +cap/transfer-manager/main/rpc/send-blob +``` + +Imported remote truth should be provenance-bearing: + +```text +raw/member//... +raw/peer//... +``` + +Fabric summaries belong under `state/fabric/...`; imported peer/member facts belong under `raw/...`. + +## Coordinator events + +Typical Fabric events: + +```text +config changed +generation completed +link completed +link state changed +transfer request received +session established +session dropped +publisher requested +``` + +Coordinator branches should not perform link I/O. Link I/O belongs in link components. + +## Ownership rules + +```text +service owns model and public transfer-manager capability +generation owns configured link scopes +link owns transport/session/bridge/transfer resources +transfer attempt owns source handoff until receiver accepts it +reader/writer own stream-side waits +``` + +Transfer handoff must be explicit: + +```text +before admission: caller owns source cleanup +after successful receiver admission: receiver owns source cleanup +on failed admission: caller still terminates source +``` + +Public transfer-manager requests are caller-owned bus requests. They should be admitted as `scoped_work`, not raw spawned fibres, and should pass `cancel_op = owner:caller_cancel_op()`. If the caller abandons `send-blob`, the transfer request scope is cancelled, source ownership is terminated or released according to the current handoff point, and late completion must not reply to the abandoned request. + +Local-to-remote bridge calls follow the same rule where the local bus caller is the owner. Remote-to-local bridge work is cancelled by Fabric session/protocol state rather than by `caller_cancel_op()` unless a local bus `Request` is the admitted owner. + +## Reviewer checklist + +```text +Link work is scoped. +Link completions carry link id and generation. +Imported retained state is cleared on peer session drop. +Transfer source ownership is transferred visibly. +send-blob does not leak ownership internals in public replies. +send-blob request work observes caller abandonment through owner:caller_cancel_op(). +Local-to-remote bridge calls propagate caller cancellation into outbound scoped work. +Fabric state is under state/fabric/... +Public transfer manager is under cap/transfer-manager/... +No Fabric coordinator branch waits on transport or bus calls. +``` + + +## Transport dependency admission + +HAL-backed Fabric links depend on their raw host transport capability. Missing status, `available=false`, or `no_route` during transport open are admission failures and should leave the generation waiting for transport. Transport accepted and then protocol/session failure is a Fabric domain failure. diff --git a/docs/specs/hal.md b/docs/specs/hal.md new file mode 100644 index 00000000..04f0a7a2 --- /dev/null +++ b/docs/specs/hal.md @@ -0,0 +1,632 @@ +# HAL architecture note + +## 1. Purpose + +HAL is the compatibility and capability boundary between Devicecode services and concrete platform implementations. + +It should make platform-specific behaviour available through a stable service shape while keeping the rest of the system: + +```text +responsive +structured +testable +scope-owned +diagnosable +independent of driver details +``` + +HAL has two jobs: + +```text +present stable capability contracts to services +adapt current and legacy provider implementations behind a compatibility seam +``` + +The design aim is that `device`, `update`, `ui`, and Fabric can depend on HAL capabilities without inheriting provider-specific lifetime, cancellation, or cleanup behaviour. + +--- + +## 2. Core model + +HAL is organised around five roles. + +```text +HAL service coordinator + Owns provider registration, capability indexes, and service-level state. + Selects events at its control point. + Must not suspend inside event branches. + +Capability provider + Owns one concrete capability implementation. + Exposes operations through a stable HAL contract. + May wrap platform, driver, or legacy implementation details. + +Operation scope + Owns one meaningful HAL operation. + Owns cancellation, timeout, cleanup, diagnostics, and completion. + Reports completion through the standard envelope. + +Compatibility adapter + Bridges old or irregular provider APIs into the current HAL model. + Is the only place where compatibility compromises should live. + +Driver/backend + Performs concrete work. + May block only through Ops selected by a scoped operation or owner. +``` + +The central rule is: + +```text +HAL coordinators decide and route; operation scopes perform and clean up. +``` + +--- + +## 3. Compatibility seam + +HAL deliberately contains a compatibility seam. + +The seam exists so that older or lower-level implementations can be adapted without leaking their shape into service code. + +Compatibility code may translate: + +```text +legacy method names +older reply shapes +driver-specific error values +backend-specific status values +different capability metadata forms +older start/stop conventions +``` + +Compatibility code must not weaken the service model. + +It must preserve: + +```text +scope-owned lifetime +bounded finalisers +standard completion shape +explicit cancellation +stale-safe identity +non-suspending coordinator branches +stable capability contracts +``` + +The compatibility seam should be narrow and named. It should normally live in adapter modules or provider wrappers, not in service coordinators. + +A useful rule is: + +```text +Compatibility belongs at the edge. HAL semantics belong in the middle. +``` + +--- + +## 4. Coordinator rule + +A strict HAL coordinator has one suspending point: its control point. + +```lua +while true do + local ev = fibers.perform(next_hal_event_op(state, scope)) + + if ev.kind == "stop" then + return + end + + local decision = reduce_event(state, ev) + dispatch_effects_nonblocking(state, decision.effects) +end +``` + +After event selection, the coordinator must not suspend before returning to the control point. + +Not allowed in coordinator branches: + +```lua +fibers.perform(provider:operation_op(...)) +fibers.perform(scope:join_op()) +fibers.perform(tx:send_op(...)) -- if it can block +fibers.perform(rx:recv_op(...)) +sleep.sleep(...) +scope:join(...) +``` + +Allowed in coordinator branches: + +```text +mutate coordinator-owned indexes +record pending operations +create a child scope +spawn operation work into a child scope +cancel a child scope +ignore stale completions +publish through APIs documented as immediate +make an or_else-backed immediate admission attempt +reply or fail an immediate in-memory request object +``` + +The important constraint is suspension. A coordinator may use a helper documented as non-yielding, including an `or_else`-backed immediate admission helper. + +--- + +## 5. Capability contracts + +A HAL capability contract should describe: + +```text +capability class and id +supported verbs +request argument shape +reply shape +error shape +operation lifetime +cancellation behaviour +cleanup ownership +diagnostic fields +``` + +Provider-specific details should not appear in callers. + +Callers should see stable capabilities such as: + +```text +control operation +status publication +metadata publication +event publication +operation completion +``` + +Provider implementations may vary internally, but the exported HAL contract should not. + +Where a provider has a legacy or irregular shape, add an adapter. Do not teach every caller about that irregularity. + +--- + +## 6. Operation scopes + +Use an operation scope when HAL work has any of the following: + +```text +blocking backend calls +temporary handles +owned buffers or files +rollback or cleanup +timeout ownership +retry state +child work +diagnostic value +completion result +``` + +A HAL operation scope should: + +```text +own the operation lifetime +register cleanup with scope:finally(...) +perform backend Ops according to local policy +return one result table on success +let unexpected errors escape to the scope +``` + +Example shape: + +```lua +local function run_hal_operation(scope, ctx, req) + scope:finally(function(aborted, status, primary) + cleanup_owned_resources() + end) + + local result = fibers.perform(ctx.backend:operation_op(req.args)) + + return { + value = result, + } +end +``` + +The coordinator starts the operation scope and observes completion later as an event. It does not perform the operation inline and does not run the operation cleanup. + +--- + +## 7. Finalisers and cleanup + +HAL cleanup is finaliser-first. + +If a HAL operation owns a resource, cleanup should be registered in the operation scope with `scope:finally(...)`. + +Finalisers must run promptly. A finaliser may perform only an Op that is ready now. In practice this permits explicit try-now helpers built with `or_else(...)`, and rejects hidden waits. + +Allowed in finalisers: + +```text +terminate immediate handles +detach local registrations +release local reservations +mark local objects closed +abandon explicitly safe-to-abandon resources +remove immediate local state +perform an explicitly ready-now Op, such as an or_else-backed try helper +``` + +Not allowed in finalisers: + +```text +performing an Op that may wait +sleep +join +join_op +queue send that may block +request/reply protocols +normal shutdown handshakes +backend calls that may block +close_op +``` + +Resources owned by finalisers must provide: + +```text +terminate(reason) +``` + +If cleanup needs protocol progress or waiting, it belongs in the operation body before completion, not in the finaliser. Graceful cleanup belongs in `close_op()` and must be called explicitly outside finalisers. + +--- + +## 8. Completion events + +HAL scoped work should use the standard completion envelope. + +```lua +{ + kind = "hal_operation_done", + operation_id = operation_id, + generation = generation, + + status = st, + report = rep, + + result = result_table_or_nil, + primary = primary_error_or_reason_or_nil, +} +``` + +Rules: + +```text +status == "ok" -> result is present +status == "failed" -> primary is present +status == "cancelled" -> primary is present +``` + +Successful operations return one result table. + +```lua +return { + reply = reply, +} +``` + +Coordinator completion handling must be stale-safe. + +```lua +local rec = state.operations[ev.operation_id] +if not rec or rec.generation ~= ev.generation then + return +end +``` + +A missing operation record is a stale completion. Stale completion is harmless and usually quiet. + +--- + +## 9. Cancellation and timeout policy + +Cancellation and timeout belong to the operation scope that owns the work. + +A coordinator may cancel an operation scope, but must not join it inline. + +HAL capability calls are bus requests. When a caller abandons a capability RPC, the HAL service should propagate that abandonment into the admitted HAL operation where the operation is caller-owned. `ControlRequest` carries an optional `cancel_op`; managers and drivers that start scoped work should pass it into `scoped_work.start`. + +For simple immediate handlers, caller abandonment may simply stop reply delivery. For blocking or resource-owning operations, caller abandonment must cancel or terminate the owned operation scope. + +There are two normal service-side cancellation policies. + +```text +reply_on_abandon + The coordinator cancels the operation scope, replies immediately, clears its + record, and treats later completion as stale. + +reply_on_scope_completion + The coordinator cancels the operation scope, keeps its record, and replies only + when the operation completion event arrives. +``` + +Use `reply_on_abandon` when the caller only needs to know that HAL has abandoned the operation. + +Use `reply_on_scope_completion` when the caller is promised that cleanup, rollback, or hardware state has reached a known point. + +The chosen policy should be explicit in the operation record where it matters. + +--- + +## 10. Provider lifecycle + +Provider start and stop should be scope-owned. + +A provider may own: + +```text +backend handles +subscriptions +watchers +driver state +operation limiters +published state +``` + +The provider scope should install finalisers for provider-owned cleanup. + +Provider start may publish initial metadata and state through APIs documented as immediate. If provider start requires blocking setup, that setup should be represented as scoped work, not hidden in a coordinator branch. + +Provider stop should normally mean: + +```text +cancel provider scope +let provider finalisers release owned resources +observe provider completion at the service boundary +ignore stale operation completions +``` + +Provider stop should not require a coordinator to synchronously join child operations. + +--- + +## 11. Admission and backpressure + +HAL must make admission policy explicit. + +For bounded in-flight operations, the coordinator may reject immediately: + +```text +busy +unavailable +unsupported +invalid_args +provider_stopped +``` + +If a bounded queue or operation lane is full, the policy must be clear: + +```text +reject the request +cancel the provider +mark degraded +drop optional telemetry +start no work +``` + +A strict coordinator should use an immediate admission helper based on `or_else` when it needs a non-blocking handoff. + +It should not block on queue capacity. + +--- + +## 12. Error policy + +Inside operation scopes, owners, and providers, ordinary unexpected errors should normally escape. + +Use `pcall` only where there is local recovery policy: + +```text +convert a backend negative result into a protocol reply +retry a transient operation +isolate optional or untrusted code +perform best-effort bounded cleanup +``` + +If catching only to log, rethrow. + +Use `status` for control flow and `report` for diagnosis. Do not make ordinary service logic depend on deep report structure. + +--- + +## 13. Module role contracts + +```text +hal service coordinator + role: provider and capability coordinator + may suspend: service control-point event selection only + owns: provider indexes, capability indexes, service-level state + delegates: provider scopes and operation scopes + +provider module + role: capability implementation boundary + owns: provider-local state and provider-owned resources + exposes: stable capability contract + may use: scoped operation work + +compatibility adapter + role: legacy or backend shape adapter + owns: translation between old and current contracts + must not own: service policy + +operation module + role: scoped work unit + may perform: backend Ops, timeout/cancellation choice, local operation policy + owns: one HAL operation lifetime + accepts: request.cancel_op where admitted from caller-owned capability RPC + cleans up: with scope:finally(...) + +resource helper + role: bounded cleanup helper + must not call: close_op; perform on Ops that may wait +``` + +--- + +## 14. Test principles + +HAL tests should cover compatibility and losing paths, not only happy paths. + +Important tests: + +```text +provider publishes stable capability metadata +legacy provider shapes are adapted at the seam +unsupported verbs return stable negative replies +invalid arguments return stable negative replies +busy is returned when max in-flight is reached +operation completion uses the standard envelope +operation cancellation runs finalisers +caller-abandoned capability RPC cancels admitted owned work +operation timeout cancels owned work +stale completions are ignored +provider stop cancels in-flight operation scopes +finalisers do not perform Ops that may wait and do not call close_op +resources are cleaned up by the scope that owns them +backend errors escape to the owning operation scope unless locally handled +``` + +For immediate admission helpers, test both cases: + +```text +admission succeeds immediately +admission would block and returns immediately without suspension +``` + +For compatibility adapters, test that callers see the same public contract regardless of backend shape. + +--- + +## 15. Review checklist + +For a HAL coordinator: + +```text +Is there one suspending wait at the control point? +Can any event branch suspend? +Are operation lifetimes represented as scopes? +Are child scopes cancelled but not joined inline? +Is admission immediate and explicit? +Are completions identified and stale-safe? +Are provider indexes and capability indexes coordinator-owned? +Are backend calls kept out of coordinator branches? +``` + +For a provider: + +```text +What capability contract does it expose? +What resources does it own? +Which scope owns those resources? +What finalisers release them? +What operations may it start? +What happens when it is stopped with work in flight? +``` + +For an operation scope: + +```text +What lifetime does this operation own? +What resources are admitted into it? +Are resources cleaned up by scope:finally(...)? +Is cleanup bounded and non-yielding? +How are timeout and cancellation handled, including caller-abandoned bus requests? +What one result table is returned on success? +What failures are normal negative replies, and what failures should escape? +``` + +For a compatibility adapter: + +```text +What legacy or backend shape is being adapted? +Is the compatibility contained at the edge? +Does the public HAL contract remain stable? +Does the adapter preserve cancellation, cleanup, and completion semantics? +Does it avoid introducing blocking work into coordinator paths? +``` + +--- + +## 16. Compact rules + +```text +HAL coordinators block only at their control point. +Capability contracts are stable; compatibility lives at the edge. +Coordinators route and decide; operation scopes perform. +A fibre runs code; a scope owns lifetime. +Use scopes for HAL operations with blocking work, cleanup, timeout, or diagnostics. +Use scope:finally(...) for scope-owned cleanup. +Finalisers must be bounded and non-yielding. +Do not call close_op in finalisers. +Use standard completion envelopes. +Carry operation identity and generation on completions. +Ignore stale completions. +Use explicit admission policy for bounded in-flight work. +Use or_else-backed immediate admission where a coordinator needs non-blocking handoff. +Let unexpected errors escape to the owning scope. +Use reports for diagnosis, not ordinary control flow. +Keep compatibility seams narrow, named, and out of callers. +``` + +--- + +## 17. Short mental model + +HAL is the compatibility and capability boundary. + +```text +coordinator: + indexes providers and capabilities + admits or rejects work + starts operation scopes + observes completion + does not perform backend work + +provider: + implements a stable capability contract + hides backend details + owns provider-local lifetime through scope + +operation scope: + owns one HAL operation + performs backend work + owns timeout and cancellation + cleans up with finalisers + reports completion + +adapter: + translates old or irregular backend shape + preserves HAL semantics + keeps compatibility out of callers +``` + +The coordinator does not ask: + +```text +can I just call the backend here? +can I wait for this operation here? +can I clean this up manually later? +``` + +It asks: + +```text +what capability is being requested? +can the work be admitted now? +which scope owns the operation? +what completion should I later accept or ignore? +``` + +The operation scope asks: + +```text +what resources do I own? +what finalisers release them? +what timeout and cancellation policy applies? +what result or failure crosses the boundary? +``` diff --git a/docs/specs/hal/band_driver.md b/docs/specs/hal/band_driver.md new file mode 100644 index 00000000..964cc168 --- /dev/null +++ b/docs/specs/hal/band_driver.md @@ -0,0 +1,432 @@ +# Band Driver (HAL) + +## Description + +The Band Driver is a HAL component that exposes DAWN (Distributed Access point with No-manager) band steering daemon configuration to the rest of the system via a single `band` capability. It: + +1. **Holds staged configuration** — all `set_*` calls accumulate changes in-memory. No config system is touched until `apply()` is called. +2. **Applies staged config on demand** — `apply()` passes the staged table to `backend:apply(staged)`. The backend is responsible for writing to the underlying config system (e.g. UCI) and restarting the daemon. +3. **Clears backend state on demand** — `clear()` calls `backend:clear()` to reset the backend's config to a known base state, ready for a fresh set of staged calls. + +The driver never touches UCI or any OS config system. All writes are delegated to the active backend. A future non-OpenWrt backend can implement the same interface without any UCI involvement. + +## Dependencies + +- `backends/band/provider.lua` — selects the active backend at driver creation. +- DAWN must be installed on the target device. If `backend:clear()` fails on startup (e.g. DAWN config absent), the WLAN Manager logs the error and does not emit a device-added event for the band capability. + +## Initialisation + +On creation by the WLAN Manager: + +1. Resolve the band backend via `provider.get_backend()`. If no supported backend is found, creation fails with an error. +2. Call `backend:clear()`. The backend is responsible for resetting the DAWN config to a clean base state. If `clear` fails, creation fails with an error. +3. Publish capability `meta`. +4. Start the RPC handler fiber. + +## Capability + +Class: `band` +Id: `'1'` + +### Meta (retained) + +Topic: `{'cap', 'band', '1', 'meta'}` + +```lua +{ + provider = 'hal', + version = 1, +} +``` + +### Offerings + +All offerings return `{ ok = true }` on success or `{ ok = false, reason = }` on validation failure or internal error. Changes are staged in-memory until `apply` is called. + +--- + +#### set_log_level + +Topic: `{'cap', 'band', '1', 'rpc', 'set_log_level'}` + +Stages the daemon log level. + +Input: + +```lua +{ + level = , -- required: non-negative integer +} +``` + +--- + +#### set_kicking + +Topic: `{'cap', 'band', '1', 'rpc', 'set_kicking'}` + +Stages the global client-kicking policy. + +Input: + +```lua +{ + mode = , -- required: "none"|"compare"|"absolute"|"both" + bandwidth_threshold = , -- required: non-negative + kicking_threshold = , -- required: non-negative + evals_before_kick = , -- required: non-negative integer +} +``` + +Mode-to-integer mapping (used by the backend): `none=0`, `compare=1`, `absolute=2`, `both=3`. + +--- + +#### set_station_counting + +Topic: `{'cap', 'band', '1', 'rpc', 'set_station_counting'}` + +Stages the station-counting policy. + +Input: + +```lua +{ + use_station_count = , -- required + max_station_diff = , -- required: non-negative integer +} +``` + +--- + +#### set_rrm_mode + +Topic: `{'cap', 'band', '1', 'rpc', 'set_rrm_mode'}` + +Stages the RRM (Radio Resource Management) mode. + +Input: + +```lua +{ + mode = , -- required: one of "PAT" +} +``` + +--- + +#### set_neighbour_reports + +Topic: `{'cap', 'band', '1', 'rpc', 'set_neighbour_reports'}` + +Stages neighbour report parameters. + +Input: + +```lua +{ + dyn_report_num = , -- required: non-negative integer (coerced with tonumber) + disassoc_report_len = , -- required: non-negative integer (coerced with tonumber) +} +``` + +--- + +#### set_legacy_options + +Topic: `{'cap', 'band', '1', 'rpc', 'set_legacy_options'}` + +Input: + +```lua +{ + opts = , -- required: key-value table; valid keys: "eval_probe_req", "eval_assoc_req", "eval_auth_req", "min_probe_count", "deny_assoc_reason", "deny_auth_reason" +} +``` + +Each key-value pair is staged in the driver's in-memory config table under the global section. The backend maps driver-level keys to the appropriate config system entries. Unknown keys or nil values return `ok = false`. + +--- + +#### set_band_priority + +Topic: `{'cap', 'band', '1', 'rpc', 'set_band_priority'}` + +Stages the initial score (priority) for a frequency band. + +Input: + +```lua +{ + band = , -- required: "2G" or "5G" (case-insensitive, normalised to upper) + priority = , -- required: non-negative number +} +``` + +The backend maps `2G`/`5G` to the appropriate config section names. + +--- + +#### set_band_kicking + +Topic: `{'cap', 'band', '1', 'rpc', 'set_band_kicking'}` + +Stages per-band RSSI and channel-utilisation scoring parameters. + +Input: + +```lua +{ + band = , -- required: "2G" or "5G" + options =
, -- required: key-value table of scoring parameters +} +``` + +Valid option keys and their descriptions (all values must be numbers). The backend maps these to the appropriate config system keys: + +| Option key | Description | +|----------------------------------|------------------------------------------------| +| `rssi_center` | RSSI centre value | +| `rssi_reward_threshold` | RSSI good threshold | +| `rssi_reward` | RSSI good reward | +| `rssi_penalty_threshold` | RSSI bad threshold | +| `rssi_penalty` | RSSI bad penalty | +| `rssi_weight` | RSSI weight | +| `channel_util_reward_threshold` | Channel utilisation good threshold | +| `channel_util_reward` | Channel utilisation good reward | +| `channel_util_penalty_threshold` | Channel utilisation bad threshold | +| `channel_util_penalty` | Channel utilisation bad penalty | + +Unknown keys return `ok = false`. Values that cannot be coerced to a number return `ok = false`. + +--- + +#### set_support_bonus + +Topic: `{'cap', 'band', '1', 'rpc', 'set_support_bonus'}` + +Input: + +```lua +{ + band = , -- required: "2G" or "5G" + support = , -- required: "ht" or "vht" + reward = , -- required +} +``` + +The backend maps driver-level keys to the appropriate config entries. Unknown keys return `ok = false`. + +Topic: `{'cap', 'band', '1', 'rpc', 'set_update_freq'}` + +Stages update frequencies for internal DAWN polling loops. + +Input: + +```lua +{ + updates =
, -- required: key-value table; valid keys: "client","chan_util","hostapd","beacon_reports","tcp_con" +} +``` + +Values must be non-negative numbers. Unknown keys return `ok = false`. + +--- + +#### set_client_inactive_kickoff + +Topic: `{'cap', 'band', '1', 'rpc', 'set_client_inactive_kickoff'}` + +Stages the inactive client kickoff timeout. + +Input: + +```lua +{ + timeout = , -- required: non-negative integer (coerced with tonumber) +} +``` + +--- + +#### set_cleanup + +Topic: `{'cap', 'band', '1', 'rpc', 'set_cleanup'}` + +Stages cleanup timeouts for probes, clients, and APs. + +Input: + +```lua +{ + timeouts =
, -- required: key-value table; valid keys: "probe","client","ap" +} +``` + +Values must be non-negative numbers. Unknown keys return `ok = false`. + +--- + +#### set_networking + +Topic: `{'cap', 'band', '1', 'rpc', 'set_networking'}` + +Stages the DAWN inter-AP networking method and options. + +Input: + +```lua +{ + method = , -- required: "broadcast"|"tcp+umdns"|"multicast"|"tcp" + options =
, -- required: key-value table of optional networking settings +} +``` + +Valid option keys (the backend maps these to the appropriate config entries): + +| Key | Type | +|----------------------|-----------| +| `ip` | string | +| `port` | number | +| `broadcast_port` | number | +| `enable_encryption` | boolean | + +Unknown keys or type mismatches return `ok = false`. + +--- + +#### apply + +Topic: `{'cap', 'band', '1', 'rpc', 'apply'}` + +Passes the full staged config table to `backend:apply(staged)`. The backend is responsible for writing to the config system and triggering a daemon restart. The backend coalesces rapid successive restarts with a 1-second debounce window. + +Input: none (empty object `{}`). + +--- + +#### clear + +Topic: `{'cap', 'band', '1', 'rpc', 'clear'}` + +Calls `backend:clear()` to reset the backend's DAWN config to a known base state. The in-memory staged config is also reset to empty. This should be called before beginning a new configuration sequence. + +Input: none (empty object `{}`). + +--- + +#### rollback + +Topic: `{'cap', 'band', '1', 'rpc', 'rollback'}` + +Discards all staged changes without any backend call. The staged config is reset to empty (same state as after `clear`). + +Input: none (empty object `{}`). + +--- + +## Types + +All capability arg and reply types follow the conventions in `src/services/hal/types/`. + +### `BandSetKickingOpts` + +```lua +---@class BandSetKickingOpts +---@field mode string -- "none"|"compare"|"absolute"|"both" +---@field bandwidth_threshold number +---@field kicking_threshold number +---@field evals_before_kick number +``` + +### `BandSetStationCountingOpts` + +```lua +---@class BandSetStationCountingOpts +---@field use_station_count boolean +---@field max_station_diff number +``` + +### `BandSetNeighbourReportsOpts` + +```lua +---@class BandSetNeighbourReportsOpts +---@field dyn_report_num number +---@field disassoc_report_len number +``` + +### `BandSetBandPriorityOpts` + +```lua +---@class BandSetBandPriorityOpts +---@field band string -- "2G" or "5G" +---@field priority number +``` + +### `BandSetBandKickingOpts` + +```lua +---@class BandSetBandKickingOpts +---@field band string -- "2G" or "5G" +---@field options table -- see set_band_kicking valid option keys +``` + +### `BandSetSupportBonusOpts` + +```lua +---@class BandSetSupportBonusOpts +---@field band string -- "2G" or "5G" +---@field support string -- "ht" or "vht" +---@field reward number +``` + +### `BandSetNetworkingOpts` + +```lua +---@class BandSetNetworkingOpts +---@field method string -- "broadcast"|"tcp+umdns"|"multicast"|"tcp" +---@field options table +``` + +### `BandCapabilityReply` + +```lua +---@class BandCapabilityReply +---@field ok boolean +---@field reason string|nil -- present on failure +``` + +## Backend Contract + +The OpenWrt-DAWN band backend (`backends/band/providers/openwrt-dawn/impl.lua`) must implement: + +| Function | Description | +|-------------------|----------------------------------------------------------------------------------------------------------------| +| `is_supported()` | Returns `true` if DAWN is installed (checks `/etc/config/dawn` exists and is readable) | +| `clear()` | Deletes non-`hostapd` DAWN UCI sections; creates required fixed sections (`global`, `802_11g`, `802_11a`, `gbltime`, `gblnet`, `localcfg`) with default values; commits. Leaves the daemon in a clean state ready for fresh staged config. | +| `apply(staged)` | Writes the staged config table to the config system and triggers `service dawn restart`. The 1-second debounce coalescing is an internal backend concern. | + +Band section name mapping (`2G → 802_11g`, `5G → 802_11a`) is a detail of the backend, not the driver. The driver passes band identifiers as-is in the staged table. + +The driver has no import of or dependency on `backends/common/uci.lua`. + +## Service Flow + +1. Resolve backend; if unsupported, stop with error. +2. Call `backend:clear()`; if it fails, stop with error. +3. Publish meta. +4. Enter `named_choice` loop on `control_ch` and scope cancellation. +5. On each RPC: validate inputs; update staged config table; reply `{ ok = true }` or `{ ok = false, reason = ... }`. +6. On `apply`: pass staged config table to `backend:apply(staged)`. +7. On `clear`: call `backend:clear()`; reset in-memory staged config. +8. On `rollback`: discard staged config table. +9. On scope cancelled: exit loop. + +## Architecture + +- The driver runs a single RPC handler fiber. There is no autonomous emission from the band driver. +- Staged config is a plain Lua table local to the driver. It is only ever accessed by the single RPC handler fiber (no concurrent access). +- `apply` passes the staged config table to `backend:apply(staged)`. The driver has no knowledge of how the backend writes the config. +- The clear sequence on startup is fully owned by the backend via `backend:clear()`. If it fails, the driver stops without advertising the capability. +- A `finally` block on the driver scope logs the reason for shutdown. diff --git a/docs/specs/hal/cpu_driver.md b/docs/specs/hal/cpu_driver.md new file mode 100644 index 00000000..db055f52 --- /dev/null +++ b/docs/specs/hal/cpu_driver.md @@ -0,0 +1,119 @@ +# CPU Driver (HAL) + +## Description + +The CPU driver is a HAL component that exposes CPU metrics to the rest of the system via a single `cpu` capability. It provides utilisation and frequency readings sourced from `/proc/stat` and `/sys/devices/system/cpu/cpuN/cpufreq/scaling_cur_freq`. The model string and core count are read once at startup from `/proc/cpuinfo` and published in the capability meta. + +All metrics are available via a single `get` RPC offering that accepts a `field` name and a `max_age` parameter. The driver maintains a shared cache of the last full set of computed readings. Any `get` call within `max_age` seconds of the last read is served from cache, avoiding the blocking 1-second double-sample required for utilisation calculation. + +The CPU driver is created and owned by the Sysmon Manager. It does not manage its own lifecycle. + +## Dependencies + +None. The driver reads directly from procfs and sysfs. + +## Initialisation + +On creation by the Sysmon Manager: + +1. Read `/proc/cpuinfo` to determine CPU model string and core count. +2. Publish capability `meta` — includes model and core count. +3. Start the RPC handler fiber. + +The readings cache is empty at startup. The first `get` call always triggers a fresh read. + +## Capability + +Class: `cpu` +Id: `'1'` + +### Meta (retained) + +Topic: `{'cap', 'cpu', '1', 'meta'}` + +```lua +{ + provider = 'hal', + version = 1, + model = , -- cpu model string e.g. "Qualcomm Atheros QCA9558" + core_count = , -- number of logical cores +} +``` + +### Offerings + +#### get + +Topic: `{'cap', 'cpu', '1', 'rpc', 'get'}` + +Input (`CpuGetOpts`): + +```lua +{ + field = , -- required: one of the field names listed below + max_age = , -- required: maximum acceptable age of reading in seconds +} +``` + +Available fields: + +| Field | Type | Description | +|--------------------|-----------------|------------------------------------------------| +| `utilisation` | number | Overall CPU utilisation as a percentage (0–100)| +| `core_utilisations`| table | Per-core utilisation `{ cpu0 = %, cpu1 = %, ... }` | +| `frequency` | number | Average frequency across all cores in kHz | +| `core_frequencies` | table | Per-core frequency `{ cpu0 = kHz, cpu1 = kHz, ... }` | + +Reply on success: + +```lua +{ ok = true, reason = } +``` + +Reply on failure: + +```lua +{ ok = false, reason = } +``` + +If the requested `field` string is not one of the four above, the driver replies with `ok = false`. + +## Cache Behaviour + +The driver uses `shared/cache.lua` (`cache.new()`), storing each field under its field name as the key. The `max_age` value from the RPC request is passed as the timeout to `cache:get(field, max_age)`. + +On any `get` call: + +1. Call `cache:get(field, max_age)`. If a non-nil value is returned, reply with it immediately. +2. Otherwise perform a fresh read and call `cache:set` for each computed field: + - For `utilisation` or `core_utilisations`: double-sample `/proc/stat` (with 1-second sleep), compute both fields, then `cache:set('utilisation', value)` and `cache:set('core_utilisations', value)`. + - For `frequency` or `core_frequencies`: read `scaling_cur_freq` for each core, compute both fields, then `cache:set('frequency', value)` and `cache:set('core_frequencies', value)`. + +The two pairs (`utilisation`/`core_utilisations` and `frequency`/`core_frequencies`) are always refreshed together since they derive from the same underlying read. A cache miss for either field in a pair refreshes both. + +## Service Flow + +```mermaid +flowchart TD + St[Start] --> A(Read /proc/cpuinfo for model + core_count) + A --> B(Publish meta) + B --> C{Wait for get RPC or scope cancelled} + C -->|get RPC| D{Cache fresh enough? now - read_at <= max_age} + D -->|yes| E(Return cached field value) + D -->|no| F(Read /proc/stat sample 1) + F --> G(Sleep 1 second) + G --> H(Read /proc/stat sample 2) + H --> I(Read scaling_cur_freq for each core) + I --> J(Compute all four fields, update cache) + J --> E + E --> C + C -->|scope cancelled| En[Stop] +``` + +## Architecture + +- The driver runs a single RPC handler fiber. No autonomous emission; all activity is request-driven. +- The 1-second sleep for utilisation sampling uses a cancellable op so the fiber exits cleanly when the scope is cancelled. +- If `/sys/devices/system/cpu/cpuN/cpufreq/scaling_cur_freq` is not readable for a given core, that core's frequency is omitted from the result without failing the whole call. +- A `finally` block logs the reason for shutdown. +- The driver does not republish meta on the report period — meta is static after startup. diff --git a/docs/specs/hal/memory_driver.md b/docs/specs/hal/memory_driver.md new file mode 100644 index 00000000..e8f8fa19 --- /dev/null +++ b/docs/specs/hal/memory_driver.md @@ -0,0 +1,111 @@ +# Memory Driver (HAL) + +## Description + +The memory driver is a HAL component that exposes RAM metrics to the rest of the system via a single `memory` capability. It sources data from `/proc/meminfo`, computing total, used, free, and utilisation fields. + +All metrics are available via a single `get` RPC offering that accepts a `field` name and a `max_age` parameter. The driver maintains a shared cache of the last full set of computed readings. Any `get` call within `max_age` seconds of the last read is served from cache, avoiding a repeated file read. + +The memory driver is created and owned by the Sysmon Manager. It does not manage its own lifecycle. + +## Dependencies + +None. The driver reads directly from procfs. + +## Initialisation + +On creation by the Sysmon Manager: + +1. Publish capability `meta`. +2. Start the RPC handler fiber. + +The readings cache is empty at startup. The first `get` call always triggers a fresh read. + +## Capability + +Class: `memory` +Id: `'1'` + +### Meta (retained) + +Topic: `{'cap', 'memory', '1', 'meta'}` + +```lua +{ + provider = 'hal', + version = 1, +} +``` + +### Offerings + +#### get + +Topic: `{'cap', 'memory', '1', 'rpc', 'get'}` + +Input (`MemGetOpts`): + +```lua +{ + field = , -- required: one of the field names listed below + max_age = , -- required: maximum acceptable age of reading in seconds +} +``` + +Available fields: + +| Field | Type | Description | +|---------|--------|-----------------------------------------------------------------------------| +| `total` | number | Total RAM in kB | +| `used` | number | Used RAM in kB (total minus free, buffers, and cached) | +| `free` | number | Usable free RAM in kB (MemFree + Buffers + Cached) | +| `util` | number | RAM utilisation as a percentage (0–100), computed as `used / total * 100` | + +Reply on success: + +```lua +{ ok = true, reason = } +``` + +Reply on failure: + +```lua +{ ok = false, reason = } +``` + +If the requested `field` string is not one of the four above, the driver replies with `ok = false`. + +## Cache Behaviour + +The driver uses `shared/cache.lua` (`cache.new()`), storing each field under its field name as the key. The `max_age` value from the RPC request is passed as the timeout to `cache:get(field, max_age)`. + +On any `get` call: + +1. Call `cache:get(field, max_age)`. If a non-nil value is returned, reply with it immediately. +2. Otherwise, read `/proc/meminfo`, compute all four fields, then `cache:set` each: + - `cache:set('total', value)`, `cache:set('used', value)`, `cache:set('free', value)`, `cache:set('util', value)`. + +All four fields are derived from the same single `/proc/meminfo` read and are always updated together on a cache miss, so a subsequent call for a different field within `max_age` will always hit its cache entry. + +## Service Flow + +```mermaid +flowchart TD + St[Start] --> A(Publish meta) + A --> B{Wait for get RPC or scope cancelled} + B -->|get RPC| C{Cache fresh enough? now - read_at <= max_age} + C -->|yes| D(Return cached field value) + C -->|no| E(Read /proc/meminfo) + E --> F(Compute total, used, free, util) + F --> G(Update cache with read_at = now) + G --> D + D --> B + B -->|scope cancelled| En[Stop] +``` + +## Architecture + +- The driver runs a single RPC handler fiber. No autonomous emission; all activity is request-driven. +- A `finally` block logs the reason for shutdown. +- `free` is computed as `MemFree + Buffers + Cached` from `/proc/meminfo`, matching existing system behaviour. +- If `/proc/meminfo` is unreadable, the driver replies with `ok = false` and does not update the cache. diff --git a/docs/specs/hal/op-only-drivers.md b/docs/specs/hal/op-only-drivers.md new file mode 100644 index 00000000..479533ca --- /dev/null +++ b/docs/specs/hal/op-only-drivers.md @@ -0,0 +1,101 @@ +# Op-only driver guide + +## Purpose + +Op-only drivers are edge components that expose backend or provider behaviour as Fibers-native Ops. + +They are used by HAL and transport services to keep backend-specific polling, callbacks and command loops out of ordinary service code. + +An Op-only driver should make this true: + +```text +service code sees Ops and terminate(reason) +driver code owns backend mechanics +``` + +## Boundary shape + +```mermaid +flowchart LR + Service["service or manager"] --> OpAPI["driver Op API"] + OpAPI --> Driver["driver command loop"] + Driver --> Backend["OS / cqueues / lua-http / file / provider"] + Backend --> Driver + Driver --> OpAPI + OpAPI --> Service +``` + +The service should not know about backend polling, raw callbacks or backend object lifetimes. + +## Driver API rules + +Prefer APIs like: + +```lua +driver:start_op(...) +driver:apply_config_op(...) +driver:shutdown_op(...) +driver:fault_op() +handle:read_op(...) +handle:write_op(...) +handle:terminate(reason) +``` + +Avoid APIs that hide blocking work behind ordinary functions. + +Synchronous helpers are acceptable only when they are pure validation, pure projection or immediate state inspection. + +## Termination and graceful closure + +Drivers should distinguish: + +```text +terminate(reason) immediate, idempotent, finaliser-safe +close_op() graceful protocol work, may wait +``` + +Every object handed to a service must have an immediate termination path if it is owned by a scope finaliser. + +## Losing choices + +Every driver Op must define what happens when it loses a choice. + +Expected patterns: + +```text +queued command loses: remove or mark abandoned +active backend job loses: abort/terminate active job +owned backend handle loses: terminate handle +borrowed backend handle loses: leave owner intact +callback registration loses: unregister callback +``` + +A losing Op must not leave a callback, file descriptor, driver command or backend job silently live. + +## Publication rules + +Op-only drivers normally do not publish canonical `state/...` directly. + +Instead: + +```text +driver/provider emits raw facts or capability events to its owning service +HAL publishes raw/... for provider-native truth +Device/Update/Fabric/UI publish canonical state where they own it +HTTP publishes narrow interface stats and obs metrics where it owns the interface +``` + +Drivers should not invent public `cap/...` or `state/...` families unless the driver itself is deliberately the service boundary. + +## Reviewer checklist + +```text +All blocking driver operations are Ops. +No service-facing method hides a wait. +No driver handle lacks terminate(reason). +close_op is not required for finaliser cleanup. +Losing Ops clean up queued and active backend work. +Callbacks do not call service reducers directly. +Backend objects are not returned as public handles. +Raw/provider facts are emitted to the owning service, not published as canonical state by accident. +``` diff --git a/docs/specs/hal/platform_driver.md b/docs/specs/hal/platform_driver.md new file mode 100644 index 00000000..c1b60b7c --- /dev/null +++ b/docs/specs/hal/platform_driver.md @@ -0,0 +1,132 @@ +# Platform Driver (HAL) + +## Description + +The platform driver is a HAL component that exposes hardware and firmware identity information for the host device. It reads static identity fields once at startup and publishes them as retained capability state so any consumer can get them immediately on subscription without making an RPC call. The only dynamic field — system uptime — is available via a `get` RPC with a cache. + +A trivial **Platform Manager** owns this driver. It has no discovery logic: it instantiates a single `PlatformDriver`, emits one HAL device-added event, and waits for its scope to be cancelled. The manager behaviour is not complex enough to warrant a separate spec file — it is described inline here. + +## Dependencies + +- **`fw_printenv`** command: required to read `board_revision`. If the command is unavailable or fails, `board_revision` is published as `nil` without failing startup. + +## Initialisation + +On startup: + +1. Read static identity fields from the filesystem: + - `hw_revision` from `/etc/hwrevision` + - `fw_version` from `/etc/fwversion` + - `board_revision` from `fw_printenv` output (extract `board_revision=` field) + - `serial` from `/data/serial` +2. Publish capability `meta`. +3. Publish capability `state` with the static identity fields (retained). Fields that could not be read are published as `nil`. +4. Start the RPC handler fiber for `get` (uptime). + +## Capability + +Class: `platform` +Id: `'1'` + +### Meta (retained) + +Topic: `{'cap', 'platform', '1', 'meta'}` + +```lua +{ + provider = 'hal', + version = 1, +} +``` + +### State (retained) + +Topic: `{'cap', 'platform', '1', 'state'}` + +Published once at startup and never updated thereafter (these fields are static for the life of the process). Consumers subscribe with retention to get the current values immediately. + +```lua +{ + hw_revision = , -- e.g. 'bigbox-v1 1.0', nil if unreadable + fw_version = , -- e.g. '1.2.3', nil if unreadable + board_revision = , -- e.g. 'rev2', nil if unreadable or fw_printenv unavailable + serial = , -- device serial number, nil if unreadable +} +``` + +Whitespace is trimmed from all string values. + +### Offerings + +#### get + +Topic: `{'cap', 'platform', '1', 'rpc', 'get'}` + +Returns the current system uptime. This is the only dynamic field and the only one that requires an RPC. + +Input (`PlatformGetOpts`): + +```lua +{ + field = , -- required: field to retrieve; currently only 'uptime' is supported + max_age = , -- required: maximum acceptable age of reading in seconds +} +``` + +Available fields: + +| Field | Type | Description | +|----------|--------|-----------------------------------------------| +| `uptime` | number | System uptime in seconds (float, from `/proc/uptime`) | + +Reply on success: + +```lua +{ ok = true, reason = } +``` + +Reply on failure: + +```lua +{ ok = false, reason = } +``` + +If the requested `field` string is not a recognised field, the driver replies with `ok = false`. + +## Cache Behaviour + +The driver uses `shared/cache.lua` (`cache.new()`), storing each field under its field name as the key. The `max_age` value from the RPC request is passed as the timeout to `cache:get(field, max_age)`. + +On any `get` call: + +1. Call `cache:get(field, max_age)`. If a non-nil value is returned, reply with it immediately. +2. Otherwise, perform the relevant read, call `cache:set(field, value)`, then reply with the value. + - `uptime`: read `/proc/uptime`, parse the first number, `cache:set('uptime', value)`. + +## Service Flow + +```mermaid +flowchart TD + St[Start] --> A(Read hw_revision, fw_version, board_revision, serial) + A --> B(Publish meta) + B --> C(Publish state with identity fields) + C --> D{Wait for get RPC or scope cancelled} + D -->|get RPC| E{Cache fresh enough? now - read_at <= max_age} + E -->|yes| F(Return cached uptime) + E -->|no| G(Read /proc/uptime) + G --> H{Read ok?} + H -->|no| I(Reply ok=false with error) + H -->|yes| J(Update cache, return uptime) + J --> D + I --> D + F --> D + D -->|scope cancelled| En[Stop] +``` + +## Architecture + +- The driver runs a single RPC handler fiber for the `get` (uptime) offering. +- Static identity fields are published as retained state at startup — no timer, no re-publication. +- Read failures for static fields at startup are logged as warnings; `nil` is published for the unreadable field rather than aborting startup. +- A `finally` block logs the reason for shutdown. +- The `hw_revision` string may contain a model prefix followed by a version (e.g. `'bigbox-v1 1.0'`). Parsing the model prefix out of `hw_revision` is the responsibility of consumers (e.g. the system service), not this driver. diff --git a/docs/specs/hal/power_driver.md b/docs/specs/hal/power_driver.md new file mode 100644 index 00000000..4a32c1ab --- /dev/null +++ b/docs/specs/hal/power_driver.md @@ -0,0 +1,101 @@ +# Power Driver (HAL) + +## Description + +The power driver is a HAL component that exposes system power control to the rest of the system. It provides two offerings — `shutdown` and `reboot` — that execute the corresponding OS commands. The driver's sole responsibility is to run the command; all orchestration (notifying services, waiting for graceful shutdown) is the responsibility of the caller (the System Service). + +A trivial **Power Manager** owns this driver. It has no discovery logic: it instantiates a single `PowerDriver`, emits one HAL device-added event, and waits for its scope to be cancelled. The manager behaviour is not complex enough to warrant a separate spec file — it is described inline here. + +## Dependencies + +- OS commands `shutdown` and `reboot` must be available on the system `PATH`. + +## Initialisation + +On startup: + +1. Publish capability `meta`. +2. Start the RPC handler fiber. + +## Capability + +Class: `power` +Id: `'1'` + +### Meta (retained) + +Topic: `{'cap', 'power', '1', 'meta'}` + +```lua +{ + provider = 'hal', + version = 1, +} +``` + +### Offerings + +Both offerings accept a `PowerActionOpts` input and send an RPC reply **before** executing the OS command. This is intentional: the command will terminate the process, so the reply must be sent first on a best-effort basis. + +#### shutdown + +Topic: `{'cap', 'power', '1', 'rpc', 'shutdown'}` + +Executes `shutdown -h now`. + +Input (`PowerActionOpts`): + +```lua +{ + reason = , -- optional: human-readable reason, logged only +} +``` + +Reply (sent before command runs): + +```lua +{ ok = true, reason = nil } +``` + +#### reboot + +Topic: `{'cap', 'power', '1', 'rpc', 'reboot'}` + +Executes `reboot`. + +Input (`PowerActionOpts`): + +```lua +{ + reason = , -- optional: human-readable reason, logged only +} +``` + +Reply (sent before command runs): + +```lua +{ ok = true, reason = nil } +``` + +## Service Flow + +```mermaid +flowchart TD + St[Start] --> A(Publish meta) + A --> B{Wait for shutdown RPC, reboot RPC, or scope cancelled} + B -->|shutdown RPC| C(Log reason if provided) + C --> D(Send ok=true reply) + D --> E(exec: shutdown -h now) + B -->|reboot RPC| F(Log reason if provided) + F --> G(Send ok=true reply) + G --> H(exec: reboot) + B -->|scope cancelled| En[Stop] +``` + +## Architecture + +- The driver runs a single RPC handler fiber with `op.choice` over both offerings; the fiber exits when its scope is cancelled. +- The RPC reply is sent before the OS command is executed. There is no guarantee the caller receives the reply before the system goes down, but it is best-effort. +- The `reason` field is logged at info level before execution. It is not transmitted or stored beyond the log. +- A `finally` block logs the reason for shutdown of the driver fiber itself (distinct from the system shutdown action). +- The driver does not validate whether the caller has performed any graceful shutdown orchestration. That is entirely the caller's responsibility. diff --git a/docs/specs/hal/radio_driver.md b/docs/specs/hal/radio_driver.md new file mode 100644 index 00000000..919c99d8 --- /dev/null +++ b/docs/specs/hal/radio_driver.md @@ -0,0 +1,370 @@ +# Radio Driver (HAL) + +## Description + +The Radio Driver is a HAL component that exposes wireless radio configuration and interface statistics to the rest of the system via a single `radio` capability. It: + +1. **Holds staged configuration** — all `set_*` calls accumulate changes in-memory. Changes are not applied until `apply()` is called. +2. **Applies staged config on demand** — `apply()` delegates the staged config table to the active backend, which is responsible for writing it to the underlying OS config system (e.g. UCI) and triggering a reload. +3. **Emits interface stats** — a stats loop calls backend methods to detect client connect/disconnect events and collect per-interface counters, emitting them via the capability `emit_ch` for HAL to publish on the bus. + +The driver never touches UCI or any OS config system directly. All config writes are delegated to the active backend via `backend:apply(staged)`. A future non-OpenWrt backend can implement the same backend interface without any UCI involvement. + +## Dependencies + +- `backends/radio/provider.lua` — selects the active backend at driver creation. + +## Initialisation + +On creation by the WLAN Manager: + +1. Resolve the radio backend via `provider.get_backend()`. If no supported backend is found, creation fails with an error; the manager logs the error and does not emit a device-added event. +2. Fetch initial radio metadata via `backend:get_meta(name)` to populate the capability meta (path, type). +3. Initialise the staged config table (empty). +4. Publish capability `meta`. +5. Start the RPC handler fiber. +6. Start the stats loop fiber. + +## Capability + +Class: `radio` +Id: UCI radio section name (e.g. `'radio0'`, `'radio1'`) + +### Meta (retained) + +Topic: `{'cap', 'radio', , 'meta'}` + +```lua +{ + provider = 'hal', + version = 1, + name = , -- UCI radio section name + path = , -- PCI/AHB device path, e.g. "platform/ahb/18100000.wmac" + type = , -- radio chipset type, e.g. "mac80211" +} +``` + +### Offerings + +All offerings return `{ ok = true }` on success or `{ ok = false, reason = }` on validation failure or internal error. + +--- + +#### set_channels + +Topic: `{'cap', 'radio', , 'rpc', 'set_channels'}` + +Stages the radio frequency parameters in the driver's staged config. No UCI write occurs until `apply` is called. + +Input: + +```lua +{ + band = , -- required: "2g" or "5g" + channel = , -- required: channel number or "auto" + htmode = , -- required: one of "HE20","HE40+","HE40-","HE80","HE160","HT20","HT40+","HT40-","VHT20","VHT40+","VHT40-","VHT80","VHT160" + channels = , -- required when channel == "auto": list of numbers or strings +} +``` + +Validation: +- `band` must be `"2g"` or `"5g"`. +- `htmode` must be one of the listed valid values. +- When `channel == "auto"`, `channels` must be a non-empty list of numbers or strings. +- When `channel ~= "auto"`, `channel` must be a number or string. + +--- + +#### set_txpower + +Topic: `{'cap', 'radio', , 'rpc', 'set_txpower'}` + +Input: + +```lua +{ + txpower = , -- required +} +``` + +Validation: `txpower` must be a number or a string. + +--- + +#### set_country + +Topic: `{'cap', 'radio', , 'rpc', 'set_country'}` + +Input: + +```lua +{ + country = , -- required: 2-letter ISO country code (normalised to uppercase) +} +``` + +Validation: must be a 2-character string. + +--- + +#### set_enabled + +Topic: `{'cap', 'radio', , 'rpc', 'set_enabled'}` + +Input: + +```lua +{ + enabled = , -- required +} +``` + +Stages the enabled/disabled state for the radio. Validation: must be boolean. + +--- + +#### add_interface + +Topic: `{'cap', 'radio', , 'rpc', 'add_interface'}` + +Stages a new wireless interface entry for this radio. The interface name is generated as `_i` where N is an auto-incrementing counter. + +Input: + +```lua +{ + ssid = , -- required: SSID string, non-empty + encryption = , -- required: one of "none","wep","psk","psk2","psk-mixed","sae","sae-mixed","owe","wpa","wpa2","wpa3" + password = , -- required: may be empty string for open networks + network = , -- required: network interface name, non-empty + mode = , -- required: one of "ap","sta","adhoc","mesh","monitor" + enable_steering = , -- required: if true, enables 802.11k/v BSS transition flags +} +``` + +Reply on success: + +```lua +{ ok = true, result = } -- result is the generated interface name e.g. "radio0_i0" +``` + +When `enable_steering = true`, the staged interface entry includes BSS transition and 802.11k flags. The backend is responsible for mapping these to the appropriate config system keys (e.g. `bss_transition`, `ieee80211k`, `rrm_neighbor_report`, `rrm_beacon_report` in UCI). + +--- + +#### delete_interface + +Topic: `{'cap', 'radio', , 'rpc', 'delete_interface'}` + +Marks an interface entry for removal in the staged config. The removal is applied by the backend on the next `apply` call. + +Input: + +```lua +{ + interface = , -- required: interface name to delete, non-empty +} +``` + +--- + +#### clear_radio_config + +Topic: `{'cap', 'radio', , 'rpc', 'clear_radio_config'}` + +Resets the in-memory staged config to the base state: only the radio's fixed identity fields (`name`, `path`, `type`). All previously staged option changes and all staged interface entries are discarded. No backend call is made — this is a purely in-memory operation. + +Input: none (empty object `{}`). + +--- + +#### set_report_period + +Topic: `{'cap', 'radio', , 'rpc', 'set_report_period'}` + +Sets the metrics loop publish interval. Takes effect on the next sleep in the metrics loop. + +Input: + +```lua +{ + period = , -- required: interval in seconds, must be > 0 +} +``` + +--- + +#### apply + +Topic: `{'cap', 'radio', , 'rpc', 'apply'}` + +Passes the full staged config table to `backend:apply(staged)`. The backend is responsible for writing to UCI (or equivalent) and triggering a reload. The backend coalesces rapid successive reloads with a 1-second debounce window. + +Input: none (empty object `{}`). + +On success, the per-interface counter is reset so future `add_interface` calls start from `_i0` again. + +--- + +#### rollback + +Topic: `{'cap', 'radio', , 'rpc', 'rollback'}` + +Discards all staged changes. The staged config is reset to the same state as after `clear_radio_config` was last called. No backend call is made. + +Input: none (empty object `{}`). + +--- + +## Stats Loop + +The stats loop runs in a separate fiber within the driver's scope. It calls backend methods to monitor client connect/disconnect events and to collect per-interface counters on each report period tick. All results are emitted via `emit_ch` as `Emit` values — HAL reads them and publishes them on the bus under `cap/radio//state/` and `cap/radio//event/`. The wifi service subscribes to these topics to perform session management, MAC hashing, and observability publishing. The driver is not aware of any of that logic. + +### Client events + +The backend monitors the wireless interface for `AP-STA-CONNECTED` and `AP-STA-DISCONNECTED` events (using `iw event` on OpenWrt). For each event the driver emits a raw `client_event` with the MAC address and connection state. The wifi service handles session ID generation, MAC address hashing, and deduplication. + +### Emitted stats + +Stats are refreshed on each `report_period` tick. All values are emitted via `emit_ch` for HAL to publish. + +| Emit key | Content | Source | +|----------------------|----------------------------------------------------|---------------------------------------------| +| `client_event` | `{ mac, connected, interface, timestamp }` | backend event monitor (e.g. `iw event`) | +| `num_sta` | Total connected station count (number) | Derived from connect/disconnect tracking | +| `iface_num_sta` | `{ interface, band, index, count }` | Per-interface station count | +| `iface_txpower` | `{ interface, value }` | backend query (e.g. `iw dev info`) | +| `iface_channel` | `{ interface, channel, freq, width }` | backend query (e.g. `iw dev info`) | +| `iface_noise` | `{ interface, value }` | backend query (e.g. `iw survey dump`) | +| `iface_rx_bytes` | `{ interface, value }` | `/sys/class/net//statistics/` | +| `iface_tx_bytes` | `{ interface, value }` | `/sys/class/net//statistics/` | +| `iface_rx_packets` | `{ interface, value }` | `/sys/class/net//statistics/` | +| `iface_tx_packets` | `{ interface, value }` | `/sys/class/net//statistics/` | +| `iface_rx_dropped` | `{ interface, value }` | `/sys/class/net//statistics/` | +| `iface_tx_dropped` | `{ interface, value }` | `/sys/class/net//statistics/` | +| `iface_rx_errors` | `{ interface, value }` | `/sys/class/net//statistics/` | +| `iface_tx_errors` | `{ interface, value }` | `/sys/class/net//statistics/` | +| `client_signal` | `{ mac, interface, signal }` | backend query (e.g. `iw dev station get `) | +| `client_tx_bytes` | `{ mac, interface, value }` | backend query (e.g. `iw dev station get `) | +| `client_rx_bytes` | `{ mac, interface, value }` | backend query (e.g. `iw dev station get `) | + +### Interface assignment + +Each radio has a **fixed interface name** assigned from config (no hotplug binding). Interface names are derived from the `add_interface` sequence (e.g. `radio0_i0`, `radio0_i1`). The mapping from SSID name to interface index is built when `add_interface` is called and held in-memory by the stats loop fiber. + +## Types + +All capability arg and reply types follow the conventions in `src/services/hal/types/`. + +### `RadioSetChannelsOpts` + +```lua +---@class RadioSetChannelsOpts +---@field band string -- "2g" or "5g" +---@field channel number|string -- channel number or "auto" +---@field htmode string +---@field channels table|nil -- required when channel=="auto" +``` + +### `RadioSetTxpowerOpts` + +```lua +---@class RadioSetTxpowerOpts +---@field txpower number|string +``` + +### `RadioSetCountryOpts` + +```lua +---@class RadioSetCountryOpts +---@field country string -- 2-letter ISO code +``` + +### `RadioSetEnabledOpts` + +```lua +---@class RadioSetEnabledOpts +---@field enabled boolean +``` + +### `RadioAddInterfaceOpts` + +```lua +---@class RadioAddInterfaceOpts +---@field ssid string +---@field encryption string +---@field password string +---@field network string +---@field mode string +---@field enable_steering boolean +``` + +### `RadioDeleteInterfaceOpts` + +```lua +---@class RadioDeleteInterfaceOpts +---@field interface string +``` + +### `RadioSetReportPeriodOpts` + +```lua +---@class RadioSetReportPeriodOpts +---@field period number -- seconds, > 0 +``` + +### `RadioCapabilityReply` + +```lua +---@class RadioCapabilityReply +---@field ok boolean +---@field reason string|nil -- present on failure; on add_interface success, holds the generated interface name +``` + +## Backend Contract + +The OpenWrt radio backend (`backends/radio/providers/openwrt/impl.lua`) must implement: + +| Function | Description | +|--------------------|--------------------------------------------------------------------------------------------------------------------| +| `is_supported()` | Returns `true` if running on OpenWrt (checks `/etc/openwrt_release`) | +| `get_meta(name)` | Returns `{ path, type }` for the radio. No config system cursor exposed to the driver. | +| `apply(staged)` | Writes the staged config table to the OS config system (UCI on OpenWrt) and triggers a reload. The 1-second debounce is an internal backend concern. | +| `watch_events(interfaces, cb)` | Opens an event monitor (e.g. `iw event`) and calls `cb(event)` for each client connect/disconnect event. The driver calls this to feed the stats loop. | +| `get_iface_info(iface)` | Returns txpower, channel, frequency, and channel width for a given interface (e.g. via `iw dev info`). | +| `get_iface_survey(iface)` | Returns noise floor for a given interface (e.g. via `iw survey dump`). | +| `get_station_info(iface, mac)` | Returns per-client signal, tx_bytes, rx_bytes (e.g. via `iw station get`). | + +The sysfs counters (`/sys/class/net//statistics/`) are read directly by the driver, not the backend, as they are a standard Linux interface not specific to any wireless stack. + +The driver passes the staged config table opaquely to `apply`. The backend is the only component that knows how to interpret and write it. + +## Service Flow + +### RPC handler fiber + +1. Resolve backend; publish meta. +2. Enter `named_choice` loop on `control_ch` and scope cancellation. +3. On each RPC: validate inputs; update staged config table; reply `{ ok = true }` or `{ ok = false, reason = ... }`. +4. On `apply`: pass staged config table to `backend:apply(staged)`; reset interface counter. +5. On `rollback`: discard staged config table. +6. On scope cancelled: exit loop. + +### Stats loop fiber + +1. Call `backend:watch_events(interfaces, cb)` to start monitoring client connect/disconnect events. +2. Enter `named_choice` loop over: client event callback, report-period tick, new report period, scope cancelled. +3. On client event: emit `client_event` with raw MAC and state; update in-memory station count; emit `num_sta` and `iface_num_sta`. +4. On report-period tick: call `backend:get_iface_info`, `backend:get_iface_survey`, `backend:get_station_info` for each known interface; read sysfs statistics; emit all `iface_*` and `client_*` stats. +5. On new report period: update sleep timer. +6. On scope cancelled: exit; signal backend to stop event monitoring. + +## Architecture + +- The driver runs two concurrent fibers: the RPC handler and the stats loop. Both are children of the driver's scope. +- Staged config is a plain Lua table local to the driver; it is only ever accessed by the RPC handler fiber (no concurrent access). +- `apply` passes the staged config table to `backend:apply(staged)`. The driver has no knowledge of how the backend writes the config. +- Stats are emitted via `emit_ch` as `Emit` objects. HAL reads `emit_ch` and publishes them to the bus under `cap/radio//state/` or `cap/radio//event/`. The driver never publishes to the bus directly. +- All OS-specific tool calls (`iw event`, `iw dev info`, `iw survey dump`, `iw station get`) are encapsulated in the backend. The driver calls backend methods; the backend handles subprocess management, output parsing, and error recovery. +- If the event monitor subprocess exits unexpectedly, the backend is responsible for restarting it. The driver is notified via the callback mechanism and may log a warning. +- A `finally` block on the driver scope logs the reason for shutdown. diff --git a/docs/specs/hal/sysmon_manager.md b/docs/specs/hal/sysmon_manager.md new file mode 100644 index 00000000..f6faa75c --- /dev/null +++ b/docs/specs/hal/sysmon_manager.md @@ -0,0 +1,66 @@ +# Sysmon Manager (HAL) + +## Description + +The Sysmon Manager is a HAL manager that owns the lifecycle of three system monitoring drivers: CPU, Memory, and Thermal. It is responsible for discovering thermal zones at startup, instantiating all drivers, and registering the corresponding capabilities with the HAL service via device events. + +The Sysmon Manager owns no capabilities directly — all capabilities are owned by the drivers it creates. + +## Dependencies + +- Sysfs path `/sys/class/thermal/` must be readable to discover thermal zones. If it is absent or unreadable, no thermal capabilities are registered and the manager logs a warning. + +## Initialisation + +On startup: + +1. Scan `/sys/class/thermal/` for directories matching `thermal_zone*`. +2. For each discovered zone, create a `ThermalDriver` instance and emit a HAL device event (`added`). +3. Create one `CpuDriver` instance and emit a HAL device event (`added`). +4. Create one `MemoryDriver` instance and emit a HAL device event (`added`). + +Each device event triggers the HAL service to register the corresponding capability and start its RPC handler fiber. + +If no thermal zones are found, log a warning and continue — CPU and memory capabilities are still registered. + +## Managed Drivers + +| Driver | Class | Id | Quantity | +|----------------|-----------|----------------------------|------------------| +| `CpuDriver` | `cpu` | `'1'` | 1 | +| `MemoryDriver` | `memory` | `'1'` | 1 | +| `ThermalDriver`| `thermal` | `'zone0'`, `'zone1'`, ... | 1 per sysfs zone | + +Thermal zone IDs are derived from the sysfs directory name by stripping the `thermal_` prefix (e.g. `thermal_zone0` → `'zone0'`). + +## Manager Interface + +The Sysmon Manager implements the standard `Manager` interface: + +- `start(scope)` — performs discovery, instantiates drivers, emits device events +- `stop(reason)` — cancels the manager scope; all child driver fibers are stopped via structured concurrency +- `apply_config(config)` — no-op for this manager; system monitoring has no configurable parameters + +## Service Flow + +```mermaid +flowchart TD + St[Start] --> A(Scan /sys/class/thermal/ for thermal_zone* dirs) + A --> B{Zones found?} + B -->|no| C(Log warning: no thermal zones) + B -->|yes| D(For each zone: create ThermalDriver, emit device added event) + C --> E(Create CpuDriver, emit device added event) + D --> E + E --> F(Create MemoryDriver, emit device added event) + F --> G{Wait for scope cancelled} + G -->|scope cancelled| H(Stop all driver scopes via structured concurrency) + H --> En[Stop] +``` + +## Architecture + +- The manager holds references to all driver instances it created so it can stop them cleanly. +- All drivers are started within a child scope of the manager scope. If the manager scope is cancelled, all drivers stop via structured concurrency — no explicit teardown loop is needed. +- The manager does not re-discover zones after startup. If a sysfs zone appears or disappears at runtime, it is not reflected. This is acceptable for the current use case (zones are fixed at boot). +- Thermal zone discovery failure (unreadable sysfs) is a warning, not a fatal error. +- A `finally` block logs the reason for manager shutdown. diff --git a/docs/specs/hal/thermal_driver.md b/docs/specs/hal/thermal_driver.md new file mode 100644 index 00000000..eef13c03 --- /dev/null +++ b/docs/specs/hal/thermal_driver.md @@ -0,0 +1,110 @@ +# Thermal Driver (HAL) + +## Description + +The thermal driver is a HAL component that exposes temperature readings for a single thermal zone to the rest of the system. Temperature is sourced from the Linux sysfs thermal subsystem at `/sys/class/thermal/thermal_zone{N}/temp`, which reports values in milli-degrees Celsius. + +One thermal driver instance is created per discovered zone. The Sysmon Manager is responsible for discovering all zones at startup and creating the corresponding drivers. + +Each driver exposes a single `get` RPC offering that accepts a `max_age` parameter. The driver caches the last reading, returning cached data for any call within `max_age` seconds. + +## Dependencies + +None. The driver reads directly from sysfs. + +## Initialisation + +On creation by the Sysmon Manager: + +1. Attempt to read the zone type from `/sys/class/thermal/thermal_zone{N}/type` (optional; used for meta only). +2. Publish capability `meta` for this zone. +3. Start the RPC handler fiber. + +The readings cache is empty at startup. The first `get` call always triggers a fresh read. + +## Capability + +One capability per discovered thermal zone. + +Class: `thermal` +Id: zone identifier string, e.g. `'zone0'`, `'zone1'` (derived from the sysfs directory name) + +### Meta (retained) + +Topic: `{'cap', 'thermal', , 'meta'}` + +```lua +{ + provider = 'hal', + version = 1, + zone = , -- zone id, e.g. 'zone0' + path = , -- sysfs path to the temp file, e.g. '/sys/class/thermal/thermal_zone0/temp' + type = , -- zone type string if readable (e.g. 'cpu-thermal'), nil otherwise +} +``` + +### Offerings + +#### get + +Topic: `{'cap', 'thermal', , 'rpc', 'get'}` + +Input (`ThermalGetOpts`): + +```lua +{ + max_age = , -- required: maximum acceptable age of reading in seconds +} +``` + +Reply on success: + +```lua +{ ok = true, reason = } -- temperature in degrees Celsius (float) +``` + +Reply on failure: + +```lua +{ ok = false, reason = } +``` + +Temperature is converted from milli-degrees Celsius by dividing by 1000. + +## Cache Behaviour + +The driver uses `shared/cache.lua` (`cache.new()`), storing the temperature under the key `'temp'`. The `max_age` value from the RPC request is passed as the timeout to `cache:get('temp', max_age)`. + +On any `get` call: + +1. Call `cache:get('temp', max_age)`. If a non-nil value is returned, reply with it immediately. +2. Otherwise, read the sysfs temp file, convert to Celsius, call `cache:set('temp', value)`, then reply with the value. + +If the sysfs read fails, the driver replies with `ok = false` and does not call `cache:set` (leaving the previous cached value intact for future requests). + +## Service Flow + +```mermaid +flowchart TD + St[Start] --> A(Read zone type from sysfs if available) + A --> B(Publish meta) + B --> C{Wait for get RPC or scope cancelled} + C -->|get RPC| D{Cache fresh enough? now - read_at <= max_age} + D -->|yes| E(Return cached Celsius value) + D -->|no| F(Read sysfs temp file) + F --> G{Read ok?} + G -->|no| H(Reply ok=false with error) + G -->|yes| I(Convert to Celsius, update cache) + I --> E + E --> C + H --> C + C -->|scope cancelled| En[Stop] +``` + +## Architecture + +- One driver instance per thermal zone. Each instance runs its own RPC handler fiber. +- No autonomous emission; all activity is request-driven. +- A `finally` block logs the reason for shutdown. +- Zone IDs are derived directly from the sysfs directory name by the Sysmon Manager (e.g. `thermal_zone0` → id `'zone0'`). +- The sysfs `type` file is read once at startup for meta only; it is not re-read during operation. diff --git a/docs/specs/hal/time_driver.md b/docs/specs/hal/time_driver.md new file mode 100644 index 00000000..2c1410de --- /dev/null +++ b/docs/specs/hal/time_driver.md @@ -0,0 +1,108 @@ +# Time Driver (HAL) + +## Description + +The time driver is a HAL component that manages the system NTP daemon (`sysntpd`) and exposes a time source capability to the rest of the system. It is responsible for all direct OS interaction related to time synchronisation: restarting the daemon, monitoring kernel hotplug NTP events via ubus, and publishing the current sync state and accuracy metadata as a capability on the bus. + +The time driver acts as both a driver and a time manager — it owns the full lifecycle of `sysntpd` within the running session. + +## Dependencies + +- **ubus capability** (`{'cap', 'ubus', '1', ...}`): required to listen for `hotplug.ntp` kernel events. The driver waits for the ubus capability to become available before starting NTP. + +## Initialisation + +On startup (once the ubus capability is available): + +1. Restart `sysntpd` via `exec.command("/etc/init.d/sysntpd", "restart"):run()`. +2. Register a ubus listener for `hotplug.ntp` events. +3. Publish initial capability `meta` and `state` (initially `synced = false`). + +## Capability + +The driver publishes a single time source capability. The capability id is a UUID generated at startup. In the future, additional time source capabilities may exist alongside this one. + +### Meta (retained) + +Topic: `{'cap', 'time', , 'meta'}` + +```lua +{ + provider = 'hal', + source = 'ntp', -- time source type + version = 1, -- interface version + accuracy_seconds = , -- estimated absolute error in seconds +} +``` + +`accuracy_seconds` is a coarse estimate of absolute clock error and is derived from NTP stratum. Lower values are better. It is `nil` when unsynced (stratum >= 16) or before first sync-quality data is available. + +### State (retained) + +Topic: `{'cap', 'time', , 'state'}` + +```lua +{ + synced = true | false, + stratum = , -- last reported stratum, nil before first event + accuracy_seconds = , +} +``` + +Published on every sync/unsync transition. Retained so new subscribers immediately get the current state. + +### Events (non-retained) + +#### synced + +Topic: `{'cap', 'time', , 'event', 'synced'}` + +Fired when the NTP daemon transitions from unsynced to synced (stratum < 16). Payload: + +```lua +{ stratum = , accuracy_seconds = } +``` + +#### unsynced + +Topic: `{'cap', 'time', , 'event', 'unsynced'}` + +Fired when the NTP daemon transitions from synced to unsynced (stratum == 16 or daemon restarts). Payload: + +```lua +{ stratum = 16, accuracy_seconds = nil } +``` + +These events are non-retained — consumers that need current state should read `{'cap', 'time', '1', 'state'}` first, then subscribe to events for transitions. + +## Service Flow + +```mermaid +flowchart TD + St[Start] --> A(Install alarm handler) + A --> B(Wait for ubus capability) + B -->|ubus cap available| C(Restart sysntpd) + C --> D{sysntpd restart ok?} + D -->|error| E[Log error and stop] + D -->|ok| F(Register ubus listener for hotplug.ntp) + F --> G{Listener registered ok?} + G -->|error| H[Log error and stop] + G -->|ok| I(Publish meta + initial state: synced=false) + I --> J{Wait for hotplug.ntp event or stream closed or context done} + J -->|hotplug.ntp event| K{stratum < 16?} + K -->|yes, was unsynced| L(Publish state synced=true + event/synced) + K -->|no, was synced| M(Publish state synced=false + event/unsynced) + L --> J + M --> J + K -->|no change| J + J -->|stream closed| N[Log warning, stop] + J -->|context done| O(Send stop_stream to ubus) +``` + +## Architecture + +- The driver runs a single main fiber that handles the full lifecycle. No child scope is needed. +- Sync and unsync events are only published on **transitions** — the driver tracks the previous sync state and only emits an event when it changes. +- The retained `state` topic is always updated on any hotplug event regardless of transition, to refresh the stratum value. +- A `finally` block logs the reason for shutdown and performs cleanup (stop_stream if still active). +- The ubus `stream_id` must be stopped cleanly on context cancellation to avoid leaking listener registrations in the ubus driver. diff --git a/docs/specs/hal/usb_driver.md b/docs/specs/hal/usb_driver.md new file mode 100644 index 00000000..ede54376 --- /dev/null +++ b/docs/specs/hal/usb_driver.md @@ -0,0 +1,123 @@ +# USB Driver (HAL) + +## Description + +The USB driver is a HAL component that exposes enable/disable control of a USB bus to the rest of the system. The initial scope covers the USB3 bus only (`id = 'usb3'`), modelled as a fixed capability. The actual mechanism for enabling and disabling the bus is device-specific (sysfs, UCI, or a vendor script) and is considered an implementation detail; this spec defines the interface only. + +A trivial **USB Manager** owns this driver. It has no discovery logic: it instantiates a single `UsbDriver` for `usb3`, emits one HAL device-added event, and waits for its scope to be cancelled. The manager behaviour is not complex enough to warrant a separate spec file — it is described inline here. + +## Dependencies + +The exact OS-level mechanism for bus control is implementation-defined. The driver implementation must document which mechanism it uses. + +## Initialisation + +On startup: + +1. Read the current USB3 bus enable/disable state from the OS (implementation-defined). +2. Publish capability `meta`. +3. Publish initial capability `state` (retained). +4. Start the RPC handler fiber. + +If the initial state cannot be determined, assume `enabled = true` and log a warning. + +## Capability + +Class: `usb` +Id: `'usb3'` + +### Meta (retained) + +Topic: `{'cap', 'usb', 'usb3', 'meta'}` + +```lua +{ + provider = 'hal', + version = 1, + bus = 'usb3', +} +``` + +### State (retained) + +Topic: `{'cap', 'usb', 'usb3', 'state'}` + +```lua +{ + enabled = true | false, +} +``` + +Updated and republished after every successful `enable` or `disable` call. + +### Offerings + +#### enable + +Topic: `{'cap', 'usb', 'usb3', 'rpc', 'enable'}` + +Enables the USB3 bus. Input is an empty table `{}`. + +Reply on success: + +```lua +{ ok = true, reason = nil } +``` + +Reply on failure: + +```lua +{ ok = false, reason = } +``` + +On success, updates retained state to `{ enabled = true }`. + +#### disable + +Topic: `{'cap', 'usb', 'usb3', 'rpc', 'disable'}` + +Disables the USB3 bus. Input is an empty table `{}`. + +Reply on success: + +```lua +{ ok = true, reason = nil } +``` + +Reply on failure: + +```lua +{ ok = false, reason = } +``` + +On success, updates retained state to `{ enabled = false }`. + +## Service Flow + +```mermaid +flowchart TD + St[Start] --> A(Read current USB3 bus state from OS) + A --> B(Publish meta) + B --> C(Publish initial state) + C --> D{Wait for enable RPC, disable RPC, or scope cancelled} + D -->|enable RPC| E(Apply enable via OS mechanism) + D -->|disable RPC| I(Apply disable via OS mechanism) + E --> F{Ok?} + I --> J{Ok?} + F -->|yes| G(Publish state enabled=true, reply ok=true) + J -->|yes| K(Publish state enabled=false, reply ok=true) + F -->|no| H(Reply ok=false with error) + J -->|no| H + G --> D + K --> D + H --> D + D -->|scope cancelled| En[Stop] +``` + +## Architecture + +- The driver runs a single RPC handler fiber with `op.choice` over both offerings; the fiber exits when its scope is cancelled. +- State is only updated on success; a failed `enable` or `disable` call leaves the retained state unchanged. +- A `finally` block logs the reason for shutdown. +- If a second `enable` is called when already enabled (or `disable` when already disabled), the driver still applies the OS action and updates the retained state. Idempotency is a property of the OS mechanism, not enforced at this layer. +- Future extension to other buses (USB2, etc.) would follow the same pattern with a different `id`. A manager with bus discovery may be introduced at that point. diff --git a/docs/specs/hal/wlan_manager.md b/docs/specs/hal/wlan_manager.md new file mode 100644 index 00000000..4f5884b4 --- /dev/null +++ b/docs/specs/hal/wlan_manager.md @@ -0,0 +1,75 @@ +# WLAN Manager (HAL) + +## Description + +The WLAN Manager is a HAL manager that owns the lifecycle of all wireless drivers: one `RadioDriver` instance per configured radio and one `BandDriver` instance. + +The WLAN Manager owns no capabilities directly — all capabilities are owned by the drivers it creates. + +## Dependencies + +- HAL device config must contain a `radios` table with one entry per physical radio. If `radios` is absent or empty, no radio drivers are started and a warning is logged. The band driver is always started regardless of radio count. + +## Initialisation + +On startup: + +1. Read the HAL device config passed by the HAL service at startup. +2. For each entry in `config.radios`: create a `RadioDriver` instance and emit a HAL device-added event. +3. Create one `BandDriver` instance and emit a HAL device-added event. + +Each device-added event triggers the HAL service to register the corresponding capability, bind RPC endpoints, and start the emit-forwarding fiber. + +If `config.radios` is absent or not a table, log a warning and start only the band driver. + +## Managed Drivers + +| Driver | Class | Id | Quantity | +|---------------|---------|------------------------------|---------------------------| +| `RadioDriver` | `radio` | UCI radio name (e.g. `'radio0'`, `'radio1'`) | 1 per entry in `config.radios` | +| `BandDriver` | `band` | `'1'` | 1 | + +Radio capability IDs are derived from the UCI radio section name as configured in the device config (e.g. `radio0` → `'radio0'`). + +## Apply Config + +When `apply_config(config)` is called by the HAL service after startup: + +1. **Stop removed drivers** — stop any `RadioDriver` instances whose name no longer appears in `config.radios`. Emit a HAL device-removed event for each. +2. **Stop changed drivers** — stop any `RadioDriver` instances whose name is present in `config.radios` but whose `path` or `type` has changed. Emit a HAL device-removed event for each. These will be recreated in the next step. +3. **Start new and changed drivers** — for each entry in `config.radios` that does not have an existing running driver: create a new `RadioDriver` and emit a HAL device-added event. +4. The `BandDriver` instance is not restarted on config update — it is persistent for the lifetime of the manager. + +## Manager Interface + +The WLAN Manager implements the standard `Manager` interface: + +- `start(scope)` — reads device config, instantiates drivers, emits device events. +- `stop(reason)` — cancels the manager scope; all child driver fibers are stopped via structured concurrency. +- `apply_config(config)` — reconciles the running driver set against the new config. + +## Service Flow + +```mermaid +flowchart TD + St[Start] --> B{config.radios present?} + B -->|no| C(Log warning: no radios in device config) + B -->|yes| D(For each radio: create RadioDriver, emit device added event) + C --> E(Create BandDriver, emit device added event) + D --> E + E --> F{Wait for apply_config or scope cancelled} + F -->|apply_config| G1(Stop drivers not in new config; emit device removed) + G1 --> G2(Stop drivers whose path/type changed; emit device removed) + G2 --> G3(Create new/replacement drivers; emit device added) + G3 --> F + F -->|scope cancelled| H(Stop all driver scopes via structured concurrency) + H --> En[Stop] +``` + +## Architecture + +- All drivers are started within child scopes of the manager scope. If the manager scope is cancelled, all drivers stop automatically via structured concurrency — no explicit teardown loop is needed. +- The manager holds a map of radio name → `{ driver, path, type }` to support reconciliation on `apply_config`. The `path` and `type` fields are compared on each `apply_config` call to detect hardware changes. +- The `BandDriver` is created unconditionally. If the DAWN config is not present on the device, the driver’s `clear` step will fail at the backend level and the capability will not be registered. This is treated as a non-fatal warning. +- UCI access is a concern of the backends only. The `backends/common/uci.lua` module exposes `uci.ensure_started()` — on first call it starts the UCI reactor fiber attached to the **root scope** (not the calling fiber’s scope), so it outlives any individual backend call. Subsequent calls to `ensure_started()` are no-ops. The manager has no dependency on `backends/common/uci.lua`. +- A `finally` block logs the reason for manager shutdown. diff --git a/docs/specs/http.md b/docs/specs/http.md new file mode 100644 index 00000000..466e0db1 --- /dev/null +++ b/docs/specs/http.md @@ -0,0 +1,196 @@ +# HTTP service guide + +## Purpose + +HTTP owns the transport boundary for HTTP and WebSocket work. + +It is responsible for: + +```text +cqueues/lua-http integration +HTTP listener creation +HTTP client exchange +server-side accepted contexts +WebSocket transport handles +request/response command loops +immediate transport termination +HTTP capability publication +transport observability +``` + +HTTP is an infrastructure service. UI and other services consume it through public HTTP capability handles. Its `start(conn, opts)` entry is a foreground service body for `main.lua` and must not return while healthy. Tests and embedded compositions that need the in-process service handle use `services.http.service.open_handle(conn, opts)`. + +## Lifetime shape + +```mermaid +flowchart TD + HttpScope["http service scope"] --> Coordinator["coordinator"] + HttpScope --> Registry["handle registry"] + HttpScope --> Backend["transport driver scope"] + HttpScope --> Lifecycle["svc/http lifecycle"] + HttpScope --> Publisher["cap/http publisher"] + + Coordinator --> Listener["listener scopes"] + Coordinator --> Exchange["client exchange scopes"] + Coordinator --> WS["websocket scopes"] + + Listener --> Context["accepted HttpContext handles"] + Context --> Consumer["consumer request scope"] + Exchange --> Backend + WS --> Backend + + Publisher --> Cap["cap/http/main/..."] + Publisher --> Stats["cap/http/main/state/stats"] + Publisher --> Obs["obs/v1/http/metric/main/stats"] +``` + +Callbacks from transport code must not mutate service model state directly. They may update backend-local state and wake an Op-facing boundary. + +## Public surfaces + +HTTP should publish: + +```text +svc/http/status +svc/http/meta +cfg/http + +cap/http/main/meta +cap/http/main/status +cap/http/main/rpc/listen +cap/http/main/rpc/open-exchange +cap/http/main/rpc/exchange +cap/http/main/rpc/connect-ws +cap/http/main/state/stats + +obs/v1/http/metric/main/stats +``` + +`cap/http/main/state/stats` is a narrow interface-scoped adjunct. Canonical domain state should not be hidden there. + +## Consumption pattern + +A consumer should use the SDK: + +```lua +local http = require('services.http').sdk +local ref = http.new_ref(conn, 'main') + +local reply = fibers.perform(ref:listen_op({ + host = '127.0.0.1', + port = 8080, +})) + +local listener = reply.listener +``` + +The consumer owns accepted contexts after handoff. HTTP owns unaccepted contexts and transport handles until ownership transfer. + +SDK operations should be used as Ops and composed by the caller for timeout policy: + +```lua +local which, result, err = fibers.perform(fibers.named_choice { + exchange = ref:exchange_op(args), + timeout = sleep.sleep_op(5):wrap(function () + return nil, "timeout" + end), +}) +``` + +The HTTP SDK must not install a hidden default bus timeout. If the SDK Op loses a choice, the bus request is abandoned and the admitted HTTP operation is cancelled through request ownership and `scoped_work.cancel_op`. + +For outgoing request bodies backed by iterators, the lua-http backend may otherwise add `Expect: 100-continue` because the content length is not known at request-construction time. HTTP suppresses that implicit header by default so that lua-http's backend-level `expect_100_timeout` does not override the caller's Op-composed timeout policy. Callers or backend owners may opt in explicitly by setting an `Expect: 100-continue` header, by passing `expect_100_continue = true` on the request, or by configuring `expect_100_continue = true` on the backend/service options. `expect_100_timeout` may be supplied alongside that opt-in when the backend wait should have an explicit deadline. + + +## Body object capabilities + +HTTP follows the local bus object-capability style used by HAL/UART. Request +and response bodies are passed as Lua objects with explicit Op-shaped methods; +they are not described by remote references and are not resolved through an HTTP +body registry. + +An outgoing request body is supplied as: + +```lua +{ + uri = 'https://example.test/api', + method = 'POST', + headers = { ['content-type'] = 'application/json' }, + body_source = blob_source.from_string('{"ok":true}'), +} +``` + +`body_source` must provide: + +```text +read_chunk_op(max_bytes) +terminate(reason) +``` + +`response_sink`, when supplied, must provide: + +```text +write_chunk_op(chunk) +terminate(reason) +finish_op() or commit_op() optional success finalisation +``` + +Inline control-plane bytes remain invalid: + +```text +body +body_string +body_chunks +data +chunk +chunks +bytes +``` + +The rule is: + +```text +The bus may carry local Lua object capabilities. +It must not carry raw bulk bytes in control-plane fields. +``` + +If a future Fabric boundary needs to carry HTTP bodies between nodes, Fabric +should adapt known interfaces such as BlobSource and BlobSink into protocol +operations. HTTP should still see the local object shape. + +## Op-only boundary + +Transport operations should be Op-first: + +```text +construct Op +admit command to driver +run backend step +return public wrapper or result +terminate immediately on losing choice/cancellation +``` + +Transport wrappers must provide `terminate(reason)`. + +Graceful protocol closure belongs in `close_op()` and only in request/worker scopes that may wait. + +## Reviewer checklist + +```text +Only transport modules require cqueues or lua-http. +Service code does not use perform_raw. +Public methods use kebab-case. +Returned handles are public wrappers, not backend objects. +Callbacks do not reduce service state directly. +Accepted context ownership transfer is explicit. +Losing HTTP/WebSocket Ops terminate active backend work. +Bus request abandonment cancels admitted HTTP operation work. +The SDK has no hidden default bus timeout policy. +Stats under cap/.../state are narrow. +Metrics also appear under obs/v1/http/... +``` + + +## Dependency-provider status + +`cap/http/main/status.available=true` means the HTTP backend is ready to admit non-status capability requests. While the shell is alive but the backend is starting, status remains queryable and non-status requests should fail with `http_backend_not_ready` or an equivalent not-ready reason. diff --git a/docs/specs/metrics.md b/docs/specs/metrics.md new file mode 100644 index 00000000..3cbfdc49 --- /dev/null +++ b/docs/specs/metrics.md @@ -0,0 +1,372 @@ +# Metrics Service + +## Description + +The metrics service listens to the bus for metrics, applies processing to these metrics and publishes the result to the cloud. + +## Config + +The metrics service takes a selection of configs, validates them and then applies all parts of the config that are valid. If globally required parts of the config are not valid then the whole service fails (publish_period, all pipelines fail). Any pipeline that uses a failed template also fails. A missing cloud URL means any HTTP protocol metrics also fail and the HTTP publisher cannot be started. + +`publish_period`: The period for which metrics are cached before publishing to the cloud, recorded in seconds + +`cloud_url`: The url used to publish http metrics to + +`templates`: a k-v set of metric pipelines that can be reused in the pipelines section, good for different metrics that use the same pipeline without repetition + +`pipelines`: a k-v set of metric pipelines (`name` to pipeline) +- `name`: The name of the metric e.g. `rx_bytes` + +A pipeline is defined as: + +- `template`: An optional argument for using a template (can be modified with further arguments) +- `protocol`: The protocol to be used for publication, either log or http +- `process`: A list of processing blocks to be used on the metric +- - `type`: The process block name +- - additional fields are the arguments for that block (flat, not nested under an `args` key) + +An example config + +```json +"metrics": { + "publish_period": 60, + "cloud_url": "cloud.com", + "templates": { + "network_stat": { + "protocol": "http", + "process": [ + { + "type": "DiffTrigger", + "diff_method": "percent", + "threshold": 5, + "initial_val": 0 + }, + { + "type": "DeltaValue" + } + ] + } + }, + "pipelines": { + "rx_bytes": { + "template": "network_stat" + }, + "sim": { + "protocol": "log", + "process": [ + { + "type": "DiffTrigger", + "diff_method": "any-change" + } + ] + } + } +} +``` + +## Metrics over the Bus + +Metrics received over the bus will need to be in a specific format to be processed. The metric payload must include a `value` field and may also contain a `namespace` field. The namespace is a list of string or integer tokens which denotes the full senml key of that metric. Without a namespace, the metric key will be derived from the bus topic path. + +The bus topic path determines which pipeline config is applied to the metric. The topic is an array of strings/integers that form the metric's endpoint identifier. The exact topic structure depends on the bus namespace conventions used (see runtime model documentation for topic organization). + +An example bus message for a simple metric: + +```lua +{ + topic = { , }, -- e.g., { 'obs', 'v1', 'network', 'metric', 'rx_bytes' } + payload = { + value = 50 + } +} +``` + +Or a metric with a namespace override: + +```lua +{ + topic = { , }, -- e.g., { 'obs', 'v1', 'modem', 'metric', 'sim' } + payload = { + value = 'present', + namespace = { 'modem', 1 } -- overrides the senml key structure + } +} +``` + +To make creation and validation of a metric easier the service will provide an sdk which will include a types module; with the types module the user can create a metric type which will validate the arguments provided and return a metric type or error. The service can accept metrics types or tables which it will validate by trying to cast to a metric type. + +## Configuration & Validation + +The metrics service validates all configuration on receipt and handles invalid configs gracefully: + +### Validation Rules + +- `publish_period`: Must be a positive number (seconds). Invalid values cause service to reject entire config. +- `cloud_url`: Must be a string. Missing URL prevents HTTP protocol from functioning. +- `templates`: Each template must have valid `protocol` and optional `process` array. Invalid templates are dropped. +- `pipelines`: Each pipeline must specify a valid `protocol`. Pipelines referencing dropped templates are also dropped. +- `protocol`: Must be one of: `http`, `log`, or `bus`. Invalid protocols cause that pipeline to be dropped. +- `process`: Must be an array of process block tables, each with a `type` field; additional fields are the block's arguments. + +### Warning Handling + +When invalid configurations are detected: + +1. **Template Failures**: Invalid templates are removed from the config +2. **Dependent Metrics**: Any pipeline referencing a failed template is also removed +3. **Protocol Errors**: Pipelines with invalid protocols are dropped +4. **Graceful Degradation**: Service continues with all valid pipelines; only fails if no valid pipelines remain or `publish_period` is invalid + +All dropped metrics and templates are logged with detailed warning messages including: +- Specific validation errors for each dropped item +- Summary of total dropped metrics and templates +- List of affected pipeline names + +This allows partial config updates to succeed even if some pipelines are misconfigured. + +### Mainflux Config Normalization + +The mainflux configuration supports both legacy and current naming conventions: +- `mainflux_id` or `thing_id` (normalized to `thing_id`) +- `mainflux_key` or `thing_key` (normalized to `thing_key`) +- `mainflux_channels` or `channels` (normalized to `channels`) + +Channel metadata is automatically injected by scanning channel names: +- Channels containing "data" get `metadata.channel_type = "data"` +- Channels containing "control" get `metadata.channel_type = "events"` + +The metrics service merges the mainflux config with the metrics `cloud_url` to build the final HTTP publication config. + +## Publication of metrics + +Once the publish period has triggered the cache will be purged and all k-v pairs will be converted to a senml representation with correct timestamping. There will be 3 possible protocol for sending of the metrics. + +### Log + +This method will iterate over the list of senml items and log them using the logging module with info mode (`log.info(key, value)`). + +### HTTP + +This method will send the senml list to the cloud by building up a cloud config by merging the cloud_url with the mainflux config. The mainflux config is retrieved by loading it from HAL via the `filesystem` capability with the mainflux.cfg file name, this is then decoded from JSON to a Lua table and then normalised (`mainflux_id` -> `thing_id`, `mainflux_key` -> `thing_key`). The mainflux config is of the structure: + +```json +{ + "thing_id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", + "thing_key": "550e8400-e29b-41d4-a716-446655440000", + "channels": [ + { + "id": "6ba7b810-9dad-11d1-80b4-00c04fd430c8", + "name": "f47ac10b-58cc-4372-a567-0e02b2c3d479_data", + "metadata": { + "channel_type": "data" + } + }, + { + "id": "6ba7b811-9dad-11d1-80b4-00c04fd430c8", + "name": "f47ac10b-58cc-4372-a567-0e02b2c3d479_control", + "metadata": { + "channel_type": "control" + } + } + ], + "content": "{\"hawkbit\":{\"hawkbit_key\": \"examplehawkbitkey0123456789abcdef\",\"hawkbit_url\": \"https://example.com/hawkbit\"},\"serial\": \"EX1234567890ABCD\",\"networks\":{\"networks\":[{\"name\": \"example-network\",\"ssid\": \"Example WiFi-aB1xY\",\"password\": \"examplepassword123\"}]}}" +} +``` +The resulting merger should be formatted into the structure: + +```lua +{ + url = "cloud.com", + data_id = "6ba7b810-9dad-11d1-80b4-00c04fd430c8", + thing_key = "550e8400-e29b-41d4-a716-446655440000" +} +``` + +The cloud URI is composed as `cloud.url` + `/http/channels/` + `cloud.data_id` + `/messages` + +The authentication header is composed as `Thing ` + `cloud.thing_key` + +The body is a json encoding of the senml list + +As HTTP publications can fail due to lack of backhaul, boot-time metrics are very important, and publications can cause long blocking time on the fiber they are running on, so we run the HTTP sending in a separate fiber with a queue between the main metrics service fiber and the HTTP fiber. + +**Queue Behavior**: +- Fixed capacity: **10 items** +- On queue full: New metrics are **dropped silently** with error log +- No retry mechanism for queue-full conditions (older metrics are preserved) + +**Network Failure Retry**: +- When HTTP send fails (network unavailable), the fiber implements exponential backoff +- Sleep duration doubles on each failure: 1s → 2s → 4s → 8s → ... → max 60s +- Retries indefinitely until successful or service shutdown +- Queue-full and network-failure are handled separately: only network failures trigger retry + +### Bus + +For testability a bus protocol should be provided which will publish the metrics under the topic +`obs/v1/metrics/output/` for example `obs/v1/metrics/output/modem/1/sim`. The payload will just be the value and timestamp of the metric. + +## Processing + +Each metric goes through a processing pipeline which can be composed of a variable amount of process blocks or none at all. + +### Pipeline Architecture + +The metrics service uses a **shared pipeline with per-endpoint state** model: +- Each configured pipeline (from config) is instantiated **once** as a processing pipeline object +- Each unique metric endpoint (topic path) gets its own **state table** +- The same pipeline logic processes metrics from different endpoints using different state tables +- This allows pipeline code reuse while maintaining isolated state per endpoint + +**State Isolation**: A metric published to namespace `network.wan.rx_bytes` and another to namespace `network.lan.rx_bytes` use the same pipeline logic but maintain separate DiffTrigger/DeltaValue state, preventing cross-endpoint interference. + +All process blocks and the pipeline itself have an interface to reset state when a value has been published. The pipeline and blocks are logic and config only; they require a state table as input for each invocation. + +### ProcessPipeline + +The process pipeline contains all processing blocks and runs them sequentially until either: +1. A block returns a short-circuit flag (stops processing early) +2. All blocks complete successfully + +The pipeline returns: `(value, short_circuit_flag, error)` + +**Short-Circuit Behavior**: When a processing block returns `short_circuit = true`, the pipeline immediately stops processing and the metric is **not stored or published**. This is intentional control flow (e.g., DiffTrigger suppressing unchanged values), not an error condition. + +**Reset Behavior**: The pipeline only resets individual block state if a complete run was achieved (no short-circuit). This ensures state is only updated when metrics are actually published. + +**Metric Storage Structure**: Successfully processed metrics are stored in a nested structure: `metric_values[protocol][endpoint] = {value, time}`, allowing the same endpoint to publish via multiple protocols (e.g., both "http" and "log"). + +### DiffTrigger + +The difference trigger is a block which only outputs a value if the difference between the last output value and the current value hits a threshold. There are 3 different types of threshold: + +- `percent`: Only triggers if the current value is a percentage difference of the last output value +- `absolute`: Only triggers if the current value is an absolute difference of the last output value +- `any-change`: Triggers if the current value does not match the last output value + +**Arguments** (fields on the process block table, alongside `type`) + +- `diff_method`: This can either be `percent`, `absolute` or `any-change` +- `initial_val`: This is an optional argument to set the previous value before any value has been set. If this value is not set then the block will always output the first value received +- `threshold`: This argument is only for `percent` and `absolute` types, the threshold that must be hit to trigger an output + +**Behavior Details** + +- **First Value**: The first value received always publishes (triggers) regardless of threshold, unless `initial_val` is specified +- **Reset**: DiffTrigger state is **not affected by reset** - it maintains `last_val` across publish cycles +- **Short-Circuit**: When threshold is not met, returns `short_circuit = true` to prevent publishing + +### TimeTrigger + +The time trigger outputs a value only once a certain time period has elapsed since the last value output. Uses monotonic time to ensure immunity to system clock changes. + +**Arguments** (fields on the process block table, alongside `type`) +- `duration`: the time between value outputs (seconds) + +**Behavior Details** + +- **Timing**: Timer is reset after each successful output, ensuring periodic publishing every `duration` seconds +- **Reset**: TimeTrigger state is **not affected by reset** - timeout persists across publish cycles +- **Short-Circuit**: When timeout has not expired, returns `short_circuit = true` to suppress output + +### DeltaValue + +The delta value block always outputs the difference between the last published value and the current value. This is useful for converting cumulative counters into per-period deltas. + +**Arguments** + +- `initial_val`: An optional initial value to compare the first values to before the first publish, if this is not set the default is 0 + +**Behavior Details** + +- **Calculation**: Returns `current_value - last_published_value` +- **Reset**: On reset, sets `last_val = current_val` (resets to the current value, not zero) +- **Reset Trigger**: Only resets after a successful publish cycle (when pipeline completes without short-circuit) +- **Always Publishes**: Does not short-circuit; always returns a delta value + +## Timestamping + +Each metric must have an accurate time stamp. As the time at boot will be incorrect until the time service has done a ntp time sync, the metrics service must hold metrics with a relative timestamp which can be converted into a real timestamp once both time has been synced and the publish cache is ready to publish (via the publish period). + +### NTP Sync Dependency + +The metrics service **blocks all publishing** until NTP time synchronization is confirmed: + +- Service subscribes to `{'state', 'time', 'synced'}` bus topic +- When `synced = true` is received: + - **First sync**: Calculates base time offset and schedules first publish + - **Subsequent syncs**: Continues using existing offset +- When `synced = false` is received: + - Publishing is suspended indefinitely + - Publish timer is disabled (`next_publish_time = infinity`) + - Metrics continue to be collected but not published + +### Time Conversion + +We can convert between relative and real time by using monotonic time stamps on the metrics and holding the monotonic time that the ntp sync was done as well as the real time that the sync was done. + +**Base Time Calculation** (on first NTP sync): +- `base_real = current_realtime - (current_monotime - initial_monotime)` +- This calculates what the real time was at the initial monotonic reference point + +**Metric Timestamp Conversion** (at publish time): +- `metric_timestamp_ms = floor(base_real + (metric_monotime - base_monotime)) * 1000` +- This converts the metric's monotonic timestamp to real time in milliseconds + +**Important**: If NTP sync is lost and regained, the original base time offset is preserved rather than recalculated. This ensures timestamp consistency across sync interruptions. + +## Testing + +Current automated coverage is in `tests/test_metrics.lua`. + +Run command: + +```bash +cd tests && luajit test_metrics.lua +``` + +Covered test groups: + +- **Processing blocks (`TestProcessing`)** + - `DiffTrigger` behavior for `absolute`, `percent`, and `any-change` + - `DeltaValue` behavior and reset semantics + - `ProcessPipeline` sequencing, reset, and short-circuit behavior + +- **Config module (`TestConfig`)** + - `validate_http_config` valid + invalid inputs + - `merge_config` recursive merge behavior + - `apply_config` builds valid pipeline maps + - `validate_config` hard rejection (`publish_period <= 0`) + - `validate_config` warning path for invalid protocol + - invalid template propagation to dependent pipeline warnings + +- **SenML encoder (`TestSenML`)** + - encode number/string/boolean/time records + - reject unsupported value types + - `encode_r` flattening behavior for nested keys + +- **HTTP publisher module (`TestHttpModule`)** + - `start_http_publisher` builds expected HTTP request shape + - verifies method/header/body composition via mocked `http.request` + +- **Service-level behavior (`TestMetricsService`)** + - bus protocol publish end-to-end + - namespace override routing/topic mapping + - unknown metrics are dropped + - `DiffTrigger` suppresses unchanged values + - `DeltaValue` emits per-period deltas + - config update replaces active pipelines + - per-endpoint state isolation across namespaces + - HTTP protocol pipeline enqueues expected Mainflux payload + +Notes: + +- Service tests use virtual time helpers from `tests/test_utils/virtual_time.lua` and `tests/test_utils/time_harness.lua`. +- Service tests subscribe to `{'svc','metrics','#'}` and intentionally filter out `status` lifecycle messages when asserting metric payload publications. + + + + + + + diff --git a/docs/specs/net.md b/docs/specs/net.md new file mode 100644 index 00000000..ee4133b0 --- /dev/null +++ b/docs/specs/net.md @@ -0,0 +1,500 @@ +# NET service specification + +This document describes the `net` service as implemented in this tree. + +`net` is the product-level network authority. It owns network intent, apply generations, observation, drift modelling, WAN runtime decisions and retained state publication. It does not own platform implementation detail. + +```text +net decides what the network should be. +HAL decides how the host implements it. +``` + +OpenWrt, UCI, netifd, ubus, dnsmasq, firewall4, MWAN3, `tc`, IFB, u32, HTB, fq_codel, init scripts, sysfs and command execution are HAL backend concerns. `net` talks to HAL through semantic operations only. + +## Products + +### Big Box + +Big Box is a rugged communications station built around: + +```text +CM5 + OpenWrt soft router + main Devicecode Lua runtime + MT7915 Wi-Fi AP card + WAN sources including cellular modems + +RP2350 MCU + TinyGo bare-metal controller + PMU and LTC4015 solar/battery charge-controller integration + +RTL8380M switch fabric + OpenWrt-capable switching fabric + PoE and wired-port fabric management +``` + +Given Big Box intended use cases (remote schools, clinics, emergency response and displaced populations), Network behaviour must be resilient, observable, explainable and adaptive. + +### Get Box + +For Get Box the same `net` contract should apply, even when Wi-Fi, switching and routing are all realised on one OpenWrt target. + +## Service boundaries + +```text +net + segments, VLAN identity, addressing, DHCP/DNS policy, firewall, + routing, WAN, multi-WAN, VPN intent, shaping, diagnostics and state + +wifi + radios, SSIDs, wireless clients, AP policy, channel/power policy, + wireless inter-links and wireless backhaul presentation + +wired + direct Ethernet surfaces, switch ports, access/trunk attachment, + PoE capability, link state, provider validation and wired topology + +HAL + all OS-facing and hardware-facing implementation +``` + +Rules: + +```text +If it changes who can talk to whom, it belongs to net. +If it changes how a wireless network is advertised or associated with, it belongs to wifi. +If it changes how a wired physical surface attaches to a segment, it belongs to wired. +If it touches OpenWrt, kernel networking, modem drivers, Wi-Fi drivers or switch ASICs, it belongs behind HAL. +``` + +`wifi` attaches SSIDs to segment names. `wired` attaches wired surfaces to segment names. Neither duplicates VLAN allocation, DHCP, firewall, routing or WAN policy. + +## Implemented configuration shape + +`src/services/net/config.lua` accepts only: + +```text +devicecode.config/net/1 +``` + +It normalises this into: + +```text +devicecode.net.intent/1 +``` + +There are no compatibility migrations inside `net`. Migration must happen before data is written to `cfg/net`. + +The current Big Box example is: + +```text +src/configs/bigbox-v1-cm-2.json +``` + +### Authority rules + +The current clean config uses this authority split: + +```text +segments + ordinary per-network facts: + VLAN identity, addressing, segment-local DHCP pool, local DNS behaviour, + firewall zone attachment and shaping profile reference + +top-level dns + resolver policy, upstreams, cache size, global records and host-file catalogue + +top-level dhcp + defaults, reservations, options and relays + +top-level firewall + defaults, zone definitions, forwarding policies and rules + +top-level routing + explicit route records. Host routes use `kind = "host"` and render `/32` netmask; subnet routes use `kind = "subnet"` with explicit netmask. The default Big Box config includes `starlink_admin` for the Starlink dish/admin UI at `192.168.100.1` via wired WAN. + +top-level wan + uplink membership, metrics, base weights, dynamic weighting and health policy + +top-level shaping + profiles and classes; segments refer to profiles + +top-level vpn + overlay/tunnel intent; OpenWrt tunnel application is not implemented yet + +top-level diagnostics/runtime + reflectors, timings and service behaviour +``` + +In practical terms: + +```text +A segment says: I am guest, VLAN 32, 172.28.32.1/24, DHCP enabled, +firewall zone lan_rst, shaping profile restricted_user_per_host. + +The shaping section says what restricted_user_per_host means. + +The DNS section says which upstreams, cache size, records and host-file sources +exist. + +A segment DNS block says which host-file ids apply to that segment. +``` + +### Content-filter host files + +`dns.host_files` is the top-level catalogue for local content-filter host files. A segment references host-file ids in its own `dns.host_files` array. + +Example shape: + +```json +{ + "dns": { + "host_files": { + "base_dir": "/data/devicecode/dns/hosts", + "addnmount": true, + "sources": { + "ads": { "file": "ads.hosts" }, + "adult": { "file": "adult.hosts" } + } + } + }, + "segments": { + "jan": { + "dns": { "host_files": ["ads", "adult"] } + } + } +} +``` + +The OpenWrt provider translates these to dnsmasq `addnhosts` and, where requested, `addnmount` entries. The location is configurable through `dns.host_files.base_dir`. + +## Current source layout + +```text +src/services/net.lua +src/services/net/service.lua +src/services/net/config.lua +src/services/net/schema.lua +src/services/net/model.lua +src/services/net/topics.lua +src/services/net/projection.lua +src/services/net/publisher.lua +src/services/net/events.lua +src/services/net/backpressure.lua +src/services/net/stale.lua +src/services/net/hal_client.lua +src/services/net/generation.lua +src/services/net/apply_runtime.lua +src/services/net/wan_runtime.lua + +src/services/net/domain/addressing.lua +src/services/net/domain/dhcp.lua +src/services/net/domain/diagnostics.lua +src/services/net/domain/dns.lua +src/services/net/domain/firewall.lua +src/services/net/domain/interfaces.lua +src/services/net/domain/multiwan.lua +src/services/net/domain/routing.lua +src/services/net/domain/segments.lua +src/services/net/domain/shaping.lua +src/services/net/domain/vpn.lua +``` + +`src/services/net.lua` is a thin entry point. `service.lua` owns the coordinator loop and service state. Domain modules are pure validation and normalisation code. + +## Service discipline + +`net` follows the Devicecode `fibers` service discipline. + +```text +Ops describe possible waits. +Scopes own lifetimes. +Coordinator branches do not block. +Workers perform blocking HAL, diagnostic or backend-facing work. +Completions carry identity and generation. +Finalisers terminate; they do not wait. +``` + +The coordinator receives a single next event, reduces it into state changes, starts scoped work where required and publishes immediate retained state. It does not perform HAL work inline. + +Implemented scoped work includes: + +```text +structural apply + services.net.apply_runtime + +WAN speedtests + services.net.wan_runtime + +live WAN weight application + services.net.wan_runtime +``` + +Stale completion rejection is centralised in `services.net.stale` and covers apply, speedtest and live-weight completions. + +## HAL network provider + +The semantic HAL network provider is under: + +```text +src/services/hal/backends/network/provider.lua +src/services/hal/backends/network/contract.lua +src/services/hal/backends/network/providers/fake/init.lua +src/services/hal/backends/network/providers/openwrt/init.lua +``` + +The OpenWrt provider implements the current platform backend. It translates `devicecode.net.intent/1` into OpenWrt packages and runtime operations. `net` must not require it directly. + +Current provider operations include: + +```text +validate_op +plan_op +apply_op +snapshot_op +watch_op +probe_link_op +read_counters_op +apply_live_weights_op +apply_shaping_op +speedtest_op +``` + +## Current OpenWrt apply coverage + +The OpenWrt provider applies the current Big Box config domains as follows: + +```text +network + unconditional loopback and globals baseline + segment trunk VLAN devices on the configured base interface + bridge-backed LAN/system/user/guest segments by default + direct WAN/uplink segments by default + segment logical interfaces + explicit interfaces + named explicit static route records + +dhcp / dnsmasq + grouped dnsmasq instances by effective DNS policy + dns cache size + upstream DNS servers + top-level DNS records as dnsmasq address entries + segment content-filter host files through addnhosts/addnmount + segment-local DHCP pools bound to their dnsmasq instance + DHCP reservations + per-segment DHCP options where configured + +firewall + defaults, including extra default keys present in config + zones and network membership + forwarding policies + rules + +mwan3 + realised WAN interfaces, members, policies and config-declared rules + HTTPS stickiness as a product policy rule, not an implicit provider default + automatic distinct /etc/config/network route metrics for realised uplinks + health basics and mark mask + structural mwan3 restart through activation runner after UCI changes + live weight application through runtime path without mwan3 restart + +traffic shaping + segment shaping profile references are compiled into HAL shaper requests + OpenWrt backend uses the u32/HTB/fq_codel shaper locally + +vpn + intent is normalised and published + OpenWrt provider currently reports configured tunnels as unsupported +``` + +Provider apply is a complete rewrite for the UCI packages it owns. Devicecode-generated OpenWrt packages do not carry `devicecode_*` watermark options; the provider instead returns the semantic-to-generated-name map from `plan_op` and `apply_op` under `openwrt_names`. UCI application uses the scoped UCI manager transaction path with package snapshots and rollback on partial failure. Missing package files are created before apply. + + +WAN route metrics are generated by NET/HAL and are not product configuration. +They exist so OpenWrt and mwan3 have distinct main-table route metrics for +realised WAN uplinks. `mwan_metric` is the mwan3 member policy tier and is a +separate concept. + +GSM-backed members are realised only after NET observes +`state/gsm/uplink/` with `linux.ifname`. NET does not guess `wwan0` or +`wwan1`, and unavailable GSM members are not emitted into `/etc/config/mwan3`. + +MWAN rules are product policy under `wan.rules`. For example, HTTPS stickiness +is expressed as a rule with `proto = tcp`, `dest_port = 443`, `sticky = true` +and `policy = balanced`; the OpenWrt provider translates that into an mwan3 +`config rule` using `option sticky '1'` and `option use_policy ''`. + +## OpenWrt VM generated-config renderer + +The OpenWrt VM lane includes a renderer for inspecting the exact generated +`/etc/config` files for the default Big Box NET config without modifying the +VM's real `/etc/config` packages. It is a documentation and regression tool for +reviewing the product config, NET normalisation, GSM realisation and OpenWrt +provider output as one chain. + +The renderer is: + +```text +tests/integration/openwrt_vm/scripts/render-default-configs +``` + +The Makefile entry points are: + +```sh +make render-default-configs +make print-default-configs +``` + +`render-default-configs` writes files under: + +```text +tests/integration/openwrt_vm/work/generated-default-etc-config/ +``` + +`print-default-configs` does the same and also prints the rendered `network`, +`dhcp`, `firewall` and `mwan3` files to stdout. The output directory also +contains `manifest.json`, which records the source config, generated OpenWrt +name map, realised WAN members, activation commands that would have been run, +and shaping commands that would have been run. + +Internally the tool copies `src` and `vendor` into the VM, normalises +`src/configs/bigbox-v1-cm-2.json`, realises the NET intent and runs the real +OpenWrt network provider against a temporary UCI `confdir`/`savedir`. Provider +activation and shaper command hooks are stubbed so the generated config can be +reviewed safely without touching the VM's live network, firewall or mwan3 +state. This makes it useful for reviewing complete UCI output while keeping the +real end-to-end connectivity tests separate. + +By default the renderer uses `eth0` as the segment trunk and has no GSM uplink +facts, so only the wired WAN member is realised. The trunk and GSM realisation +inputs can be overridden with environment variables: + +```sh +DEVICECODE_RENDER_TRUNK_IFNAME=eth0 make print-default-configs + +DEVICECODE_RENDER_GSM_PRIMARY_IFNAME=wwan0 \ +DEVICECODE_RENDER_GSM_SECONDARY_IFNAME=wwan1 \ +make print-default-configs +``` + +The GSM override path is the preferred way to inspect configs where semantic +WAN members such as `modem_primary` and `modem_secondary` have been realised +into concrete Linux interfaces. It should show the generated OpenWrt logical +interface names consistently across `network`, `dhcp`, `firewall` and `mwan3`, +while preserving semantic member names where they are valid bounded mwan3 +section names. + +The renderer complements, but does not replace, the real VM connectivity tests. +`test_openwrt_vm_generated_configs_expected.sh` asserts expected generated UCI +shape, while `test_openwrt_vm_mwan_connected.sh` applies generated config to +the VM and proves that routes and mwan3 are active over the VM WAN links. + +## Traffic shaping + +The current OpenWrt shaping backend is: + +```text +src/services/hal/backends/network/providers/openwrt/tc_u32_shaper.lua +``` + +It is HAL-local. `net` describes product-level shaping profiles and segment attachments; the backend owns `tc`, IFB, u32, HTB and fq_codel detail. + +The model supports per-direction ingress/egress policy, IFB ingress, host-set expansion, per-host HTB/fq_codel deltas, dirty-state recovery and `tc -batch` programming. + +## WAN runtime + +Speedtests apply to WAN members generally, not only cellular/GSM members. The current service starts scoped speedtest work for eligible WAN members when enabled by config, records identity-bearing completions, and then computes live weights from current speedtest results. + +Live weight application is also scoped work and is stale-checked before reduction into the model. + +## Observation and drift + +Observation is event-led. + +```text +events tell us something changed +snapshots tell us what is now true +``` + +The OpenWrt provider owns raw event ingestion and live snapshots. The current event sources are: + +```text +hotplug-style UNIX socket ingress +mwan3.user-style UNIX socket ingress +optional ubus network.interface listener +``` + +The provider coalesces raw events, takes targeted live snapshots and emits semantic observed events to `net`. `net` reduces those events into observed state and drift. + +Current drift is intentionally shallow but structured. It is expected to grow by domain, including firewall, DNS/DHCP, routes, MWAN, shaping and VPN. + +## Retained state + +`net` currently publishes: + +```text +state/net/summary +state/net/apply +state/net/segments +state/net/vlan-policy +state/net/segment/ +state/net/interface/ +state/net/addressing +state/net/dns +state/net/dhcp +state/net/firewall +state/net/routing +state/net/wan +state/net/wan_runtime +state/net/shaping +state/net/vpn +state/net/diagnostics +state/net/observed +state/net/drift +``` + +`net` owns `state/net/...` only. It does not publish into `state/wifi/...`, `state/wired/...` or `state/device/...`. + +## Current tests + +Relevant coverage in this tree includes: + +```text +unit net config validation and Big Box clean config shape +unit net service behaviour, stale completion handling and WAN runtime behaviour +unit HAL OpenWrt provider planning/application for DNS, firewall, routes and shaping +unit UCI manager transaction and rollback behaviour +OpenWrt VM baseline +OpenWrt VM UCI manager +OpenWrt VM network provider apply/snapshot/live snapshot +OpenWrt VM observer ingress +OpenWrt VM VLAN/MWAN/shaping +OpenWrt VM live MWAN weights +OpenWrt VM segment trunk +Big Box phase-one composition and broken-trunk checks +``` + +The VM suite confirms the main OpenWrt-facing seams against a real OpenWrt test target. + +## Current limits + +Current deliberate limits: + +```text +VPN tunnel application is not yet implemented in the OpenWrt provider. +Drift modelling is still early and should be expanded by domain. +The segment trunk implementation is Phase 1 and assumes the configured base interface. +Traffic shaping is powerful but still one backend strategy, not a product-wide optimiser. +``` + +## Design north star + +`net` is the connectivity brain of Big Box and Get Box while remaining platform-agnostic. + +```text +net decides connectivity policy. +wifi realises wireless access and wireless transport. +wired realises wired fabric. +HAL performs host/device-specific work. +Devicecode publishes clear, explainable state and decisions. +``` diff --git a/docs/specs/system.md b/docs/specs/system.md new file mode 100644 index 00000000..49f6ce89 --- /dev/null +++ b/docs/specs/system.md @@ -0,0 +1,171 @@ +# System Service + +## Description + +The System Service is an application-layer service responsible for: + +1. **Sysinfo reporting** — periodically gathering system metrics from HAL capabilities and publishing each as an individual `obs/v1/system/metric/` metric on the bus. +2. **Alarm management** — scheduling and firing time-based alarms (daily or one-shot). +3. **Shutdown orchestration** — on an alarm, broadcasting a shutdown signal to all services, waiting for graceful shutdown, then issuing a power command via the HAL `power` capability. +4. **USB3 control** — on config update, disabling USB3 via the HAL `usb` capability if configured. + +The service interacts with the hardware exclusively through HAL capabilities on the bus. No direct file reads or command execution are performed by the service itself — all OS interaction is delegated to HAL. + +## Dependencies + +HAL capabilities consumed: + +| Capability class | Id | Usage | +|------------------|-----------|------------------------------------------------------------| +| `cpu` | `'1'` | Get CPU utilisation and frequency fields | +| `memory` | `'1'` | Get RAM total, used, free, util fields | +| `thermal` | any | Get temperature per discovered zone | +| `platform` | `'1'` | Subscribe to retained state (hw_revision etc.); get uptime to compute boot_time | +| `power` | `'1'` | Issue shutdown or reboot command | +| `usb` | `'usb3'` | Disable USB3 bus on config | + +Service topics consumed: + +| Topic | Usage | +|-------------------------------|--------------------------------------------------------------------| +| `{'state', 'time', 'synced'}` | Retained bool — both fibers subscribe; sysinfo fiber uses it to gate `boot_time` computation; main fiber uses it to sync/desync alarms | + +## Configuration + +Received via retained bus message on `{'cfg', 'system'}`. + +```lua +{ + report_period = , -- required: sysinfo publish interval in seconds + usb3_enabled = , -- required: if false, disable USB3 via HAL + alarms = , -- optional: list of alarm config tables +} +``` + +Each alarm config: + +```lua +{ + time = , -- required: "HH:MM" (24-hour, wall-clock) + repeats = , -- optional: "daily" or omitted for one-shot + payload = { + name = , -- human-readable label, used in shutdown reason + type = , -- "shutdown" or "reboot" + } +} +``` + +## Sysinfo Reporting + +On each report interval the sysinfo fiber calls `get` on each consumed capability with a `max_age` equal to `report_period` (so readings cached from the previous iteration are reused when the period is short). Each field is published individually as an observability metric on the bus following the `{'obs', 'v1', 'system', 'metric', }` topic convention. + +The sysinfo fiber subscribes to `{'svc', 'time', 'synced'}` and tracks a local `time_synced` flag. This is required because `boot_time` is computed as `os.time() - uptime` — if published before NTP sync, `os.time()` returns the wrong value and the stored metric would be incorrect even after the metrics service eventually ships it. The `boot_time` metric is therefore only computed and published when `time_synced = true`. + +Static platform identity fields (`hw_revision`, `fw_version`, `serial`, `board_revision`) are read once from the retained platform state topic at sysinfo fiber startup and published as metrics at that point. They are not re-fetched or re-published on each report cycle. + +### Published Metrics + +Each metric payload contains a `value` field. Metrics that need a namespace override (to set the senml key independently of the topic) also carry a `namespace` field. + +| Topic (`{'obs','v1','system','metric', …}`) | Namespace override | Source | Notes | +|---------------------------------------------|--------------------------------------------|-------------------------------|-------------------------------------------------| +| `'boot_time'` | — | `os.time() - platform uptime` | **only when `time_synced = true`**; Unix epoch | +| `'hw_revision'` | — | platform retained state | published once at startup | +| `'fw_version'` | — | platform retained state | published once at startup | +| `'serial'` | — | platform retained state | published once at startup | +| `'board_revision'` | — | platform retained state | published once at startup | +| `'cpu_utilisation'` | — | cpu cap `get utilisation` | | +| `'cpu_frequency'` | — | cpu cap `get frequency` | | +| `'mem_total'` | — | memory cap `get total` | bytes | +| `'mem_used'` | — | memory cap `get used` | bytes | +| `'mem_free'` | — | memory cap `get free` | bytes | +| `'mem_util'` | — | memory cap `get util` | percentage | +| `'temperature'` | — | thermal cap `get` on zone `'zone0'` | single value; historically zone 0 only | + +Temperature is always reported from zone `'zone0'` — a single metric matching the historical name. Multi-zone reporting is intentionally not implemented to preserve collected metric continuity. + +### Thermal zone discovery + +The sysinfo fiber subscribes to `{'cap', 'thermal', '+', 'meta'}` to discover available thermal zones dynamically. The subscription is used only to confirm that `zone0` is present before attempting reads. No per-zone metric publications are made. + +## Alarm Management + +Alarms are managed internally by an `AlarmManager` instance (same logic as the old service). Alarms are wall-clock scheduled and require NTP sync to fire correctly, so the alarm manager is only activated once the time service reports `synced = true`. + +- On sync: `alarm_manager:sync()` — recalculates all next-trigger times from the current wall clock +- On unsync: `alarm_manager:desync()` — prevents alarms from firing until re-synced +- On alarm fire: call `_handle_alarm` (see Shutdown Orchestration) + +Alarms are replaced wholesale on each config update: `delete_all()` followed by `add()` for each entry. + +## Shutdown Orchestration + +When an alarm with `type = 'shutdown'` or `type = 'reboot'` fires: + +1. Compute `deadline = monotime() + 10`. +2. Publish retained `{'state', 'system', 'shutdown'}` with `{ reason = alarm.payload.name, deadline = deadline }`. +3. Create an independent scope with a deadline of `deadline + 1` (one extra second beyond the broadcast deadline). +4. Subscribe to `{'+', 'health'}` and track all services that are not yet `'disabled'`. +5. Wait for all tracked services to reach `'disabled'` state, or until the deadline scope expires. +6. For any service that did not shut down in time, subscribe to `{service, 'health', 'fibers', '+'}` and log which fibers are stuck. +7. Call the `power` capability RPC (`shutdown` or `reboot`) with `{ reason = alarm.payload.name }`. + +The shutdown scope is independent of the service scope, since the service scope will itself be cancelled by the broadcast shutdown signal. + +## Service Flow + +### System Main fiber + +```mermaid +flowchart TD + St[Start] --> A(Subscribe to cfg/system + state/time/synced) + A --> B{op.choice: config msg, time synced msg, alarm fires, scope cancelled} + B -->|config msg| C(Apply config: update report_period channel, handle USB3, reload alarms) + C --> B + B -->|time synced msg payload=true| D(alarm_manager:sync) + D --> B + B -->|time synced msg payload=false| E(alarm_manager:desync) + E --> B + B -->|alarm fires| F(_handle_alarm) + F --> G(Broadcast shutdown signal) + G --> H(Wait for services to stop or deadline) + H --> I(Log any stuck fibers) + I --> J(Call power cap RPC: shutdown or reboot) + J --> B + B -->|scope cancelled| En[Stop] +``` + +### Sysinfo fiber + +```mermaid +flowchart TD + St[Start] --> A(Subscribe to state/time/synced + thermal cap meta) + A --> B(Receive report_period from config channel) + B --> C(Read platform retained state; publish hw_revision, fw_version, serial, board_revision metrics) + C --> D{op.choice: sleep report_period, new report_period, time synced msg, thermal meta msg, scope cancelled} + D -->|sleep expires| E(Call get RPC: cpu utilisation + frequency) + E --> E2(Call get RPC: memory total, used, free, util) + E2 --> E3(Call get RPC: platform uptime) + E3 --> E4("If time_synced: compute boot_time = os.time() - uptime; publish boot_time metric") + E4 --> E5(Call get RPC: zone0 temperature, if zone0 is known) + E5 --> F(Publish each result to obs/v1/system/metric/name) + F --> D + D -->|new report_period| I(Update report_period) + I --> D + D -->|time synced msg| J(Set time_synced = msg.payload) + J --> D + D -->|thermal meta msg| K(Track whether zone0 is present) + K --> D + D -->|scope cancelled| En[Stop] +``` + +## Architecture + +- Two long-running fibers: `System Main` (config, alarms, shutdown) and `System Sysinfo` (metrics loop). +- A channel carries `report_period` updates from the main fiber to the sysinfo fiber. The sysinfo fiber blocks on `channel:get()` before its first loop iteration, waiting for config. +- Platform static identity fields are read once from the retained `platform` state topic at sysinfo fiber startup. Each field is immediately published as an `obs/v1/system/metric/` metric. They are not re-published on each report cycle. +- USB3 handling: if `usb3_enabled = false` in config, call `cap/usb/usb3/rpc/disable`. If `usb3_enabled = true`, call `cap/usb/usb3/rpc/enable`. This is idempotent (calling disable when already disabled is harmless). +- The sysinfo fiber does **not** fail if any individual `get` RPC fails — it logs a warning and skips publishing that metric. +- The sysinfo fiber subscribes to `{'state', 'time', 'synced'}` and maintains a local `time_synced` flag. `boot_time` (`os.time() - uptime`) is only computed and published when `time_synced = true`, ensuring the wall-clock value is correct before it enters the metrics pipeline. +- `finally` blocks in both fibers log the reason for shutdown. +- Thermal zone subscription tracks whether `zone0` is present. Temperature is only read and published when `zone0` is known. No per-zone metric publications are made — a single `temperature` metric name is preserved to maintain continuity with historically collected data. diff --git a/docs/specs/time.md b/docs/specs/time.md new file mode 100644 index 00000000..13d851b7 --- /dev/null +++ b/docs/specs/time.md @@ -0,0 +1,76 @@ +# Time Service + +## Description + +The time service listens to time capabilities for sync and unsync events. It uses the events to sync and unsync the alarm module of fibers and broadcast sync/unsync events to the bus for other services to listen to. + +## Time capability + +There is initially only one time capability, provided by the time driver in HAL. In the future we can discover multiple time capabilities and balance multiple sources of time intelligently (e.g. prefer the capability reporting the lowest stratum). + +For now, the time service subscribes to `{'cap', 'time', '+', 'meta', 'source'}` and uses the **first** capability announced. Subsequent announcements are ignored. + +## Bus Outputs + +### Service status (retained) + +Topic: `{'svc', 'time', 'status'}` + +```lua +{ + state = 'starting' | 'running' | 'stopped', + ts = , +} +``` + +### Sync state (retained) + +Topic: `{'state', 'time', 'synced'}` + +Payload: `true` (synced) or `false` (unsynced). + +This is the canonical retained domain truth for time-sync state. Services that need to gate behaviour on NTP sync (e.g. alarm management, boot-time metric publication) subscribe here. + +### Time transition events (non-retained) + +Topics: +- `{'obs', 'v1', 'time', 'event', 'synced'}` +- `{'obs', 'v1', 'time', 'event', 'unsynced'}` + +Published on state transitions for consumers that need edge-triggered behaviour. + +## Service Flow + +```mermaid +flowchart TD + St[Start] --> B(Publish status: starting) + B --> C(Subscribe to cap/time/+/meta/source) + C --> D(Publish status: running) + D --> E{Wait for first time capability meta message or scope done} + E -->|scope done| Z[Publish status: stopped] + E -->|first capability meta received| F(Extract uuid from meta topic) + F --> G(Subscribe to cap/time/uuid/state/synced\ncap/time/uuid/event/synced\ncap/time/uuid/event/unsynced) + G --> H(Read single retained state message and apply sync state) + H --> H2(Unsubscribe from state/synced) + H2 --> K{Wait for synced event, unsynced event, or scope done} + K -->|synced event| L(Apply synced: retain state/time/synced=true, emit obs/v1/time/event/synced) + L --> K + K -->|unsynced event| M(Apply unsynced: retain state/time/synced=false, emit obs/v1/time/event/unsynced) + M --> K + K -->|scope done| Z +``` + +All three subscriptions are created before any message is read, so no events are lost during initialisation. The retained `{'cap', 'time', , 'state', 'synced'}` payload is consumed as a one-shot read to bootstrap sync state, then the state subscription is dropped. Ongoing sync state changes are tracked exclusively through the `event/synced` and `event/unsynced` transition topics. + +With the new fibers alarm API, the service calls: +- `alarm.set_time_source(fibers.utils.time.realtime)` on first synced state +- `alarm.time_changed()` on subsequent synced transitions/events + +There is no direct equivalent of `clock_desynced` in the new API; unsynced updates still propagate over bus outputs. + +## Architecture + +- Everything runs in a single fiber — no child fibers needed. The fiber blocks waiting for the first capability, then transitions directly into the event loop for that capability. +- The service does not interact with the OS directly — all time source information arrives through the capability published by the time driver in HAL. +- Use `finally` to log shutdown reason and publish `stopped` status. + diff --git a/docs/specs/ui.md b/docs/specs/ui.md new file mode 100644 index 00000000..7d324272 --- /dev/null +++ b/docs/specs/ui.md @@ -0,0 +1,163 @@ +# UI service guide + +## Purpose + +UI owns operator-facing application semantics. + +It is responsible for: + +```text +static UI route policy +read-model queries +server-sent events +local sessions +authentication policy +firmware upload policy +update/artifact manager handoff +UI summary publication +``` + +UI does not own HTTP transport. It consumes the HTTP service. + +## Lifetime shape + +```mermaid +flowchart TD + UiScope["ui service scope"] --> Coordinator["coordinator"] + UiScope --> ReadModel["read-model component"] + UiScope --> Sessions["session store"] + UiScope --> Listener["HTTP listener consumer"] + UiScope --> Publisher["UI publisher"] + + Listener --> HttpCap["cap/http/main/rpc/listen"] + Listener --> RequestScopes["HTTP request scopes"] + + RequestScopes --> Static["static response"] + RequestScopes --> SSE["SSE stream"] + RequestScopes --> Upload["firmware upload"] + RequestScopes --> Query["read-model query"] + RequestScopes --> SessionOps["login/session/logout"] + + Upload --> Ingest["cap/artifact-ingest/main"] + Upload --> Update["cap/update-manager/main"] + + ReadModel --> State["retained service state watches"] + Publisher --> UiState["state/ui/..."] +``` + +A request scope owns one accepted HTTP context after listener handoff. + +## Public surfaces + +UI should publish: + +```text +svc/ui/status +svc/ui/meta +cfg/ui + +state/ui/summary +state/ui/read-model +state/ui/sessions +``` + +The UI service may publish narrow UI-domain retained state. It must not become the naming boundary for workflows owned by Update or Artifact Ingest. + +Firmware upload should call: + +```text +cap/artifact-ingest/main/rpc/create +cap/artifact-ingest/main/rpc/append +cap/artifact-ingest/main/rpc/commit +cap/artifact-ingest/main/rpc/abort + +cap/update-manager/main/rpc/create-job +cap/update-manager/main/rpc/start-job +``` + +UI must not expose upload workflows as `cmd/ui/...` or `cmd/update/...`. + +## HTTP request ownership + +```mermaid +sequenceDiagram + participant HTTP as HTTP service + participant UI as UI listener component + participant Req as UI request scope + participant Ctx as HttpContext + + HTTP-->>UI: accepted context handle + UI->>Req: start request scope + Req->>Ctx: install termination ownership + Req->>Ctx: read request op + Req->>Req: route request + Req->>Ctx: write response ops + Req->>Ctx: terminate on cancellation/finaliser +``` + +The UI coordinator should not perform HTTP I/O. Request scopes may perform HTTP reads and writes. + +## Read model + +The read model is observable state, not a worker that performs application policy. + +It should provide: + +```text +snapshot +query +watch +changed_op +terminate +``` + +Read-model updates should be non-yielding. Expensive work belongs in request scopes or workers. + +## Upload ownership + +Upload owns: + +```text +accepted HTTP context +request body streaming +artifact ingest session until commit or abort +timeout policy +update job creation/start request +``` + +UI request scopes own user-facing timeout policy. Reusable update and ingest clients must not install hidden bus timeouts; they should call update and ingest capabilities with `timeout = false` by default and let the UI request scope compose timeout/cancellation using Ops. + +On cancellation or append failure: + +```text +abort uncommitted ingest +terminate owned body/context resources +resolve request once +``` + +After successful commit: + +```text +do not abort committed artifact +create update job through cap/update-manager +start update job if policy requires it +``` + +## Reviewer checklist + +```text +UI does not require cqueues, lua-http or HTTP transport modules. +UI obtains HTTP through services.http.sdk. +Request scopes own HTTP I/O. +Upload uses cap/artifact-ingest and cap/update-manager. +UI update and ingest clients leave bus timeout policy to the owning request scope. +UI does not publish workflow records owned by Update. +UI state stays narrow under state/ui/... +Session changes are observable through the store/model, not callback sinks. +Finalisers terminate contexts and unresolved requests immediately. +``` + + +## HTTP listener dependency + +A configured UI HTTP listener depends on `cap/http//status`. Missing HTTP status, `available=false`, `no_route`, or `http_backend_not_ready` are admission failures. The UI read model and session machinery may continue running while listener work is `waiting_for_http`. diff --git a/docs/specs/update.md b/docs/specs/update.md new file mode 100644 index 00000000..7cea4d73 --- /dev/null +++ b/docs/specs/update.md @@ -0,0 +1,157 @@ +# Update service guide + +## Purpose + +Update owns firmware update workflows, artifact ingestion and update-domain summaries. + +It is responsible for: + +```text +durable update jobs +artifact ingest workflows +single-active-update admission +component stage/commit/reconcile work +restart adoption +bundled-image policy +update-domain summaries +workflow records +public update manager and ingest capabilities +``` + +Update must remain responsive while storage, artifact handling, component operations or restart reconciliation are slow. + +## Lifetime shape + +```mermaid +flowchart TD + UpdateScope["update service scope"] --> Coordinator["service coordinator"] + UpdateScope --> JobRuntime["job runtime"] + UpdateScope --> ActiveRuntime["active runtime"] + UpdateScope --> Model["update model"] + UpdateScope --> Publisher["publisher"] + UpdateScope --> Generation["generation scope"] + + Generation --> Manager["manager request scopes"] + Generation --> Ingest["artifact ingest instance scopes"] + Generation --> Bundled["bundled policy coordinator"] + Generation --> Observers["component observers"] + + Manager --> JobRuntime + Manager --> ActiveRuntime + Ingest --> Workflow["state/workflow/artifact-ingest/"] + + ActiveRuntime --> Stage["stage operation scope"] + ActiveRuntime --> Commit["commit operation scope"] + ActiveRuntime --> Reconcile["reconcile operation scope"] + + Publisher --> Summary["state/update/summary"] + Publisher --> Jobs["state/workflow/update-job/"] +``` + +The key split is: + +```text +generation admits and routes +job_runtime owns durable authority +active_runtime owns active execution +publisher projects public retained state +``` + +Generation replacement must not cancel already accepted durable active work. + +## Public surfaces + +Update should publish: + +```text +svc/update/status +svc/update/meta +cfg/update + +cap/update-manager/main/meta +cap/update-manager/main/status +cap/update-manager/main/rpc/create-job +cap/update-manager/main/rpc/start-job +cap/update-manager/main/rpc/commit-job +cap/update-manager/main/rpc/cancel-job +cap/update-manager/main/rpc/retry-job +cap/update-manager/main/rpc/discard-job + +cap/artifact-ingest/main/meta +cap/artifact-ingest/main/status +cap/artifact-ingest/main/rpc/create +cap/artifact-ingest/main/rpc/append +cap/artifact-ingest/main/rpc/commit +cap/artifact-ingest/main/rpc/abort + +state/update/summary +state/update/component/ +state/workflow/update-job/ +state/workflow/artifact-ingest/ +``` + +Do not expose public update workflows under `cmd/update/...`. + +## Durable and active split + +```mermaid +sequenceDiagram + participant Caller + participant Manager as cap/update-manager + participant Gen as generation + participant Job as job_runtime + participant Active as active_runtime + participant Pub as publisher + + Caller->>Manager: start-job + Manager->>Gen: request event + Gen->>Job: admit durable transition + Job-->>Manager: durable admission result + Job->>Active: active intent + Active->>Active: stage/commit/reconcile worker + Active-->>Job: stored completion fact + Job->>Pub: model changed + Pub->>Caller: retained state visible +``` + +Manager requests may wait for durable admission. They must not mutate durable state directly. + +Manager requests are caller-owned until admitted. The manager should create a `request_owner` at admission and pass `owner:caller_cancel_op()` to any scoped request work. If the caller abandons the bus request before durable admission completes, the request scope is cancelled and late completion is local only. + +Durable admission is a boundary. Once an update job transition has been durably accepted, caller abandonment stops waiting for the reply but does not imply rollback of the durable fact unless the operation explicitly promises rollback. + +Active workers report facts. They do not directly update durable job records. + +## Artifact ownership + +Artifact ingest is a workflow. + +Rules: + +```text +uncommitted sink belongs to ingest instance +append operations are serialised per instance +commit transfers sink/artifact ownership durably +abort terminates uncommitted resources immediately +finalisers abandon unresolved requests and terminate uncommitted resources +queued ingest requests abandoned before active admission are skipped +active ingest requests observe caller abandonment through owner:caller_cancel_op() +``` + +## Reviewer checklist + +```text +No public cmd/update/... paths. +Manager calls are under cap/update-manager/... +Manager request work observes caller abandonment through owner:caller_cancel_op(). +Ingest calls are under cap/artifact-ingest/... +Ingest append/commit/abort work observes caller abandonment through owner:caller_cancel_op(). +Jobs are retained under state/workflow/update-job/. +Ingest records are retained under state/workflow/artifact-ingest/. +Update summaries are under state/update/... +job_runtime owns durable authority. +active_runtime is service-owned, not generation-owned. +Completions are stored before reporting. +Generation replacement does not cancel accepted active work. +Workers report facts rather than mutating durable state directly. +``` diff --git a/docs/specs/wifi.md b/docs/specs/wifi.md new file mode 100644 index 00000000..dd5d3225 --- /dev/null +++ b/docs/specs/wifi.md @@ -0,0 +1,235 @@ +# Wifi Service + +## Description + +The Wifi Service is an application-layer service responsible for: + +1. **Radio configuration** — forwarding per-radio settings (channel, txpower, country, enabled state) to HAL radio capabilities. +2. **SSID management** — configuring wireless interfaces (SSIDs) on each radio, sourcing SSID credentials from `mainflux.json` via the `configs` filesystem capability when a `mainflux_path` is present in the config. +3. **Band steering configuration** — forwarding band steering parameters to the HAL `band` capability (DAWN daemon). +4. **Stats forwarding** — subscribing to `cap/radio//state|event` topics emitted by radio capabilities and forwarding them to the observability bus. +5. **Session management** — tracking client association/disassociation events, hashing MAC addresses, and emitting session start/end records. +6. **Capability lifecycle tracking** — starting and stopping radio configuration when radio capabilities come and go. + +The service interacts with hardware exclusively through HAL capabilities on the bus. No direct file reads or command execution are performed by the service itself. + +## Dependencies + +HAL capabilities consumed: + +| Capability class | Id | Usage | +|------------------|------------------------------|----------------------------------------------------------| +| `radio` | per radio (e.g. `'radio0'`) | Configure channels, txpower, country, SSIDs; `apply` | +| `band` | `'1'` | Configure band steering (DAWN); `apply` | +| `filesystem` | `'configs'` | Read `mainflux.json` for SSID credential sourcing | + +Service topics consumed: + +| Topic | Usage | +|--------------------|-----------------------------------------------------------------------------| +| `{'cfg', 'wifi'}` | Retained config — the main fiber subscribes to this for all wifi settings | + +Cap topics subscribed for radio stats: + +| Topic | Usage | +|------------------------------------------|--------------------------------------------------------------------| +| `{'cap', 'radio', id, 'state', name}` | Named stat emitted by the radio driver; forwarded to obs bus | +| `{'cap', 'radio', id, 'event', name}` | Named client event emitted by the radio driver; used for sessions | + +## Configuration + +Received via retained bus message on `{'cfg', 'wifi'}`. The message uses the standard versioned config envelope: + +```lua +{ + rev = , -- config revision; service skips messages with rev <= last applied rev + data = { + schema = "devicecode.config/wifi/1", + report_period = , -- required: stats publish interval in seconds + radios = { -- required: array of per-radio configs + { + name = , -- required: radio section name, e.g. "radio0" + band = , -- required: "2g" or "5g" + channel = , -- required: channel number or "auto" + htmode = , -- required: e.g. "HE80", "VHT80", "HT40+" + channels = , -- required when channel == "auto": list of allowed channels + txpower = , -- optional: transmit power + country = , -- optional: 2-letter ISO country code + disabled = , -- optional: if true, radio is administratively disabled + }, + ... + }, + ssids = { -- required: array of SSID configs + { + name = , -- required: SSID string + mode = , -- required: "access_point"|"client"|"adhoc"|"mesh"|"monitor" + radios =
, -- required: list of radio names this SSID applies to + segment = , -- required: NET segment id for this SSID attachment + mainflux_path = , -- optional: path within the configs filesystem cap to the mainflux credential file (e.g. "mainflux.json") + encryption = , -- optional: default "none" + password = , -- optional + }, + ... + }, + band_steering = { -- required: band steering settings + globals = { + kicking = { + kick_mode = , -- required: "none"|"compare"|"absolute"|"both" + bandwidth_threshold = , -- required + kicking_threshold = , -- required + evals_before_kick = , -- required + }, + stations = { -- optional + use_station_count = , + max_station_diff = , + }, + rrm_mode = , -- optional + neighbor_reports = { -- optional + dyn_report_num = , + disassoc_report_len = , + }, + legacy = , -- optional: key-value legacy options + }, + timings = { + updates = { -- required: update frequencies in seconds + client = , + chan_util = , + hostapd = , + }, + inactive_client_kickoff = , -- required + cleanup = { -- required + client = , + probe = , + ap = , + }, + }, + bands = { -- required: per-band scoring config + ["2G"] = { + initial_score = , + rssi_scoring = { ... }, -- optional + chan_util_scoring = { ... }, -- optional + support_bonuses = { ... }, -- optional + }, + ["5G"] = { ... }, -- same structure + }, + }, + }, +} +``` + +> **Underflow note:** JSON decoding may silently convert negative numbers to large positive integers due to integer underflow. The service must check decoded numeric fields that are expected to be non-negative and, if the value exceeds a sanity threshold (e.g. > 2^31), treat it as if the original value were negative by subtracting 2^32. + +## SSID Sourcing Policy + +When an SSID entry contains `mainflux_path`, the service reads that file from the `configs` filesystem capability: + +```lua +cap_ref:call_control('read', FilesystemReadOpts(mainflux_path)) +``` + +The file is expected to contain a JSON object with a `networks.networks` array holding credential payloads. Each entry is mapped into an SSID record. The Wi-Fi service attaches the resulting SSID to the configured `segment`; it does not use an OpenWrt `network` field in service configuration. + +**If the file read fails or the content cannot be decoded, no SSIDs are configured for that entry at all.** No hardcoded fallback SSIDs are produced. This is an intentional strict policy — partial SSID configuration is considered worse than no configuration. + +When `mainflux_path` is absent, the SSID entry is applied directly using its `name`, `segment`, and other fields. `network` is not a supported Wi-Fi service configuration field. + +## Radio Capability Lifecycle + +The service discovers radio capabilities using `cap_sdk.new_cap_listener(conn, 'radio')`. When a radio capability is added: + +- The service applies the matching radio config (by name) from the stored `data.radios` config. +- It configures SSIDs that reference that radio, sourcing credentials from the `configs` filesystem cap via `mainflux_path` when present. +- It starts the stats forwarding and session tracking loop, subscribed to `{'cap', 'radio', id, 'state', '+'}` and `{'cap', 'radio', id, 'event', '+'}`. + +When a radio capability is removed, the service cancels its associated child scope, stopping all loops and releasing subscriptions. + +The band capability (`class = 'band'`, `id = '1'`) is discovered independently. Band steering configuration is applied whenever both the band capability is available and a valid config has been received. + +## Radio Configuration Sequence + +For each radio capability added, the service invokes RPCs via `cap_ref:call_control(method, args)` in the following order: + +1. `clear_radio_config` — resets the driver's staged radio config to base state. +2. `set_report_period` — sets the stats emit interval. +3. `set_channels` — sets band, channel, htmode, and optional auto-channels list. +4. `set_txpower` — if present in config. +5. `set_country` — if present in config. +6. `set_enabled` — if `disabled` is present in config. +7. `add_interface` (repeated per SSID) — stages each SSID as a wireless interface. For SSIDs with `mainflux_path`, credentials are fetched from the `configs` filesystem cap before this call. +8. `apply` — tells the driver to commit staged config and reload wireless. + +Rollback is not called by the service on error; the radio driver's staged state is simply abandoned. + +## Session Management + +Client association and disassociation events arrive on `{'cap', 'radio', id, 'event'}`. The wifi service is responsible for: + +1. **MAC hashing** — raw MAC addresses are never forwarded to the observability bus. Each MAC is hashed using `gen.userid(mac)` to produce a stable, anonymous user identifier. +2. **Session tracking** — on association, a new session ID is generated via `gen.gen_session_id()` and a `session_start` record is emitted. On disassociation, a `session_end` record is emitted with the same session ID. +3. **Published session events** — emitted to the specific metric topic for the event type: `{'obs', 'v1', 'wifi', 'metric', 'session_start'}` and `{'obs', 'v1', 'wifi', 'metric', 'session_end'}`. Each record includes fields `user_id`, `session_id`, `radio`, and `timestamp`. + +The radio driver itself has no knowledge of sessions or MAC hashing — it emits raw events and the wifi service applies the anonymisation and session logic. + +## Published Topics + +All stats and events are forwarded to `obs/v1/wifi/metric/` topics, matching the metrics collection keys used in device configs. + +| Topic | Content | +|-----------------------------------------------------|-----------------------------------------------------------------| +| `{'svc', 'wifi', 'status'}` | Service status (via `service_base`) | +| `{'obs', 'v1', 'wifi', 'metric', }` | Named stat or event from a radio (e.g. `num_sta`, `iface_rx_bytes`, `client_signal`) | +| `{'obs', 'v1', 'wifi', 'metric', 'session_start'}` | Anonymised session start record (MAC hashed to `user_id`) | +| `{'obs', 'v1', 'wifi', 'metric', 'session_end'}` | Anonymised session end record (MAC hashed to `user_id`) | + +## Service Flow + +```mermaid +flowchart TD + St[Start] --> A(service_base.new: publish status) + A --> B(Subscribe to cfg/wifi) + B --> C(Start cap_listener for radio + band) + C --> D{named_choice: cfg msg, radio added, radio removed, band added, scope cancelled} + D -->|cfg msg| E(Validate config rev+schema; store configs) + E --> F(Re-apply config to all known radios) + F --> G(If band cap known: apply band steering) + G --> D + D -->|radio added| H(Spawn child scope for radio) + H --> I(Apply radio config sequence) + I --> J(Fetch mainflux_path from configs fs cap if present; add_interface per SSID) + J --> K(apply) + K --> L(Start stats + session forwarding loops) + L --> D + D -->|radio removed| M(Cancel child scope; stop loops) + M --> D + D -->|band added| N(Apply band steering if config received) + N --> D + D -->|scope cancelled| En[Stop] +``` + +### Stats forwarding loop (per radio) + +Runs in child scope, subscribes to `{'cap', 'radio', id, 'state', '+'}` and `{'cap', 'radio', id, 'event', '+'}`: + +1. On `state/` message: forward payload to `{'obs', 'v1', 'wifi', 'metric', name}`. +2. On `event/client_event` message (association): + - Hash MAC via `gen.userid(mac)` → `user_id`. + - Generate session ID via `gen.gen_session_id()` → `session_id`. + - Store `(mac → session_id)` in per-radio session table. + - Publish `session_start` record to `{'obs', 'v1', 'wifi', 'metric', 'session_start'}`. +3. On `event/client_event` message (disassociation): + - Look up `session_id` from per-radio session table. + - Publish `session_end` record to `{'obs', 'v1', 'wifi', 'metric', 'session_end'}`. + - Remove entry from session table. +4. On other `event/` messages: forward payload to `{'obs', 'v1', 'wifi', 'metric', name}`. +5. On scope cancellation: exit loop. + +## Architecture + +- The service uses `service_base.new(conn, opts)` for `svc:status`, `svc:obs_log`, and `svc:obs_event`. +- Radio caps and the band cap are tracked independently. A config update always attempts to re-apply to all currently known capabilities. +- Each radio capability runs its lifecycle in a child scope of the main service scope. Cancelling the child scope stops the stats/session forwarding loops and releases all subscriptions for that radio. +- SSID configuration is always idempotent: each full config update clears and rebuilds all interfaces from scratch. +- Mainflux credentials are read from the `configs` filesystem capability via `cap_ref:call_control('read', FilesystemReadOpts(mainflux_path))`. No bus topic subscription is used for mainflux data. On read failure, the SSID entry is skipped entirely. +- `band_steering.data.globals.kicking.kick_mode` determines whether `has_band_steering = true` is set on each SSID. When `kick_mode == "none"`, band steering flags are not added to the wifi-iface entries. +- Config version gating: the service checks `msg.rev` on each `cfg/wifi` update and skips messages with `rev <= last_rev` to avoid re-applying stale retained messages. +- A `finally` block on the service scope logs the shutdown reason. diff --git a/docs/specs/wired.md b/docs/specs/wired.md new file mode 100644 index 00000000..8703ade5 --- /dev/null +++ b/docs/specs/wired.md @@ -0,0 +1,313 @@ +# Wired service specification + +This document describes the `wired` service as implemented in this tree. + +`wired` is the appliance authority for wired physical surfaces. It attaches physical wired surfaces to `net` segments, validates observed source capabilities and publishes appliance-level wired state. It does not define segments, VLAN allocation, addressing, DHCP, DNS, routing, firewall, WAN, VPN or shaping policy. + +## Current scope + +The current implementation is Phase 1 for Big Box wired composition. + +It supports: + +```text +CM5 Ethernet surface + protected appliance surface `cm5-eth0` + represented by a static HAL wired provider + +RTL8380M switch uplink surface + protected appliance surface `switch-uplink-cm5` + represented by a read-only RTL8380M HTTP HAL wired provider + +observation composition + raw wired observations under raw/host/wired/provider/... + Device physical assembly under state/device/assembly + appliance-level surface projection under state/wired/... + +protected trunk validation + required system segments must be carried + realised user segments may be required through all-realised-user-segments + source availability and observed-surface capabilities are checked +``` + +Phase 1 does not apply switch configuration. The RTL8380M HTTP provider is read-only observation. Control operations should report read-only or unsupported rather than pretending to apply. + +## Authority split + +```text +net + segment identity, VLAN ids, addressing, DNS/DHCP, firewall, routing, + WAN, VPN and shaping policy + +wired + appliance wired surfaces, access/trunk attachment, observed-source capability + validation, protected trunk invariants, wired topology and violations + +device + appliance component composition and physical product assembly + +HAL/provider + static wired facts, RTL8380M HTTP observation, future switch APIs, + OpenWrt switch work or fabric-member calls +``` + +Rules: + +```text +wired consumes state/net/segments. +wired consumes state/device/assembly. +wired consumes raw wired observations. +wired publishes state/wired/.... +wired does not expose provider-shaped public capabilities. +wired does not define network policy. +``` + +## Current source layout + +```text +src/services/wired/config.lua +src/services/wired/model.lua +src/services/wired/projection.lua +src/services/wired/publisher.lua +src/services/wired/service.lua +src/services/wired/topics.lua + +src/services/hal/backends/wired/contract.lua +src/services/hal/backends/wired/provider.lua +src/services/hal/backends/wired/providers/static.lua +src/services/hal/backends/wired/providers/rtl8380m_http.lua +``` + +`wired` is a house-style service. Its coordinator composes retained state and raw wired observations. It does not perform OS or switch work inline. + +## Configuration shape + +`src/services/wired/config.lua` accepts only: + +```text +devicecode.config/wired/1 +``` + +and normalises it into: + +```text +devicecode.wired.intent/1 +``` + +The current Big Box config defines protected internal trunks and the fixed external switch surface inventory: + +```text +cm5-eth0 + direct-nic + internal-trunk + backing surface supplied by state/device/assembly + required segments adm, int + user_segments all-realised-user-segments + +switch-uplink-cm5 + switch-port + internal-trunk + backed by RTL8380M GE8 via state/device/assembly + required segments adm, int + user_segments all-realised-user-segments + +lan-1 .. lan-7 + external RJ45 surfaces + backed by RTL8380M GE1 .. GE7 via state/device/assembly + semantic attachment not yet assigned in cfg/wired + +sfp-1 .. sfp-2 + external SFP surfaces + backed by RTL8380M GE9 .. GE10 via state/device/assembly + semantic attachment not yet assigned in cfg/wired +``` + +## Raw wired observations + +HAL wired providers publish observation-shaped facts below the public Wired +contract. For a local source, observations are retained under: + +```text +raw/host/wired/provider//status +raw/host/wired/provider//state/identity +raw/host/wired/provider//state/runtime +raw/host/wired/provider//state/power +raw/host/wired/provider//state/surfaces +raw/host/wired/provider//state/topology +``` + +A surface observation is keyed by the source component's observed surface id, for example +`GE8` on the RTL8380M switch. Raw observations may carry diagnostic +provenance fields such as `provider_surface_id`, but those names are not part of +the public semantic Wired contract. + +`wired` combines these observations with `state/device/assembly`, where the +one true backing vocabulary is `component` plus `observed_surface`. No caller +above HAL should know manufacturer URL paths, cookies, forms, switch CLI syntax +or ASIC register names. + +## Protected trunk invariants + +`wired` enforces product-level safety around protected internal trunks. + +Protected surfaces must: + +```text +be enabled +be configured as trunk attachments +name required system segments +have an available source component +have an available observed source surface +be backed by an observed source surface capable of trunk operation +carry the VLAN ids for all required segments +carry all realised user segments when configured to do so +``` + +If an invariant is broken, `wired` publishes a violation rather than modifying network policy. + +Example violation kinds include: + +```text +protected_surface_disabled +protected_surface_not_trunk +protected_source_missing +protected_source_unavailable +protected_observed_surface_missing +observed_surface_does_not_support_trunk +missing_required_segment_definition +missing_required_segment_vlan +missing_required_segment_carriage +missing_user_segment_carriage +unknown_segment +``` + +The Big Box broken-trunk VM test proves that a provider can be physically available while the appliance is still degraded because required VLAN carriage is wrong. + +## Retained state + +`wired` currently publishes: + +```text +state/wired/summary +state/wired/surface/ +state/wired/topology +state/wired/violations +``` + +It consumes: + +```text +cfg/wired +state/net/segments +state/device/assembly +raw/host/wired/provider/# +``` + +The service projects stable appliance surface ids. A UI should display `lan-1`, `cm5-eth0` or `switch-uplink-cm5`, not raw provider internals. + +## Phase plan + +### Phase 1: one-way observation + +Current state. + +```text +RTL8380M manufacturer firmware +read-only HTTP observation provider +wired composes and validates appliance surfaces +wired publishes violations and topology +no switch configuration is applied +``` + +### Phase 2: controlled provider + +The same provider family may become writable, but `wired` remains the owner of physical attachment policy and the provider remains the implementation boundary. + +Expected operations: + +```text +snapshot_op +watch_op +apply_attachments_op +set_poe_op +bounce_op +``` + +Apply work must be scoped and stale-safe: + +```text +configuration or request event + -> coordinator validates and records desired attachment state + -> coordinator starts scoped apply work + -> worker calls provider operation + -> worker reports wired_apply_done + -> coordinator stale-checks generation and apply_id + -> model updates and publishes +``` + +Even when writable, the provider must not allow the protected CM5-to-switch trunk to be cut. + +### Phase 3: switch fabric as a Devicecode member + +When the RTL8380M runs OpenWrt and Devicecode, it may publish local `state/wired/...` and `cap/wired/...` through `fabric`. + +The CM5-side flow should be: + +```text +fabric imports switch member state + -> device describes switch-main in the physical assembly + -> wired consumes assembly plus switch observations + -> ui consumes state/wired/... +``` + +`wired` should not consume raw member topics directly. `device` owns appliance component and capability promotion. + +The preferred final Big Box model is: + +```text +CM5 net + remains the appliance segment and VLAN authority + +switch-local wired + owns switch-local ports and ASIC implementation + +CM5 wired + composes appliance-level surfaces from promoted capabilities +``` + +## Current tests + +Relevant coverage includes: + +```text +unit wired config validation +unit static wired provider validation +Big Box phase-one composition +Big Box broken-trunk degradation +OpenWrt VM Big Box composition and broken trunk checks +``` + +## Current limits + +```text +No writable switch application is implemented in Phase 1. +User-facing switch surfaces are present as product inventory; their final access/trunk segment policies remain a later product decision. +PoE policy is modelled as capability information but not yet controlled. +Raw observations are semantic but deliberately minimal for the current phase. +``` + +## Design north star + +`wired` realises the wired fabric of the appliance without claiming network policy. + +```text +net owns network meaning. +wired owns physical wired attachment. +HAL/provider code owns implementation-specific observation. +ui consumes appliance-level state. +``` + + +## Provider dependency projection + +Raw wired observations are tracked as input facts. Source absence is projected as unavailable/degraded surface state and violations; it is not a service startup failure. diff --git a/docs/switch.md b/docs/switch.md new file mode 100644 index 00000000..d4a34a93 --- /dev/null +++ b/docs/switch.md @@ -0,0 +1,428 @@ +# Big Box RTL8380M switch integration + +This document records the useful results from the Big Box PoE/VLAN switch discovery work and the current architecture for using those results. It is intended to avoid repeating the UI/API exploration when the read-only driver is extended into a controlled switch driver. + +The explored switch was first discovered at `http://192.168.1.1/` with the factory `admin` account. In Big Box it is addressed on the internal management segment at `http://172.28.100.9/`. The current configuration uses `switch-main` as the raw observation id and Device component id. + +## Current architectural boundary + +The switch is currently an attached local hardware element observed by the CM5 over HTTP. It is not yet a fabric peer. The current boundary is therefore: + +```text +RTL8380M manufacturer HTTP UI/API + -> services.hal.backends.wired.providers.rtl8380m_http + -> raw/host/wired/provider/switch-main/... + + state/device/assembly + -> services.wired + -> state/wired/... +``` + +The public seam is `state/wired/...`. There is no public `cap/wired-provider` seam. Provider-shaped data remains raw input or diagnostic provenance only; the public seam is source/component terminology. + +The intended future boundary, once the switch itself runs OpenWrt, devicecode and fabric, is: + +```text +switch fabric peer observations + + state/device/assembly + -> services.wired + -> state/wired/... +``` + +`services.net` should remain unchanged across that transition. It consumes semantic wired state, not the RTL8380M driver, raw switch topics or manufacturer port names. + +## Big Box physical map + +The CM5's single Ethernet port is connected to the RTL8380M switch on `GE8`. + +The current product assembly uses: + +```text +cm5-local-wired / eth0 <-> switch-main / GE8 +``` + +Stable product surfaces are mapped by Device assembly: + +```text +cm5-eth0 -> cm5-local-wired / eth0 +switch-uplink-cm5 -> switch-main / GE8 +lan-1 -> switch-main / GE1 +lan-2 -> switch-main / GE2 +lan-3 -> switch-main / GE3 +lan-4 -> switch-main / GE4 +lan-5 -> switch-main / GE5 +lan-6 -> switch-main / GE6 +lan-7 -> switch-main / GE7 +sfp-1 -> switch-main / GE9 +sfp-2 -> switch-main / GE10 +``` + +`GE1` to `GE8` are copper ports and are the PoE-capable surfaces. `GE9` and `GE10` are fibre/SFP surfaces and do not appear in the PoE port list. + +`cfg/wired` must not repeat this physical mapping. It describes semantic wired intent for product surfaces: protected trunks, access/trunk mode, required segments, user-segment expansion, PoE expectations and related policy. Device assembly says what backs a surface; Wired says what that surface means. + +## Driver status + +The current driver is: + +```text +src/services/hal/backends/wired/providers/rtl8380m_http.lua +``` + +It is deliberately read-only. It implements one-shot snapshot observation and returns `read_only` for configuration operations until the write forms have dedicated tests against the switch UI. + +The production HTTP path is through the `services.http` capability. The driver does not use `curl`, shell-out HTTP, raw sockets or direct bus access. The HAL wired manager obtains the narrowed HTTP dependency at the service boundary and passes an HTTP dependency factory to the provider. + +The only supported switch HTTP provider configuration spellings on this path are listed below. `base_url` must include the scheme and trailing slash: + +```lua +{ + provider = "rtl8380m_http", + base_url = "http://172.28.100.9/", + username = "$SWITCH_USERNAME", + password = "$SWITCH_PASSWORD", + timeout_s = 0.8, + poll = { + fast = { interval_s = 1.0, groups = { "panel", "poe", "counters" } }, + medium = { interval_s = 5.0, groups = { "vlan", "lldp" } }, + slow = { interval_s = 30.0, groups = { "identity", "runtime" } }, + }, + http = { + capability = "main", + response_parser = "legacy-http1-close", + max_response_bytes = 1024 * 1024, + }, +} +``` + +The HAL wired manager treats the providers-map key as the observation id and passes it to the backend as `opts.provider_id`. It must not be repeated inside provider configuration. Do not add compatibility aliases such as `id`, `url`, `user`, `pass`, `capability_id`, `http_id` or top-level parser settings. + +## HTTP and session behaviour + +The root page only redirects with JavaScript: + +```html +window.location.href="login.html?ver=" + fileVer; +``` + +The real browser entry points are `login.html` and, after authentication, `home.html?ver=`. + +The UI is a JavaScript application using CGI command endpoints: + +```text +GET /cgi/get.cgi?cmd= +POST /cgi/set.cgi?cmd= +``` + +Template paths often use `../cgi/...`, but the driver normalises commands to root-relative `/cgi/...`. + +### Legacy HTTP parser + +Ordinary HTML and JavaScript pages are readable with `lua-http`. Several CGI responses from this firmware are not accepted by the normal `lua-http` response parser. The production driver therefore asks the HTTP service for: + +```text +response_parser = "legacy-http1-close" +``` + +This parser is inside `services.http`, not the switch provider. It is opt-in through HTTP policy, bounded by timeout and maximum response size, limited to HTTP GET/POST/HEAD over `http://`, requires a valid HTTP status line, rejects unsupported transfer encodings, does not follow redirects, and does not expose raw sockets to callers. + +The strict HTTP path uses the normal `lua-http`/cqueues transport. The `legacy-http1-close` path cannot use the `lua-http` response parser because the switch CGI replies are malformed for that parser. It therefore uses the HTTP service's bounded legacy transport with Fibers non-blocking socket operations. It remains scheduled work inside the HTTP service capability and should not block the rest of the system, but it is not parsed by `lua-http`. + +HTTP policy must explicitly admit it: + +```lua +policy = { + allowed_response_parsers = { + strict = true, + ["legacy-http1-close"] = true, + }, + legacy_http1_close_max_response_bytes = 1024 * 1024, +} +``` + +### Cookies + +The only cookie consistently observed during exploration was: + +```text +cookie_language=defLang_en +``` + +The driver preserves `Set-Cookie` values and replays the cookie header. The authenticated session may be IP-bound or implicit in the switch firmware; no separate durable API token was identified. + +## Login flow + +The switch does not use HTTP Basic authentication. The UI performs an RSA-encrypted password login. + +Step 1: + +```text +GET /cgi/get.cgi?cmd=home_login +``` + +The response includes a public RSA modulus. + +Step 2: + +```text +POST /cgi/set.cgi?cmd=home_loginAuth +``` + +The browser sends an encrypted password generated with the modulus from `home_login`. The Lua driver uses `openssl` for this RSA operation. The successful crawler run identified the login method as `rtl8380-rsa`; `home_loginStatus` returned `status = "ok"` after authentication. + +## Read-side CGI commands + +The read-only snapshot uses these commands: + +```text +home_main +panel_info +sys_sysinfo +sys_cpumem +port_port +vlan_create +vlan_conf +vlan_port +vlan_membership +poe_poe +rmon_statistics +lldp_local +lldp_neighbor +``` + +The provider has narrow read groups for grouped polling. Surface-bearing groups include `home_main` so that rows can be attached to the canonical switch surface names (`GE1` ... `GE10`): + +```text +panel path: home_main, panel_info +identity path: sys_sysinfo +vlan path: home_main, vlan_create, vlan_conf, vlan_port, vlan_membership +poe path: home_main, poe_poe +lldp path: lldp_local, lldp_neighbor +runtime path: sys_cpumem +counters path: home_main, rmon_statistics +``` + +The poll plan is based on timings measured against the fixed RTL8380M switch on 192.168.1.1 using a retained admin session: + +```text +panel avg 0.303 s, max 0.344 s +vlan avg 0.363 s, max 0.389 s +poe avg 0.077 s, max 0.084 s +lldp avg 0.160 s, max 0.183 s +counters avg 0.203 s, max 0.257 s +runtime avg 2.085 s, max 2.089 s +full read avg 3.522 s, max 3.554 s, with observed timeout +``` + +A concurrent probe over `panel,poe,counters,runtime` improved wall-clock time only modestly, from 2.688 s sequential average to 2.309 s concurrent average, so the production poller remains grouped and sequential. + +The driver captures the full snapshot into normalised provider observations: + +```text +raw/host/wired/provider/switch-main/status +raw/host/wired/provider/switch-main/state/identity +raw/host/wired/provider/switch-main/state/runtime +raw/host/wired/provider/switch-main/state/power +raw/host/wired/provider/switch-main/state/surfaces +raw/host/wired/provider/switch-main/state/topology +``` + +If `include_raw = true` is set in a test, the snapshot also keeps the source command payloads for parser debugging. Full raw CGI bodies should not be promoted to public retained state by default. + +The HAL wired manager owns scheduling. For the RTL8380M provider, manager apply admits the provider and starts one owned provider runner; switch observation is not part of configuration admission. The Big Box poll plan is grouped: + +```text +fast, 1 Hz: panel, poe, counters +medium, 5 s: vlan, lldp +slow, 30 s: identity, runtime +``` + +There is one runner per provider, not one fibre per poll group. The runner lives in `services/hal/managers/wired/provider_runner.lua` and owns the backend object, request mailbox, switch session, observation cache and due-time schedule. Capability snapshot/control requests are sent to the runner mailbox, so the RTL8380M backend is touched only by the runner fibre. This gives serialisation by ownership rather than a lock or semaphore. + +Each runner cycle coalesces all due poll groups, calls the mandatory backend `observe_groups_op` once, and lets the backend de-duplicate shared CGI commands such as `home_main`. A saturated cycle schedules the next attempt from the finish time rather than trying to catch up, and applies a short minimum idle interval before another due cycle. Slow runtime reads can therefore degrade runtime status without creating overlapping switch sessions or a busy catch-up loop. + +Successful groups merge into the retained raw observation cache, so `state/surfaces` carries last-known link, PoE, counter and VLAN facts together. Group failures update provider status but leave the last good identity/runtime/power/surfaces/topology retained facts in place. + +The HAL wired manager emits raw provider facts on a changed-retained basis. A successful `panel` group can update `state/surfaces` without re-emitting unchanged identity/runtime/power/topology facts, and repeated identical provider statuses are suppressed. This keeps switch visibility in the provider status and semantic `state/wired/...` surfaces rather than turning the monitor into a per-request trace. + +Canonical observation names are deliberately strict. CPU and memory are published as `runtime.cpu` and `runtime.memory`; PoE device-level power and temperature are published as `power.poe`; port counters are published under each surface as `counters`. The switch path must not publish `telemetry.cpu`, `telemetry.mem`, `telemetry.poe`, or any compatibility topic for `state/telemetry`. + +## Snapshot shape + +The provider snapshot includes: + +```lua +{ + ok = true, + provider_id = "switch-main", + mode = "read_only", + writable = false, + status = { + state = "available", + available = true, + driver = "rtl8380m_http", + login = "confirmed", + }, + identity = { + model = "RTL8380", + hostname = "...", + mac = "...", + firmware = "...", + loader = "...", + management_ipv4 = "...", + }, + runtime = { + cpu = { utilisation_pct = ... }, + memory = { utilisation_pct = ... }, + }, + power = { + poe = { + total_power_mw = ..., + total_power_w = ..., + temperature_c = ..., + }, + }, + surfaces = { + GE1 = { + provider_surface_id = "GE1", + kind = "ethernet-port", + capabilities = { access = true, trunk = true, poe = true }, + link = { state = "up", speed_mbps = 1000, duplex = "full", media = "copper" }, + attachment = { mode = "trunk", pvid = 1, tagged = {...}, untagged = {...} }, + poe = { state = "off", enabled = true, ... }, + counters = { + rx = { bytes = ..., packets = ..., drops = ..., errors = ... }, + }, + }, + }, + topology = { + lldp_local = {...}, + lldp_neighbor = {...}, + }, +} +``` + +The `provider_surface_id` field is raw provider provenance. Device assembly and Wired use the canonical backing vocabulary `component` plus `observed_surface`. + +## VLAN model + +The UI exposes four useful read-side VLAN views: + +```text +vlan_create configured VLAN list +vlan_conf per-selected-VLAN membership for each port/LAG +vlan_port per-port mode, PVID, accepted-frame type and TPID +vlan_membership per-port admin/oper VLAN membership strings +``` + +The current default observed state is VLAN 1 untagged/PVID on all ports and LAGs. + +VLAN mode mapping: + +```text +0 -> hybrid +1 -> access +2 -> trunk +3 -> tunnel +``` + +Accepted frame type mapping: + +```text +0 -> all +1 -> tag_only +2 -> untag_only +``` + +Per-VLAN membership mapping: + +```text +0 -> excluded +2 -> tagged +3 -> untagged +``` + +Membership string examples: + +```text +1UP VLAN 1, untagged, PVID +8T VLAN 8, tagged +32F VLAN 32, forbidden +100U VLAN 100, untagged +``` + +The UI uses `4095P` as a placeholder when a hybrid port has no real PVID. Do not treat `4095P` as a real configured VLAN. + +## PoE model + +PoE is exposed through `poe_poe`. It includes eight ports, corresponding to `GE1` through `GE8`. It does not include `GE9` or `GE10`. + +Useful PoE fields include: + +```text +portEnable admin enabled +portStatus delivering/not delivering +portType detected class/type string +portPowerLimit configured power limit +portPriority priority, where exposed +portLegacy legacy detection mode, where exposed +devPower source field mapped to power.poe.total_power_mw / total_power_w +devTemp source field mapped to power.poe.temperature_c +``` + +The driver normalises PoE-capable surfaces with `capabilities.poe = true` and includes a `poe` table for those surfaces. + +## Control endpoints discovered but not yet enabled + +The UI JavaScript exposes write commands under `set.cgi`, including login/logout/save and edit forms for system, ports, VLAN, PoE, LLDP and related configuration. The current production driver must not submit these until each operation has a test covering the exact payload shape and required persistence behaviour. + +Known write categories include: + +```text +home_loginAuth +home_save +home_logout +port_port +vlan_create +vlan_conf +vlan_port +poe_poe +poe_poeEdit +poe_poeTimer +sys_sysinfo +lldp_* edit commands +``` + +Likely control questions still to resolve: + +```text +whether every write requires home_save to persist across reboot +exact POST body for each VLAN and PoE write +whether port/LAG indexing is zero- or one-based for all edit forms +whether firmware 1.0.0.6 is representative of production units +how failed validation is reported by set.cgi +``` + +## Test coverage + +The real-switch tests are opt-in and environment gated: + +```sh +cd tests +SWITCH_TEST_BASE_URL=http://192.168.1.1/ \ +SWITCH_TEST_USERNAME=admin \ +SWITCH_TEST_PASSWORD=admin \ +TEST_FILTER=rtl8380m_real_switch \ +lua run.lua +``` + +They prove: + +```text +the HTTP capability and legacy parser can snapshot the real switch +login succeeds through the RSA flow +GE8 is present as the Big Box internal switch uplink +GE9 and GE10 are fibre/SFP surfaces +raw switch observations project through Device assembly into state/wired +``` + +The OpenWrt VM tests cover the static equivalent of the same seam: Device assembly plus raw wired observations are transformed into semantic Wired state, and Net remains above that boundary. diff --git a/src/cache.lua b/src/cache.lua deleted file mode 100644 index c2761fd2..00000000 --- a/src/cache.lua +++ /dev/null @@ -1,51 +0,0 @@ -Cache = {} -Cache.__index = Cache - --- Constructor -function Cache.new(default_timeout, custom_time_func, separator) - local self = setmetatable({}, Cache) - self.default_timeout = default_timeout or 10 - self.time_func = custom_time_func or os.time - self.separator = separator or string.char(31) - self.store = {} - return self -end - --- Utility function to check if a table is an array -local function is_array(table) - local i = 0 - for _ in pairs(table) do - i = i + 1 - if table[i] == nil then return false end - end - return true -end - --- Setting a value in the cache -function Cache:set(key, value, timeout) - timeout = timeout or self.default_timeout - if type(value) == 'table' then - if is_array(value) then - self.store[key] = {value=value, timestamp=self.time_func() + timeout} - else - for k, v in pairs(value) do - self:set(key .. self.separator .. k, v, timeout) - end - end - else - self.store[key] = {value = value, timestamp=self.time_func() + timeout} - end -end - --- Getting a value from the cache -function Cache:get(key, stale) - if stale == nil then stale = false end - key = type(key) == 'string' and key or table.concat(key, self.separator) - local item = self.store[key] - if item and (self.time_func() < item.timestamp or stale) then - return item.value - end - return nil -- or a default value -end - -return Cache diff --git a/src/configs/bigbox-ss.json b/src/configs/bigbox-ss.json deleted file mode 100644 index 78aabda5..00000000 --- a/src/configs/bigbox-ss.json +++ /dev/null @@ -1,1055 +0,0 @@ -{ - "hal": { - "managers": { - "modemcard": {}, - "uci": {}, - "ubus": {} - } - }, - "net": { - "report_period": 60, - "default": { - "backhauls": { - "network": { - "ipv4": { - "proto": "dhcp" - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - } - }, - "dns": { - "upstream_servers": [ - "8.8.8.8", - "1.1.1.1" - ], - "default_cache_size": 1000 - }, - "dhcp": { - "domains": [ - { - "name": "unifi", - "ip": "$UNIFI_IP" - }, - { - "name": "config.bigbox.home", - "ip": "172.28.8.1" - } - ] - }, - "network": [ - { - "name": "Loopback", - "id": "loopback", - "type": "local", - "interfaces": [ - "lo" - ], - "ipv4": { - "proto": "static", - "ip_address": "127.0.0.1", - "netmask": "255.0.0.0" - }, - "dns_server": { - "local_server": true, - "default_hosts": [] - } - }, - { - "name": "Admin Network", - "id": "adm", - "type": "local", - "is_bridge": true, - "interfaces": [ - "eth0.8" - ], - "ipv4": { - "proto": "static", - "ip_address": "172.28.8.1", - "netmask": "255.255.255.0" - }, - "dhcp_server": { - "range_skip": "10", - "range_extent": "240", - "lease_time": "12h" - }, - "dns_server": { - "local_server": true, - "default_hosts": [ - "ads" - ] - }, - "firewall": { - "zone": "lan" - } - }, - { - "name": "Jangala Network", - "id": "jan", - "type": "local", - "is_bridge": true, - "interfaces": [ - "eth0.32" - ], - "ipv4": { - "proto": "static", - "ip_address": "172.28.32.1", - "netmask": "255.255.255.0" - }, - "dhcp_server": { - "range_skip": "10", - "range_extent": "240", - "lease_time": "12h" - }, - "dns_server": { - "local_server": true, - "default_hosts": [ - "ads", - "adult" - ] - }, - "firewall": { - "zone": "lan_rst" - }, - "shaping": { - "egress": { - "qdisc": "htb", - "filters": [ - { - "id": "per_ip_filter", - "kind": "u32", - "hash_key": "dest_ip", - "target_class_template": "per_ip_shapers" - } - ], - "class_template": { - "id": "per_ip_shapers", - "kind": "per_dest_ip", - "classes": [ - { - "qdisc": "htb", - "config": { - "rate": "6mbit", - "ceil": "24mbit", - "burst": "1500k" - }, - "classes": [ - { - "qdisc": "fq_codel" - } - ] - } - ] - } - }, - "ingress": { - "qdisc": "htb", - "filters": [ - { - "id": "per_ip_filter", - "kind": "u32", - "hash_key": "src_ip", - "target_class_template": "per_ip_shapers" - } - ], - "class_template": { - "id": "per_ip_shapers", - "kind": "per_src_ip", - "classes": [ - { - "qdisc": "htb", - "config": { - "rate": "3mbit", - "ceil": "12mbit", - "burst": "750k" - }, - "classes": [ - { - "qdisc": "fq_codel" - } - ] - } - ] - } - } - } - }, - { - "name": "Internal Network", - "id": "int", - "type": "local", - "is_bridge": true, - "interfaces": [ - "eth0.100" - ], - "ipv4": { - "proto": "static", - "ip_address": "172.28.100.1", - "netmask": "255.255.255.0" - }, - "dhcp_server": { - "range_skip": "10", - "range_extent": "240", - "lease_time": "12h" - }, - "dns_server": {}, - "firewall": { - "zone": "lan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - } - }, - { - "name": "Primary Modem", - "id": "mdm0", - "type": "backhaul", - "modem_id": "primary", - "ipv4": { - "enabled": true, - "proto": "dhcp", - "peerdns": 0 - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - }, - { - "name": "Secondary Modem", - "id": "mdm1", - "type": "backhaul", - "modem_id": "secondary", - "ipv4": { - "enabled": true, - "proto": "dhcp" - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - }, - { - "name": "Wired Internet", - "id": "wan", - "type": "backhaul", - "interfaces": [ - "eth0.4" - ], - "ipv4": { - "enabled": true, - "proto": "dhcp", - "peerdns": 0 - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - } - ], - "multiwan": { - "globals": { - "mmx_mask": "0x3F00" - }, - "rules": {}, - "default_strategy": "dynamic_weight", - "health_checks": { - "down": 2, - "up": 1, - "initial_state": "offline", - "track_method": "ping", - "track_ip": [ - "1.1.1.1", - "8.8.8.8" - ], - "timeout": 2, - "interval": 1 - } - }, - "firewall": { - "defaults": { - "syn_flood": "1", - "input": "ACCEPT", - "forward": "REJECT", - "output": "ACCEPT", - "disable_ipv6": "1" - }, - "zones": [ - { - "description": "Local Networks", - "name": "lan", - "config": { - "name": "lan", - "input": "ACCEPT", - "output": "ACCEPT", - "forward": "ACCEPT" - }, - "forwarding": [ - { - "dest": "wan" - } - ] - }, - { - "description": "Local Networks - restricted", - "name": "lan_rst", - "config": { - "name": "lan_rst", - "input": "REJECT", - "output": "ACCEPT", - "forward": "REJECT" - }, - "forwarding": [ - { - "dest": "wan" - } - ] - }, - { - "description": "Internet", - "name": "wan", - "config": { - "name": "wan", - "input": "REJECT", - "output": "ACCEPT", - "forward": "REJECT", - "masq": 1, - "mtu_fix": 1 - } - } - ], - "rules": [ - { - "description": "Allows for DHCP renewal from upstream routers", - "config": { - "name": "Allow-DHCP-Renew", - "src": "wan", - "proto": "udp", - "dest_port": "68", - "target": "ACCEPT" - } - }, - { - "description": "Responds to pings from the Internet", - "config": { - "name": "Allow-Ping", - "src": "wan", - "proto": "icmp", - "icmp_type": "echo-request", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-IGMP", - "src": "wan", - "proto": "igmp", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-IPSec-ESP", - "src": "wan", - "dest": "lan", - "proto": "esp", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-ISAKMP", - "src": "wan", - "dest": "lan", - "proto": "udp", - "dest_port": "500", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-IPSec-ESP (RST)", - "src": "wan", - "dest": "lan_rst", - "proto": "esp", - "target": "ACCEPT" - } - }, - { - "name": "", - "config": { - "name": "Allow-ISAKMP (RST)", - "src": "wan", - "dest": "lan_rst", - "proto": "udp", - "dest_port": "500", - "target": "ACCEPT" - } - }, - { - "name": "", - "config": { - "name": "Allow DHCP request (RST)", - "src": "lan_rst", - "proto": "udp", - "src_port": "67-68", - "dest_port": "67-68", - "target": "ACCEPT" - } - }, - { - "name": "", - "config": { - "name": "Allow DNS queries (RST)", - "src": "lan_rst", - "proto": "tcp udp", - "dest_port": "53", - "target": "ACCEPT" - } - } - ] - }, - "static_route": [ - { - "target": "192.168.100.1", - "interface": "wan" - } - ] - }, - "gsm": { - "modems": { - "default": { - "enabled": true - }, - "known": [ - { - "name": "primary", - "id_field": "device", - "device": "/sys/devices/platform/scb/fd500000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0/usb1/1-1/1-1.2", - "autoconnect": true - }, - { - "name": "secondary", - "id_field": "device", - "device": "/sys/devices/platform/scb/fd500000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0/usb1/1-1/1-1.4", - "autoconnect": true - } - ] - } - }, - "metrics": { - "cloud_url": "$CLOUD_URL", - "publish_period": 60, - "templates": { - "network_stat": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "any_change": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "percent_5": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5 - } - ] - }, - "absolute_10": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "absolute", - "threshold": 10 - } - ] - } - }, - "collections": { - "net/adm/rx_bytes": { - "template": "network_stat" - }, - "net/adm/rx_packets": { - "template": "network_stat" - }, - "net/adm/rx_dropped": { - "template": "network_stat" - }, - "net/adm/rx_errors": { - "template": "network_stat" - }, - "net/adm/tx_bytes": { - "template": "network_stat" - }, - "net/adm/tx_packets": { - "template": "network_stat" - }, - "net/adm/tx_dropped": { - "template": "network_stat" - }, - "net/adm/tx_errors": { - "template": "network_stat" - }, - "net/jan/rx_bytes": { - "template": "network_stat" - }, - "net/jan/rx_packets": { - "template": "network_stat" - }, - "net/jan/rx_dropped": { - "template": "network_stat" - }, - "net/jan/rx_errors": { - "template": "network_stat" - }, - "net/jan/tx_bytes": { - "template": "network_stat" - }, - "net/jan/tx_packets": { - "template": "network_stat" - }, - "net/jan/tx_dropped": { - "template": "network_stat" - }, - "net/jan/tx_errors": { - "template": "network_stat" - }, - "net/mdm0/rx_bytes": { - "template": "network_stat" - }, - "net/mdm0/rx_packets": { - "template": "network_stat" - }, - "net/mdm0/rx_dropped": { - "template": "network_stat" - }, - "net/mdm0/rx_errors": { - "template": "network_stat" - }, - "net/mdm0/tx_bytes": { - "template": "network_stat" - }, - "net/mdm0/tx_packets": { - "template": "network_stat" - }, - "net/mdm0/tx_dropped": { - "template": "network_stat" - }, - "net/mdm0/tx_errors": { - "template": "network_stat" - }, - "net/mdm1/rx_bytes": { - "template": "network_stat" - }, - "net/mdm1/rx_packets": { - "template": "network_stat" - }, - "net/mdm1/rx_dropped": { - "template": "network_stat" - }, - "net/mdm1/rx_errors": { - "template": "network_stat" - }, - "net/mdm1/tx_bytes": { - "template": "network_stat" - }, - "net/mdm1/tx_packets": { - "template": "network_stat" - }, - "net/mdm1/tx_dropped": { - "template": "network_stat" - }, - "net/mdm1/tx_errors": { - "template": "network_stat" - }, - "net/wan/rx_bytes": { - "template": "network_stat" - }, - "net/wan/rx_packets": { - "template": "network_stat" - }, - "net/wan/rx_dropped": { - "template": "network_stat" - }, - "net/wan/rx_errors": { - "template": "network_stat" - }, - "net/wan/tx_bytes": { - "template": "network_stat" - }, - "net/wan/tx_packets": { - "template": "network_stat" - }, - "net/wan/tx_dropped": { - "template": "network_stat" - }, - "net/wan/tx_errors": { - "template": "network_stat" - }, - "net/mdm0/download_speed": { - "template": "any_change" - }, - "net/mdm1/download_speed": { - "template": "any_change" - }, - "net/wan/download_speed": { - "template": "any_change" - }, - "net/mdm0/status": { - "template": "any_change" - }, - "net/mdm1/status": { - "template": "any_change" - }, - "net/wan/status": { - "template": "any_change" - }, - "gsm/modem/primary/firmware": { - "template": "any_change", - "rename": [ - "modem", - "1", - "fw_version" - ] - }, - "gsm/modem/primary/band": { - "template": "any_change", - "rename": [ - "modem", - "1", - "band" - ] - }, - "gsm/modem/primary/access-tech": { - "template": "any_change", - "rename": [ - "modem", - "1", - "access_tech" - ] - }, - "gsm/modem/primary/access-family": { - "template": "any_change", - "rename": [ - "modem", - "1", - "access_fam" - ] - }, - "gsm/modem/primary/imei": { - "template": "any_change", - "rename": [ - "modem", - "1", - "imei" - ] - }, - "gsm/modem/primary/sim": { - "template": "any_change", - "rename": [ - "modem", - "1", - "sim" - ] - }, - "gsm/modem/primary/signal/rssi": { - "template": "percent_5", - "rename": [ - "modem", - "1", - "rssi" - ] - }, - "gsm/modem/primary/signal/rsrp": { - "template": "percent_5", - "rename": [ - "modem", - "1", - "rsrp" - ] - }, - "gsm/modem/primary/signal/rsrq": { - "template": "percent_5", - "rename": [ - "modem", - "1", - "rsrq" - ] - }, - "gsm/modem/primary/bars": { - "template": "any_change", - "rename": [ - "modem", - "1", - "bars" - ] - }, - "gsm/modem/primary/operator": { - "template": "any_change", - "rename": [ - "modem", - "1", - "operator" - ] - }, - "gsm/modem/primary/iccid": { - "template": "any_change", - "rename": [ - "modem", - "1", - "iccid" - ] - }, - "gsm/modem/primary/state": { - "template": "any_change", - "field": "curr_state", - "rename": [ - "modem", - "1", - "state" - ] - }, - "gsm/modem/primary/interface": { - "template": "any_change", - "rename": [ - "modem", - "1", - "wann_type" - ] - }, - "gsm/modem/primary/rx_bytes": { - "template": "network_stat", - "rename": [ - "modem", - "1", - "rx_bytes" - ] - }, - "gsm/modem/primary/tx_bytes": { - "template": "network_stat", - "rename": [ - "modem", - "1", - "tx_bytes" - ] - }, - "gsm/modem/secondary/firmware": { - "template": "any_change", - "rename": [ - "modem", - "2", - "fw_version" - ] - }, - "gsm/modem/secondary/band": { - "template": "any_change", - "rename": [ - "modem", - "2", - "band" - ] - }, - "gsm/modem/secondary/access-tech": { - "template": "any_change", - "rename": [ - "modem", - "2", - "access_tech" - ] - }, - "gsm/modem/secondary/access-family": { - "template": "any_change", - "rename": [ - "modem", - "2", - "access_fam" - ] - }, - "gsm/modem/secondary/imei": { - "template": "any_change", - "rename": [ - "modem", - "2", - "imei" - ] - }, - "gsm/modem/secondary/sim": { - "template": "any_change", - "rename": [ - "modem", - "2", - "sim" - ] - }, - "gsm/modem/secondary/signal/rssi": { - "template": "percent_5", - "rename": [ - "modem", - "2", - "rssi" - ] - }, - "gsm/modem/secondary/signal/rsrp": { - "template": "percent_5", - "rename": [ - "modem", - "2", - "rsrp" - ] - }, - "gsm/modem/secondary/signal/rsrq": { - "template": "percent_5", - "rename": [ - "modem", - "2", - "rsrq" - ] - }, - "gsm/modem/secondary/bars": { - "template": "any_change", - "rename": [ - "modem", - "2", - "bars" - ] - }, - "gsm/modem/secondary/operator": { - "template": "any_change", - "rename": [ - "modem", - "2", - "operator" - ] - }, - "gsm/modem/secondary/iccid": { - "template": "any_change", - "rename": [ - "modem", - "2", - "iccid" - ] - }, - "gsm/modem/secondary/state": { - "template": "any_change", - "field": "curr_state", - "rename": [ - "modem", - "2", - "state" - ] - }, - "gsm/modem/secondary/interface": { - "template": "any_change", - "rename": [ - "modem", - "2", - "wann_type" - ] - }, - "gsm/modem/secondary/rx_bytes": { - "template": "network_stat", - "rename": [ - "modem", - "2", - "rx_bytes" - ] - }, - "gsm/modem/secondary/tx_bytes": { - "template": "network_stat", - "rename": [ - "modem", - "2", - "tx_bytes" - ] - }, - "system/info/boot_time": { - "protocol": "http", - "rename": [ - "system", - "boot_time" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "absolute", - "threshold": 1000 - } - ] - }, - "system/info/firmware/version": { - "template": "any_change", - "rename": [ - "system", - "fw_id" - ] - }, - "system/info/hardware/revision": { - "template": "any_change", - "rename": [ - "system", - "hw_id" - ] - }, - "system/info/hardware/serial": { - "template": "any_change", - "rename": [ - "system", - "serial" - ] - }, - "system/info/hardware/board/revision": { - "template": "any_change", - "rename": [ - "system", - "board_revision" - ] - }, - "system/info/cpu/overall_utilisation": { - "template": "percent_5", - "rename": [ - "system", - "cpu_util" - ] - }, - "system/info/mem/util": { - "template": "percent_5", - "rename": [ - "system", - "mem_util" - ] - }, - "system/info/temperature": { - "protocol": "http", - "rename": [ - "system", - "temp" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "absolute", - "threshold": 2 - } - ] - }, - "system/info/power/under_voltage_detected": { - "template": "any_change", - "rename": [ - "system", - "power_undervoltage" - ] - }, - "system/info/power/arm_frequency_capped": { - "template": "any_change", - "rename": [ - "system", - "power_arm_frequency_capped" - ] - }, - "system/info/power/currently_throttled": { - "template": "any_change", - "rename": [ - "system", - "power_currently_throttled" - ] - }, - "system/info/power/under_voltage_occurred": { - "template": "any_change", - "rename": [ - "system", - "power_under_voltage_occurred" - ] - } - } - }, - "system": { - "report_period": 60, - "usb3_enabled": false, - "alarms": [ - { - "time": "5:00", - "repeats": "daily", - "payload": { - "name": "daily reboot", - "type": "reboot" - } - } - ] - } -} diff --git a/src/configs/bigbox-v1-cm-2.json b/src/configs/bigbox-v1-cm-2.json new file mode 100644 index 00000000..c95dfe87 --- /dev/null +++ b/src/configs/bigbox-v1-cm-2.json @@ -0,0 +1,1883 @@ +{ + "hal": { + "rev": 1, + "data": { + "schema": "devicecode.config/hal/1", + "filesystem": [ + { + "name": "config", + "root": "/data/devicecode/configs" + }, + { + "name": "credentials", + "root": "/data/configs" + }, + { + "name": "state", + "root": "/tmp/devicecode-state" + } + ], + "network": { + "provider": "openwrt", + "platform": { + "segment_trunk": { + "ifname": "eth0", + "protected": true + }, + "shaping": { + "marks": { + "mask": "0x00f00000", + "control": "0x00100000", + "client": "0x00200000" + } + } + } + }, + "wired": { + "providers": { + "cm5-local-wired": { + "provider": "static", + "mode": "read_only", + "poll": { + "static": { + "interval_s": 30.0, + "groups": [ + "snapshot" + ] + } + }, + "surfaces": { + "eth0": { + "provider_surface_id": "eth0", + "kind": "direct-nic", + "capabilities": { + "trunk": true, + "access": false, + "poe": false + }, + "link": { + "state": "unknown" + }, + "attachment": { + "mode": "trunk", + "vlans": [ + 8, + 100, + 32, + 4 + ] + } + } + } + }, + "switch-main": { + "provider": "rtl8380m_http", + "mode": "read_only", + "base_url": "http://172.28.100.9/", + "username": "$SWITCH_USERNAME", + "password": "$SWITCH_PASSWORD", + "timeout_s": 3.0, + "poll": { + "fast": { + "interval_s": 5.0, + "groups": [ + "panel", + "poe", + "counters" + ] + }, + "medium": { + "interval_s": 30.0, + "groups": [ + "vlan", + "lldp" + ] + }, + "slow": { + "interval_s": 180.0, + "groups": [ + "identity", + "runtime" + ] + } + }, + "http": { + "response_parser": "legacy-http1-close", + "capability": "main", + "max_response_bytes": 1048576 + } + } + } + }, + "wlan": { + "radios": [ + { + "name": "radio0", + "type": "mac80211", + "path": "axi/1000110000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0" + }, + { + "name": "radio1", + "type": "mac80211", + "path": "axi/1000110000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0+1" + } + ] + }, + "uart": { + "serial_ports": [ + { + "id": "uart0", + "name": "uart0", + "path": "/dev/ttyAMA0", + "baud": 115200, + "mode": "8N1" + } + ] + }, + "artifact_store": { + "stores": [ + { + "id": "main", + "transient_root": "/tmp/devicecode-artifacts", + "durable_root": "/data/devicecode/artifacts", + "import_root": "/data/devicecode/artifacts/import", + "durable_enabled": true + } + ] + }, + "control_store": [ + { + "name": "update", + "root": "/data/devicecode/control/update" + }, + { + "name": "gsm", + "root": "/data/devicecode/control/gsm" + } + ], + "time": {}, + "platform": {}, + "power": {}, + "usb": {}, + "modemcard": {}, + "sysmon": {} + } + }, + "net": { + "rev": 1, + "data": { + "schema": "devicecode.config/net/1", + "version": 1, + "product": "bigbox-legacy-converted-phase1", + "description": "Big Box default config using segment-first NET model, GSM-sourced modem uplinks, generated WAN route metrics, and Devicecode-owned OpenWrt UCI output.", + "vlan_policy": { + "reserved": { + "internal": 100 + }, + "ranges": { + "user": { + "from": 1, + "to": 99 + }, + "system": { + "from": 100, + "to": 399 + } + }, + "metadata": { + "note": "Converted from legacy VLANs eth0.8, eth0.32, eth0.100 and eth0.4." + } + }, + "segments": { + "adm": { + "name": "Admin Network", + "kind": "system", + "protected": true, + "user_editable": false, + "purpose": "admin_network", + "vlan": { + "id": 8 + }, + "addressing": { + "ipv4": { + "mode": "static", + "cidr": "172.28.8.1/24" + } + }, + "dhcp": { + "enabled": true, + "start": 10, + "limit": 240, + "lease_time": "12h" + }, + "dns": { + "local_server": true, + "host_files": [ + "ads" + ], + "domain": "bigbox.home" + }, + "firewall": { + "zone": "lan" + } + }, + "jan": { + "name": "Jangala Network", + "kind": "user", + "protected": false, + "user_editable": true, + "purpose": "restricted_user_access", + "vlan": { + "id": 32 + }, + "addressing": { + "ipv4": { + "mode": "static", + "cidr": "172.28.32.1/24" + } + }, + "dhcp": { + "enabled": true, + "start": 10, + "limit": 240, + "lease_time": "12h" + }, + "dns": { + "local_server": true, + "host_files": [ + "ads", + "adult" + ], + "domain": "bigbox.home" + }, + "firewall": { + "zone": "lan_rst" + }, + "shaping": { + "host_default": { + "mode": "budgeted_peak", + "all_hosts": true, + "download": { + "sustained_rate": "500kbit", + "peak_rate": "2mbit", + "burst_budget": "500k" + }, + "upload": { + "sustained_rate": "250kbit", + "peak_rate": "1mbit", + "burst_budget": "250k" + }, + "fq_codel": { + "flows": 1024, + "limit": 10240, + "memory_limit": "16Mb", + "target": "5ms", + "interval": "100ms" + } + } + } + }, + "int": { + "name": "Internal Network", + "kind": "system", + "protected": true, + "user_editable": false, + "purpose": "switch_ext_aps_and_internal_services", + "vlan": { + "reserved": "internal" + }, + "addressing": { + "ipv4": { + "mode": "static", + "cidr": "172.28.100.1/24" + } + }, + "dhcp": { + "enabled": true, + "start": 10, + "limit": 240, + "lease_time": "12h" + }, + "dns": {}, + "firewall": { + "zone": "lan" + } + }, + "wan": { + "name": "Wired Internet", + "kind": "wan", + "protected": false, + "user_editable": false, + "purpose": "wired_backhaul", + "vlan": { + "id": 4 + }, + "addressing": { + "ipv4": { + "mode": "dhcp", + "peerdns": false + } + }, + "dhcp": { + "enabled": false + }, + "firewall": { + "zone": "wan" + }, + "dns": {} + } + }, + "interfaces": {}, + "addressing": {}, + "dns": { + "enabled": true, + "domain": "bigbox.home", + "upstreams": [ + "8.8.8.8", + "1.1.1.1" + ], + "cache": { + "size": 10000 + }, + "host_files": { + "base_dir": "/etc/jng/hosts", + "addnmount": true, + "sources": { + "ads": { + "file": "ads", + "description": "advertising and tracking block list" + }, + "adult": { + "file": "adult", + "description": "adult-content block list" + } + } + }, + "records": { + "unifi": { + "name": "unifi", + "address": "$UNIFI_ADDRESS", + "segments": [ + "int" + ] + }, + "config.bigbox.home": { + "name": "config.bigbox.home", + "address": "172.28.8.1" + } + } + }, + "dhcp": { + "enabled": true, + "defaults": { + "lease_time": "12h", + "authoritative": true + }, + "reservations": {}, + "options": {}, + "relays": {} + }, + "firewall": { + "defaults": { + "syn_flood": "1", + "input": "ACCEPT", + "forward": "REJECT", + "output": "ACCEPT" + }, + "zones": { + "lan": { + "input": "ACCEPT", + "output": "ACCEPT", + "forward": "ACCEPT" + }, + "lan_rst": { + "input": "REJECT", + "output": "ACCEPT", + "forward": "REJECT" + }, + "wan": { + "input": "REJECT", + "output": "ACCEPT", + "forward": "REJECT", + "masq": 1, + "mtu_fix": 1 + } + }, + "policies": { + "lan_to_wan_1": { + "from": "lan", + "to": "wan", + "action": "allow" + }, + "lan_rst_to_wan_1": { + "from": "lan_rst", + "to": "wan", + "action": "allow" + } + }, + "rules": { + "Allow-DHCP-Renew": { + "name": "Allow-DHCP-Renew", + "src": "wan", + "proto": "udp", + "dest_port": "68", + "target": "ACCEPT" + }, + "Allow-Ping": { + "name": "Allow-Ping", + "src": "wan", + "proto": "icmp", + "icmp_type": "echo-request", + "target": "ACCEPT" + }, + "Allow-IGMP": { + "name": "Allow-IGMP", + "src": "wan", + "proto": "igmp", + "target": "ACCEPT" + }, + "Allow-IPSec-ESP": { + "name": "Allow-IPSec-ESP", + "src": "wan", + "dest": "lan", + "proto": "esp", + "target": "ACCEPT" + }, + "Allow-ISAKMP": { + "name": "Allow-ISAKMP", + "src": "wan", + "dest": "lan", + "proto": "udp", + "dest_port": "500", + "target": "ACCEPT" + }, + "Allow-IPSec-ESP_RST": { + "name": "Allow-IPSec-ESP (RST)", + "src": "wan", + "dest": "lan_rst", + "proto": "esp", + "target": "ACCEPT" + }, + "Allow-ISAKMP_RST": { + "name": "Allow-ISAKMP (RST)", + "src": "wan", + "dest": "lan_rst", + "proto": "udp", + "dest_port": "500", + "target": "ACCEPT" + }, + "Allow_DHCP_request_RST": { + "name": "Allow DHCP request (RST)", + "src": "lan_rst", + "proto": "udp", + "src_port": "67-68", + "dest_port": "67-68", + "target": "ACCEPT" + }, + "Allow_DNS_queries_RST": { + "name": "Allow DNS queries (RST)", + "src": "lan_rst", + "proto": "tcp udp", + "dest_port": "53", + "target": "ACCEPT" + } + } + }, + "routing": { + "routes": { + "starlink_admin": { + "kind": "host", + "target": "192.168.100.1", + "interface": "wan", + "description": "Starlink dish/admin UI reachable via the wired WAN uplink" + } + } + }, + "wan": { + "enabled": true, + "members": { + "wan": { + "interface": "wan", + "weight": 1, + "dynamic_weight": true, + "family": "ipv4", + "mwan_metric": 1 + }, + "modem_primary": { + "interface": "modem_primary", + "weight": 1, + "dynamic_weight": true, + "family": "ipv4", + "source": { + "kind": "gsm-uplink", + "id": "primary" + }, + "mwan_metric": 1 + }, + "modem_secondary": { + "interface": "modem_secondary", + "weight": 1, + "dynamic_weight": true, + "family": "ipv4", + "source": { + "kind": "gsm-uplink", + "id": "secondary" + }, + "mwan_metric": 1 + } + }, + "health": { + "track_ip": [ + "1.1.1.1", + "8.8.8.8" + ], + "track_method": "ping", + "timeout": 2, + "interval": 1, + "down": 2, + "up": 1 + }, + "runtime": { + "mmx_mask": "0x3F00" + }, + "load_balancing": { + "policy": "balanced", + "speedtests": { + "enabled": true, + "interval_s": 21600, + "retry_after_s": 10, + "retry_max_s": 3600 + } + }, + "rules": { + "https": { + "family": "ipv4", + "proto": "tcp", + "dest_port": "443", + "policy": "balanced", + "sticky": true + } + } + }, + "vpn": { + "enabled": false + }, + "diagnostics": { + "reflectors": { + "cloudflare": { + "address": "1.1.1.1" + }, + "google": { + "address": "8.8.8.8" + } + } + }, + "runtime": { + "apply": { + "debounce_s": 0.25 + }, + "observe": { + "interval_s": 5 + } + }, + "metadata": {} + } + }, + "wired": { + "rev": 1, + "data": { + "schema": "devicecode.config/wired/1", + "version": 1, + "product": "bigbox-legacy-converted-phase1", + "surfaces": { + "cm5-eth0": { + "name": "CM5 Ethernet", + "kind": "direct-nic", + "role": "internal-trunk", + "protected": true, + "attachment": { + "mode": "trunk", + "required_segments": [ + "adm", + "int" + ], + "user_segments": "all-realised-user-segments" + } + }, + "switch-uplink-cm5": { + "name": "Switch uplink to CM5", + "kind": "switch-port", + "role": "internal-trunk", + "protected": true, + "attachment": { + "mode": "trunk", + "required_segments": [ + "adm", + "int" + ], + "user_segments": "all-realised-user-segments" + } + }, + "lan-1": { + "name": "LAN 1", + "kind": "ethernet-port", + "role": "external-port", + "capabilities": { + "poe": true + }, + "attachment": { + "mode": "none" + } + }, + "lan-2": { + "name": "LAN 2", + "kind": "ethernet-port", + "role": "external-port", + "capabilities": { + "poe": true + }, + "attachment": { + "mode": "none" + } + }, + "lan-3": { + "name": "LAN 3", + "kind": "ethernet-port", + "role": "external-port", + "capabilities": { + "poe": true + }, + "attachment": { + "mode": "none" + } + }, + "lan-4": { + "name": "LAN 4", + "kind": "ethernet-port", + "role": "external-port", + "capabilities": { + "poe": true + }, + "attachment": { + "mode": "none" + } + }, + "lan-5": { + "name": "LAN 5", + "kind": "ethernet-port", + "role": "external-port", + "capabilities": { + "poe": true + }, + "attachment": { + "mode": "none" + } + }, + "lan-6": { + "name": "LAN 6", + "kind": "ethernet-port", + "role": "external-port", + "capabilities": { + "poe": true + }, + "attachment": { + "mode": "none" + } + }, + "lan-7": { + "name": "LAN 7", + "kind": "ethernet-port", + "role": "external-port", + "capabilities": { + "poe": true + }, + "attachment": { + "mode": "none" + } + }, + "sfp-1": { + "name": "SFP 1", + "kind": "sfp-port", + "role": "external-port", + "attachment": { + "mode": "none" + } + }, + "sfp-2": { + "name": "SFP 2", + "kind": "sfp-port", + "role": "external-port", + "attachment": { + "mode": "none" + } + } + }, + "metadata": { + "todo": "Add user-facing switch port surfaces once the fixed RTL8380M port-to-segment layout is known." + } + } + }, + "device": { + "rev": 1, + "data": { + "schema": "devicecode.config/device/1", + "components": { + "cm5": { + "class": "host", + "subtype": "cm5", + "role": "primary", + "member": "local", + "required_facts": [ + "software", + "updater" + ], + "facts": { + "software": [ + "raw", + "host", + "updater", + "cap", + "updater", + "cm5", + "state", + "software" + ], + "updater": [ + "raw", + "host", + "updater", + "cap", + "updater", + "cm5", + "state", + "updater" + ], + "health": [ + "raw", + "host", + "updater", + "cap", + "updater", + "cm5", + "state", + "health" + ] + }, + "actions": { + "prepare-update": { + "kind": "rpc", + "call_topic": [ + "raw", + "host", + "updater", + "cap", + "updater", + "cm5", + "rpc", + "prepare" + ] + }, + "stage-update": { + "kind": "rpc", + "call_topic": [ + "raw", + "host", + "updater", + "cap", + "updater", + "cm5", + "rpc", + "stage" + ] + }, + "commit-update": { + "kind": "rpc", + "call_topic": [ + "raw", + "host", + "updater", + "cap", + "updater", + "cm5", + "rpc", + "commit" + ] + } + } + }, + "mcu": { + "class": "member", + "subtype": "mcu", + "role": "controller", + "member": "mcu", + "required_facts": [ + "software", + "updater" + ], + "facts": { + "software": [ + "raw", + "member", + "mcu", + "state", + "software" + ], + "updater": [ + "raw", + "member", + "mcu", + "state", + "updater" + ], + "health": [ + "raw", + "member", + "mcu", + "state", + "health" + ], + "power_battery": [ + "raw", + "member", + "mcu", + "state", + "power", + "battery" + ], + "power_charger": [ + "raw", + "member", + "mcu", + "state", + "power", + "charger" + ], + "power_charger_config": [ + "raw", + "member", + "mcu", + "state", + "power", + "charger", + "config" + ], + "environment_temperature": [ + "raw", + "member", + "mcu", + "state", + "environment", + "temperature" + ], + "environment_humidity": [ + "raw", + "member", + "mcu", + "state", + "environment", + "humidity" + ], + "runtime_memory": [ + "raw", + "member", + "mcu", + "state", + "runtime", + "memory" + ] + }, + "events": { + "charger_alert": [ + "raw", + "member", + "mcu", + "cap", + "telemetry", + "main", + "event", + "power", + "charger", + "alert" + ] + }, + "actions": { + "restart": { + "kind": "rpc", + "call_topic": [ + "raw", + "member", + "mcu", + "cap", + "control", + "main", + "rpc", + "restart" + ] + }, + "prepare-update": { + "kind": "rpc", + "call_topic": [ + "raw", + "member", + "mcu", + "cap", + "updater", + "main", + "rpc", + "prepare-update" + ] + }, + "stage-update": { + "kind": "fabric_stage", + "target": "updater/main", + "chunk_size": 2048, + "artifact_store": "main", + "timeout_s": 300 + }, + "commit-update": { + "kind": "rpc", + "call_topic": [ + "raw", + "member", + "mcu", + "cap", + "updater", + "main", + "rpc", + "commit-update" + ] + } + } + }, + "switch-main": { + "kind": "switch", + "module": "switch", + "class": "host", + "role": "switch-fabric", + "member": "switch-main", + "facts": { + "wired_observation_status": [ + "raw", + "host", + "wired", + "provider", + "switch-main", + "status" + ], + "wired_observation_surfaces": [ + "raw", + "host", + "wired", + "provider", + "switch-main", + "state", + "surfaces" + ], + "wired_observation_topology": [ + "raw", + "host", + "wired", + "provider", + "switch-main", + "state", + "topology" + ] + } + } + }, + "assembly": { + "schema": "devicecode.device.assembly/1", + "product": "big-box", + "components": { + "cm5": { + "kind": "compute", + "role": "controller" + }, + "mcu": { + "kind": "microcontroller", + "role": "power-sensor-controller" + }, + "switch-main": { + "kind": "switch", + "role": "wired-fabric", + "observation": { + "kind": "local-provider", + "source": [ + "raw", + "host", + "wired", + "provider", + "switch-main" + ] + } + }, + "cm5-local-wired": { + "kind": "direct-nic", + "role": "controller-wired-port", + "observation": { + "kind": "local-provider", + "source": [ + "raw", + "host", + "wired", + "provider", + "cm5-local-wired" + ] + } + } + }, + "links": { + "cm5-switch": { + "kind": "wired", + "role": "controller-switch-uplink", + "internal": true, + "a": { + "component": "cm5-local-wired", + "observed_surface": "eth0" + }, + "b": { + "component": "switch-main", + "observed_surface": "GE8" + } + } + }, + "surfaces": { + "cm5-eth0": { + "kind": "ethernet", + "exposure": "internal", + "component": "cm5-local-wired", + "observed_surface": "eth0" + }, + "switch-uplink-cm5": { + "kind": "ethernet", + "exposure": "internal", + "component": "switch-main", + "observed_surface": "GE8", + "connector": "internal-ge8" + }, + "lan-1": { + "kind": "ethernet", + "exposure": "external", + "component": "switch-main", + "observed_surface": "GE1", + "connector": "RJ45-1" + }, + "lan-2": { + "kind": "ethernet", + "exposure": "external", + "component": "switch-main", + "observed_surface": "GE2", + "connector": "RJ45-2" + }, + "lan-3": { + "kind": "ethernet", + "exposure": "external", + "component": "switch-main", + "observed_surface": "GE3", + "connector": "RJ45-3" + }, + "lan-4": { + "kind": "ethernet", + "exposure": "external", + "component": "switch-main", + "observed_surface": "GE4", + "connector": "RJ45-4" + }, + "lan-5": { + "kind": "ethernet", + "exposure": "external", + "component": "switch-main", + "observed_surface": "GE5", + "connector": "RJ45-5" + }, + "lan-6": { + "kind": "ethernet", + "exposure": "external", + "component": "switch-main", + "observed_surface": "GE6", + "connector": "RJ45-6" + }, + "lan-7": { + "kind": "ethernet", + "exposure": "external", + "component": "switch-main", + "observed_surface": "GE7", + "connector": "RJ45-7" + }, + "sfp-1": { + "kind": "sfp", + "exposure": "external", + "component": "switch-main", + "observed_surface": "GE9", + "connector": "SFP-1" + }, + "sfp-2": { + "kind": "sfp", + "exposure": "external", + "component": "switch-main", + "observed_surface": "GE10", + "connector": "SFP-2" + } + } + } + } + }, + "wifi": { + "rev": 1, + "data": { + "schema": "devicecode.config/wifi/1", + "report_period": 10, + "radios": [ + { + "name": "radio0", + "band": "2g", + "channel": "auto", + "channels": [ + "1", + "6", + "11" + ], + "country": "GB", + "htmode": "HE20", + "txpower": "20", + "disabled": false + }, + { + "name": "radio1", + "band": "5g", + "channel": "36", + "country": "GB", + "htmode": "HE80", + "txpower": "23", + "disabled": false + } + ], + "ssids": [ + { + "mainflux_path": "mainflux.cfg", + "encryption": "psk2", + "mode": "access_point", + "radios": [ + "radio0", + "radio1" + ] + }, + { + "name": "Jangala", + "mode": "access_point", + "encryption": "none", + "radios": [ + "radio0", + "radio1" + ], + "segment": "jan" + } + ], + "band_steering": { + "globals": { + "rrm_mode": "PAT", + "kicking": { + "kick_mode": "both", + "bandwidth_threshold": 0, + "kicking_threshold": 15, + "evals_before_kick": 3 + }, + "stations": { + "use_station_count": false, + "max_station_diff": 1 + }, + "neighbor_reports": { + "dyn_report_num": 2, + "disassoc_report_len": 6 + }, + "legacy": { + "eval_probe_req": true, + "eval_assoc_req": false, + "eval_auth_req": false, + "min_probe_count": 2, + "deny_auth_reason": 1, + "deny_assoc_reason": 17 + } + }, + "timings": { + "updates": { + "client": 10, + "chan_util": 5, + "beacon_reports": 5, + "hostapd": 10, + "tcp_con": 10 + }, + "cleanup": { + "client": 15, + "probe": 40, + "ap": 460 + }, + "inactive_client_kickoff": 60 + }, + "bands": { + "2g": { + "initial_score": 90, + "rssi_scoring": { + "center": "-69", + "weight": 2, + "good_threshold": "-66", + "good_reward": 12, + "bad_threshold": "-75", + "bad_penalty": "-15" + }, + "chan_util_scoring": { + "good_threshold": 140, + "good_reward": 5, + "bad_threshold": 170, + "bad_penalty": "-20" + }, + "support_bonuses": { + "vht": 5, + "ht": 5 + } + }, + "5g": { + "initial_score": 95, + "rssi_scoring": { + "center": "-64", + "weight": 4, + "good_threshold": "-62", + "good_reward": 10, + "bad_threshold": "-70", + "bad_penalty": "-28" + }, + "chan_util_scoring": { + "good_threshold": 140, + "good_reward": 5, + "bad_threshold": 170, + "bad_penalty": "-20" + }, + "support_bonuses": { + "vht": 2, + "ht": 2 + } + } + } + } + } + }, + "gsm": { + "rev": 1, + "data": { + "schema": "devicecode.config/gsm/1", + "modems": { + "default": { + "enabled": true, + "signal_freq": 10 + }, + "known": [ + { + "name": "primary", + "id_field": "device", + "device": "/sys/devices/platform/axi/1000120000.pcie/1f00200000.usb/xhci-hcd.0/usb3/3-1", + "autoconnect": true, + "metrics_interval": 60 + }, + { + "name": "secondary", + "id_field": "device", + "device": "/sys/devices/platform/axi/1000480000.usb/usb1/1-1", + "autoconnect": true, + "metrics_interval": 60 + } + ] + }, + "apn_store": { + "kind": "control-store", + "id": "gsm", + "key": "custom-apns-v1" + } + } + }, + "fabric": { + "rev": 1, + "data": { + "schema": "devicecode.config/fabric/1", + "local_node": "bigbox-cm5", + "links": [ + { + "id": "mcu-uart0", + "peer_id": "mcu", + "transport": { + "kind": "uart", + "source": "uart_manager", + "class": "uart", + "id": "uart0", + "terminator": "\n" + }, + "session": { + "identity_claim": { + "id": "bigbox-cm5", + "role": "controller" + } + }, + "bridge": { + "imports": [ + { + "id": "mcu-state", + "remote": [ + "state", + "self" + ], + "local": [ + "raw", + "member", + "mcu", + "state" + ] + }, + { + "id": "mcu-event", + "remote": [ + "event", + "self" + ], + "local": [ + "raw", + "member", + "mcu", + "cap", + "telemetry", + "main", + "event" + ] + }, + { + "id": "mcu-cap", + "remote": [ + "cap", + "self" + ], + "local": [ + "raw", + "member", + "mcu", + "cap" + ] + } + ], + "exports": [], + "rpc": { + "outbound": [ + { + "id": "mcu-updater-prepare", + "local": [ + "raw", + "member", + "mcu", + "cap", + "updater", + "main", + "rpc", + "prepare-update" + ], + "remote": [ + "cap", + "self", + "updater", + "main", + "rpc", + "prepare-update" + ] + }, + { + "id": "mcu-updater-commit", + "local": [ + "raw", + "member", + "mcu", + "cap", + "updater", + "main", + "rpc", + "commit-update" + ], + "remote": [ + "cap", + "self", + "updater", + "main", + "rpc", + "commit-update" + ] + } + ], + "inbound": [] + } + } + } + ] + } + }, + "http": { + "rev": 1, + "data": { + "schema": "devicecode.config/http/1", + "enabled": true, + "id": "main", + "policy": { + "allowed_schemes": { + "http": true, + "https": true, + "ws": true, + "wss": true + }, + "allow_loopback": true, + "allowed_response_parsers": { + "strict": true, + "legacy-http1-close": true + }, + "legacy_http1_close_max_response_bytes": 1048576 + } + } + }, + "ui": { + "rev": 1, + "data": { + "schema": "devicecode.config/ui/1", + "enabled": true, + "http": { + "enabled": true, + "cap_id": "main", + "host": "0.0.0.0", + "port": 80 + }, + "updates": { + "upload": { + "enabled": true, + "max_bytes": 67108864, + "require_auth": false, + "component": "mcu", + "create_job": true, + "start_job": true + }, + "commit": { + "require_auth": false + } + } + } + }, + "update": { + "rev": 1, + "data": { + "schema": "devicecode.update/1", + "components": { + "cm5": { + "component": "cm5" + }, + "mcu": { + "component": "mcu", + "backend": "component", + "stage_timeout_s": 300 + }, + "switch-main": { + "component": "switch-main" + } + }, + "bundled": { + "enabled": true, + "max_attempts": 30, + "components": { + "mcu": { + "component": "mcu", + "source": { + "kind": "file", + "path": "/artifacts/mcu.dcmcu", + "policy": "transient_only" + }, + "job": { + "job_id": "bundled-mcu", + "create_if": "image_differs", + "start": "auto", + "commit": "auto", + "reconcile": "required", + "supersede": "same_job_if_image_changed" + } + } + } + }, + "publish": { + "enabled": true + }, + "retention": { + "terminal_max_count": 50, + "terminal_max_age_s": 604800, + "active_intent_restart_max": 1, + "prune_on_startup": true + } + } + }, + "system": { + "rev": 1, + "data": { + "schema": "devicecode.config/system/1", + "report_period": 60, + "usb3_enabled": true, + "alarms": [ + { + "time": "5:00", + "repeats": "daily", + "payload": { + "name": "daily reboot", + "type": "reboot" + } + } + ] + } + }, + "metrics": { + "rev": 1, + "data": { + "schema": "devicecode.config/metrics/1", + "cloud_url": "$CLOUD_URL", + "publish_period": 60, + "templates": { + "network_stat": { + "protocol": "http", + "process": [ + { + "type": "DiffTrigger", + "diff_method": "percent", + "threshold": 5, + "initial_val": 0 + }, + { + "type": "DeltaValue" + } + ] + }, + "any_change": { + "protocol": "http", + "process": [ + { + "type": "DiffTrigger", + "diff_method": "any-change" + } + ] + }, + "percent_5": { + "protocol": "http", + "process": [ + { + "type": "DiffTrigger", + "diff_method": "percent", + "threshold": 5 + } + ] + }, + "absolute_10": { + "protocol": "http", + "process": [ + { + "type": "DiffTrigger", + "diff_method": "absolute", + "threshold": 10 + } + ] + } + }, + "pipelines": { + "wann_type": { + "template": "any_change" + }, + "access_tech": { + "template": "any_change" + }, + "access_fam": { + "template": "any_change" + }, + "rx_bytes": { + "template": "network_stat" + }, + "rx_packets": { + "template": "network_stat" + }, + "rx_dropped": { + "template": "network_stat" + }, + "rx_errors": { + "template": "network_stat" + }, + "tx_bytes": { + "template": "network_stat" + }, + "tx_packets": { + "template": "network_stat" + }, + "tx_dropped": { + "template": "network_stat" + }, + "tx_errors": { + "template": "network_stat" + }, + "speedtest": { + "protocol": "http", + "process": [] + }, + "firmware": { + "template": "any_change" + }, + "band": { + "template": "any_change" + }, + "access-tech": { + "template": "any_change" + }, + "access-family": { + "template": "any_change" + }, + "imei": { + "template": "any_change" + }, + "sim": { + "template": "any_change" + }, + "rssi": { + "template": "percent_5" + }, + "rsrp": { + "template": "percent_5" + }, + "rsrq": { + "template": "percent_5" + }, + "bars": { + "template": "any_change" + }, + "operator": { + "template": "any_change" + }, + "iccid": { + "template": "any_change" + }, + "state": { + "template": "any_change" + }, + "interface": { + "template": "any_change" + }, + "boot_time": { + "protocol": "http", + "process": [ + { + "type": "DiffTrigger", + "diff_method": "absolute", + "threshold": 1000 + } + ] + }, + "version": { + "template": "any_change" + }, + "fw_version": { + "template": "any_change" + }, + "revision": { + "template": "any_change" + }, + "serial": { + "template": "any_change" + }, + "overall_utilisation": { + "template": "percent_5" + }, + "util": { + "template": "percent_5" + }, + "temperature": { + "protocol": "http", + "process": [ + { + "type": "DiffTrigger", + "diff_method": "absolute", + "threshold": 2 + } + ] + }, + "alloc": { + "protocol": "http", + "process": [ + { + "type": "TimeTrigger", + "duration": 60 + } + ] + }, + "internal": { + "protocol": "http", + "process": [ + { + "type": "DiffTrigger", + "diff_method": "absolute", + "threshold": 1 + } + ] + }, + "core": { + "template": "percent_5" + }, + "vbat": { + "template": "absolute_10" + }, + "ibat": { + "template": "absolute_10" + }, + "vin": { + "template": "absolute_10" + }, + "vsys": { + "template": "absolute_10" + }, + "iin": { + "template": "absolute_10" + }, + "charger_enabled": { + "template": "any_change" + }, + "mppt_en_pin": { + "template": "any_change" + }, + "equalize_req": { + "template": "any_change" + }, + "drvcc_good": { + "template": "any_change" + }, + "cell_count_error": { + "template": "any_change" + }, + "ok_to_charge": { + "template": "any_change" + }, + "no_rt": { + "template": "any_change" + }, + "thermal_shutdown": { + "template": "any_change" + }, + "vin_ovlo": { + "template": "any_change" + }, + "vin_gt_vbat": { + "template": "any_change" + }, + "intvcc_gt_4p3v": { + "template": "any_change" + }, + "intvcc_gt_2p8v": { + "template": "any_change" + }, + "iin_limited": { + "template": "any_change" + }, + "uvcl_active": { + "template": "any_change" + }, + "cc_phase": { + "template": "any_change" + }, + "cv_phase": { + "template": "any_change" + }, + "bat_short": { + "template": "any_change" + }, + "bat_missing": { + "template": "any_change" + }, + "max_charge_time_fault": { + "template": "any_change" + }, + "c_over_x_term": { + "template": "any_change" + }, + "timer_term": { + "template": "any_change" + }, + "ntc_pause": { + "template": "any_change" + }, + "precharge": { + "template": "any_change" + }, + "cccv": { + "template": "any_change" + }, + "absorb": { + "template": "any_change" + }, + "equalize": { + "template": "any_change" + }, + "suspended": { + "template": "any_change" + }, + "session_start": { + "protocol": "http", + "process": [] + }, + "session_end": { + "protocol": "http", + "process": [] + }, + "hostname": { + "template": "any_change" + }, + "signal": { + "template": "percent_5" + }, + "power": { + "protocol": "http", + "process": [ + { + "type": "DiffTrigger", + "diff_method": "percent", + "threshold": 10, + "initial_val": 0 + } + ] + }, + "channel": { + "template": "any_change" + }, + "noise": { + "protocol": "http", + "process": [ + { + "type": "DiffTrigger", + "diff_method": "percent", + "threshold": 10, + "initial_val": 0 + } + ] + }, + "num_sta": { + "template": "percent_5" + }, + "mem_util": { + "template": "any_change" + }, + "cpu_util": { + "template": "any_change" + }, + "temp": { + "template": "any_change" + }, + "curr_time": { + "template": "any_change" + }, + "fw_id": { + "template": "any_change" + }, + "hw_id": { + "template": "any_change" + } + } + } + } +} diff --git a/src/configs/bigbox-v1-cm.json b/src/configs/bigbox-v1-cm.json deleted file mode 100644 index 426599a3..00000000 --- a/src/configs/bigbox-v1-cm.json +++ /dev/null @@ -1,1460 +0,0 @@ -{ - "hal": { - "managers": { - "modemcard": {}, - "uci": {}, - "ubus": {}, - "wlan": { - "radios": { - "radio0": { - "type": "mac80211", - "path": "axi/1000110000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0" - }, - "radio1": { - "type": "mac80211", - "path": "axi/1000110000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0+1" - } - } - }, - "uart": { - "serial_ports": [ - { - "name": "uart0", - "path": "/dev/ttyAMA0" - } - ] - } - } - }, - "net": { - "report_period": 60, - "default": { - "backhauls": { - "network": { - "ipv4": { - "proto": "dhcp" - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - } - }, - "dns": { - "upstream_servers": [ - "8.8.8.8", - "1.1.1.1" - ], - "default_cache_size": 1000 - }, - "dhcp": { - "domains": [ - { - "name": "unifi", - "ip": "$UNIFI_IP" - }, - { - "name": "config.bigbox.home", - "ip": "172.28.8.1" - } - ] - }, - "network": [ - { - "name": "Loopback", - "id": "loopback", - "type": "local", - "interfaces": [ - "lo" - ], - "ipv4": { - "proto": "static", - "ip_address": "127.0.0.1", - "netmask": "255.0.0.0" - }, - "dns_server": { - "local_server": true, - "default_hosts": [] - } - }, - { - "name": "Admin Network", - "id": "adm", - "type": "local", - "is_bridge": true, - "interfaces": [ - "eth0.8" - ], - "ipv4": { - "proto": "static", - "ip_address": "172.28.8.1", - "netmask": "255.255.255.0" - }, - "dhcp_server": { - "range_skip": "10", - "range_extent": "240", - "lease_time": "12h" - }, - "dns_server": { - "local_server": true, - "default_hosts": [ - "ads" - ] - }, - "firewall": { - "zone": "lan" - } - }, - { - "name": "Jangala Network", - "id": "jan", - "type": "local", - "is_bridge": true, - "interfaces": [ - "eth0.32" - ], - "ipv4": { - "proto": "static", - "ip_address": "172.28.32.1", - "netmask": "255.255.255.0" - }, - "dhcp_server": { - "range_skip": "10", - "range_extent": "240", - "lease_time": "12h" - }, - "dns_server": { - "local_server": true, - "default_hosts": [ - "ads", - "adult" - ] - }, - "firewall": { - "zone": "lan_rst" - }, - "shaping": { - "egress": { - "qdisc": "htb", - "filters": [ - { - "id": "per_ip_filter", - "kind": "u32", - "hash_key": "dest_ip", - "target_class_template": "per_ip_shapers" - } - ], - "class_template": { - "id": "per_ip_shapers", - "kind": "per_dest_ip", - "classes": [ - { - "qdisc": "htb", - "config": { - "rate": "400kbit", - "ceil": "2mbit", - "burst": "200k" - }, - "classes": [ - { - "qdisc": "fq_codel" - } - ] - } - ] - } - }, - "ingress": { - "qdisc": "htb", - "filters": [ - { - "id": "per_ip_filter", - "kind": "u32", - "hash_key": "src_ip", - "target_class_template": "per_ip_shapers" - } - ], - "class_template": { - "id": "per_ip_shapers", - "kind": "per_src_ip", - "classes": [ - { - "qdisc": "htb", - "config": { - "rate": "400kbit", - "ceil": "2mbit", - "burst": "200k" - }, - "classes": [ - { - "qdisc": "fq_codel" - } - ] - } - ] - } - } - } - }, - { - "name": "Internal Network", - "id": "int", - "type": "local", - "is_bridge": true, - "interfaces": [ - "eth0.100" - ], - "ipv4": { - "proto": "static", - "ip_address": "172.28.100.1", - "netmask": "255.255.255.0" - }, - "dhcp_server": { - "range_skip": "10", - "range_extent": "240", - "lease_time": "12h" - }, - "dns_server": {}, - "firewall": { - "zone": "lan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - } - }, - { - "name": "Primary Modem", - "id": "mdm0", - "type": "backhaul", - "modem_id": "primary", - "ipv4": { - "enabled": true, - "proto": "dhcp", - "peerdns": 0 - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - }, - { - "name": "Secondary Modem", - "id": "mdm1", - "type": "backhaul", - "modem_id": "secondary", - "ipv4": { - "enabled": true, - "proto": "dhcp" - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - }, - { - "name": "Wired Internet", - "id": "wan", - "type": "backhaul", - "interfaces": [ - "eth0.4" - ], - "ipv4": { - "enabled": true, - "proto": "dhcp", - "peerdns": 0 - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - } - ], - "multiwan": { - "globals": { - "mmx_mask": "0x3F00" - }, - "rules": {}, - "default_strategy": "dynamic_weight", - "health_checks": { - "down": 2, - "up": 1, - "initial_state": "offline", - "track_method": "ping", - "track_ip": [ - "1.1.1.1", - "8.8.8.8" - ], - "timeout": 2, - "interval": 1 - } - }, - "firewall": { - "defaults": { - "syn_flood": "1", - "input": "ACCEPT", - "forward": "REJECT", - "output": "ACCEPT", - "disable_ipv6": "1" - }, - "zones": [ - { - "description": "Local Networks", - "name": "lan", - "config": { - "name": "lan", - "input": "ACCEPT", - "output": "ACCEPT", - "forward": "ACCEPT" - }, - "forwarding": [ - { - "dest": "wan" - } - ] - }, - { - "description": "Local Networks - restricted", - "name": "lan_rst", - "config": { - "name": "lan_rst", - "input": "REJECT", - "output": "ACCEPT", - "forward": "REJECT" - }, - "forwarding": [ - { - "dest": "wan" - } - ] - }, - { - "description": "Internet", - "name": "wan", - "config": { - "name": "wan", - "input": "REJECT", - "output": "ACCEPT", - "forward": "REJECT", - "masq": 1, - "mtu_fix": 1 - } - } - ], - "rules": [ - { - "description": "Allows for DHCP renewal from upstream routers", - "config": { - "name": "Allow-DHCP-Renew", - "src": "wan", - "proto": "udp", - "dest_port": "68", - "target": "ACCEPT" - } - }, - { - "description": "Responds to pings from the Internet", - "config": { - "name": "Allow-Ping", - "src": "wan", - "proto": "icmp", - "icmp_type": "echo-request", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-IGMP", - "src": "wan", - "proto": "igmp", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-IPSec-ESP", - "src": "wan", - "dest": "lan", - "proto": "esp", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-ISAKMP", - "src": "wan", - "dest": "lan", - "proto": "udp", - "dest_port": "500", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-IPSec-ESP (RST)", - "src": "wan", - "dest": "lan_rst", - "proto": "esp", - "target": "ACCEPT" - } - }, - { - "name": "", - "config": { - "name": "Allow-ISAKMP (RST)", - "src": "wan", - "dest": "lan_rst", - "proto": "udp", - "dest_port": "500", - "target": "ACCEPT" - } - }, - { - "name": "", - "config": { - "name": "Allow DHCP request (RST)", - "src": "lan_rst", - "proto": "udp", - "src_port": "67-68", - "dest_port": "67-68", - "target": "ACCEPT" - } - }, - { - "name": "", - "config": { - "name": "Allow DNS queries (RST)", - "src": "lan_rst", - "proto": "tcp udp", - "dest_port": "53", - "target": "ACCEPT" - } - } - ] - }, - "static_route": [ - { - "target": "192.168.100.1", - "interface": "wan" - } - ] - }, - "wifi": { - "report_period": 10, - "radios": [ - { - "name": "radio0", - "band": "2g", - "channel": "auto", - "channels": [ - "1", - "6", - "11" - ], - "country": "GB", - "htmode": "HE20", - "txpower": "20", - "disabled": false - }, - { - "name": "radio1", - "band": "5g", - "channel": "36", - "country": "GB", - "htmode": "HE80", - "txpower": "23", - "disabled": false - } - ], - "ssids": [ - { - "mainflux_path": "config/mainflux", - "encryption": "psk2", - "mode": "access_point", - "radios": [ - "radio0", - "radio1" - ] - }, - { - "name": "Jangala", - "mode": "access_point", - "encryption": "none", - "network": "jan", - "radios": [ - "radio0", - "radio1" - ] - } - ], - "band_steering": { - "log_level": 2, - "globals": { - "rrm_mode": "PAT", - "kicking": { - "kick_mode": "both", - "bandwidth_threshold": 0, - "kicking_threshold": 15, - "evals_before_kick": 3 - }, - "stations": { - "use_station_count": false, - "max_station_diff": 1 - }, - "neighbor_reports": { - "dyn_report_num": 2, - "disassoc_report_len": 6 - }, - "legacy": { - "eval_probe_req": true, - "eval_assoc_req": false, - "eval_auth_req": false, - "min_probe_count": 2, - "deny_auth_reason": 1, - "deny_assoc_reason": 17 - } - }, - "timings": { - "updates": { - "client": 10, - "chan_util": 5, - "beacon_reports": 5, - "hostapd": 10, - "tcp_con": 10 - }, - "cleanup": { - "client": 15, - "probe": 40, - "ap": 460 - }, - "inactive_client_kickoff": 60 - }, - "networking": { - "method": "tcp+umdns", - "ip": "0.0.0.0", - "port": 1026, - "broadcast_port": 1025, - "enable_encryption": false - }, - "bands": { - "2g": { - "initial_score": 90, - "rssi_scoring": { - "center": "-69", - "weight": 2, - "good_threshold": "-66", - "good_reward": 12, - "bad_threshold": "-75", - "bad_penalty": "-15" - }, - "chan_util_scoring": { - "good_threshold": 140, - "good_reward": 5, - "bad_threshold": 170, - "bad_penalty": "-20" - }, - "support_bonuses": { - "vht": 5, - "ht": 5 - } - }, - "5g": { - "initial_score": 95, - "rssi_scoring": { - "center": "-64", - "weight": 4, - "good_threshold": "-62", - "good_reward": 10, - "bad_threshold": "-70", - "bad_penalty": "-28" - }, - "chan_util_scoring": { - "good_threshold": 140, - "good_reward": 5, - "bad_threshold": 170, - "bad_penalty": "-20" - }, - "support_bonuses": { - "vht": 2, - "ht": 2 - } - } - } - } - }, - "gsm": { - "modems": { - "default": { - "enabled": true - }, - "known": [ - { - "name": "primary", - "id_field": "device", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00200000.usb/xhci-hcd.0/usb3/3-1", - "autoconnect": true - }, - { - "name": "secondary", - "id_field": "device", - "device": "/sys/devices/platform/axi/1000480000.usb/usb1/1-1", - "autoconnect": true - } - ] - } - }, - "bridge": { - "sources": [ - { - "name": "pico_mcu", - "type": "uart", - "capability_id": "uart0", - "options": { - "baudrate": 115200 - } - } - ], - "connections": [ - { - "source": "pico_mcu", - "topic": "power/#", - "direction": "in", - "local_prefix": "cm", - "remote_prefix": "mcu" - } - ] - }, - "metrics": { - "cloud_url": "$CLOUD_URL", - "publish_period": 60, - "templates": { - "network_stat": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "any_change": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "percent_5": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5 - } - ] - }, - "absolute_10": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "absolute", - "threshold": 10 - } - ] - } - }, - "collections": { - "net/adm/rx_bytes": { - "template": "network_stat" - }, - "net/adm/rx_packets": { - "template": "network_stat" - }, - "net/adm/rx_dropped": { - "template": "network_stat" - }, - "net/adm/rx_errors": { - "template": "network_stat" - }, - "net/adm/tx_bytes": { - "template": "network_stat" - }, - "net/adm/tx_packets": { - "template": "network_stat" - }, - "net/adm/tx_dropped": { - "template": "network_stat" - }, - "net/adm/tx_errors": { - "template": "network_stat" - }, - "net/jan/rx_bytes": { - "template": "network_stat" - }, - "net/jan/rx_packets": { - "template": "network_stat" - }, - "net/jan/rx_dropped": { - "template": "network_stat" - }, - "net/jan/rx_errors": { - "template": "network_stat" - }, - "net/jan/tx_bytes": { - "template": "network_stat" - }, - "net/jan/tx_packets": { - "template": "network_stat" - }, - "net/jan/tx_dropped": { - "template": "network_stat" - }, - "net/jan/tx_errors": { - "template": "network_stat" - }, - "net/mdm0/rx_bytes": { - "template": "network_stat" - }, - "net/mdm0/rx_packets": { - "template": "network_stat" - }, - "net/mdm0/rx_dropped": { - "template": "network_stat" - }, - "net/mdm0/rx_errors": { - "template": "network_stat" - }, - "net/mdm0/tx_bytes": { - "template": "network_stat" - }, - "net/mdm0/tx_packets": { - "template": "network_stat" - }, - "net/mdm0/tx_dropped": { - "template": "network_stat" - }, - "net/mdm0/tx_errors": { - "template": "network_stat" - }, - "net/mdm1/rx_bytes": { - "template": "network_stat" - }, - "net/mdm1/rx_packets": { - "template": "network_stat" - }, - "net/mdm1/rx_dropped": { - "template": "network_stat" - }, - "net/mdm1/rx_errors": { - "template": "network_stat" - }, - "net/mdm1/tx_bytes": { - "template": "network_stat" - }, - "net/mdm1/tx_packets": { - "template": "network_stat" - }, - "net/mdm1/tx_dropped": { - "template": "network_stat" - }, - "net/mdm1/tx_errors": { - "template": "network_stat" - }, - "net/wan/rx_bytes": { - "template": "network_stat" - }, - "net/wan/rx_packets": { - "template": "network_stat" - }, - "net/wan/rx_dropped": { - "template": "network_stat" - }, - "net/wan/rx_errors": { - "template": "network_stat" - }, - "net/wan/tx_bytes": { - "template": "network_stat" - }, - "net/wan/tx_packets": { - "template": "network_stat" - }, - "net/wan/tx_dropped": { - "template": "network_stat" - }, - "net/wan/tx_errors": { - "template": "network_stat" - }, - "net/mdm0/download_speed": { - "template": "any_change" - }, - "net/mdm1/download_speed": { - "template": "any_change" - }, - "net/wan/download_speed": { - "template": "any_change" - }, - "net/mdm0/status": { - "template": "any_change" - }, - "net/mdm1/status": { - "template": "any_change" - }, - "net/wan/status": { - "template": "any_change" - }, - "gsm/modem/primary/firmware": { - "template": "any_change", - "rename": [ - "modem", - "1", - "fw_version" - ] - }, - "gsm/modem/primary/band": { - "template": "any_change", - "rename": [ - "modem", - "1", - "band" - ] - }, - "gsm/modem/primary/access-tech": { - "template": "any_change", - "rename": [ - "modem", - "1", - "access_tech" - ] - }, - "gsm/modem/primary/access-family": { - "template": "any_change", - "rename": [ - "modem", - "1", - "access_fam" - ] - }, - "gsm/modem/primary/imei": { - "template": "any_change", - "rename": [ - "modem", - "1", - "imei" - ] - }, - "gsm/modem/primary/sim": { - "template": "any_change", - "rename": [ - "modem", - "1", - "sim" - ] - }, - "gsm/modem/primary/signal/rssi": { - "template": "percent_5", - "rename": [ - "modem", - "1", - "rssi" - ] - }, - "gsm/modem/primary/signal/rsrp": { - "template": "percent_5", - "rename": [ - "modem", - "1", - "rsrp" - ] - }, - "gsm/modem/primary/signal/rsrq": { - "template": "percent_5", - "rename": [ - "modem", - "1", - "rsrq" - ] - }, - "gsm/modem/primary/bars": { - "template": "any_change", - "rename": [ - "modem", - "1", - "bars" - ] - }, - "gsm/modem/primary/operator": { - "template": "any_change", - "rename": [ - "modem", - "1", - "operator" - ] - }, - "gsm/modem/primary/iccid": { - "template": "any_change", - "rename": [ - "modem", - "1", - "iccid" - ] - }, - "gsm/modem/primary/state": { - "template": "any_change", - "field": "curr_state", - "rename": [ - "modem", - "1", - "state" - ] - }, - "gsm/modem/primary/interface": { - "template": "any_change", - "rename": [ - "modem", - "1", - "wann_type" - ] - }, - "gsm/modem/primary/rx_bytes": { - "template": "network_stat", - "rename": [ - "modem", - "1", - "rx_bytes" - ] - }, - "gsm/modem/primary/tx_bytes": { - "template": "network_stat", - "rename": [ - "modem", - "1", - "tx_bytes" - ] - }, - "gsm/modem/secondary/firmware": { - "template": "any_change", - "rename": [ - "modem", - "2", - "fw_version" - ] - }, - "gsm/modem/secondary/band": { - "template": "any_change", - "rename": [ - "modem", - "2", - "band" - ] - }, - "gsm/modem/secondary/access-tech": { - "template": "any_change", - "rename": [ - "modem", - "2", - "access_tech" - ] - }, - "gsm/modem/secondary/access-family": { - "template": "any_change", - "rename": [ - "modem", - "2", - "access_fam" - ] - }, - "gsm/modem/secondary/imei": { - "template": "any_change", - "rename": [ - "modem", - "2", - "imei" - ] - }, - "gsm/modem/secondary/sim": { - "template": "any_change", - "rename": [ - "modem", - "2", - "sim" - ] - }, - "gsm/modem/secondary/signal/rssi": { - "template": "percent_5", - "rename": [ - "modem", - "2", - "rssi" - ] - }, - "gsm/modem/secondary/signal/rsrp": { - "template": "percent_5", - "rename": [ - "modem", - "2", - "rsrp" - ] - }, - "gsm/modem/secondary/signal/rsrq": { - "template": "percent_5", - "rename": [ - "modem", - "2", - "rsrq" - ] - }, - "gsm/modem/secondary/bars": { - "template": "any_change", - "rename": [ - "modem", - "2", - "bars" - ] - }, - "gsm/modem/secondary/operator": { - "template": "any_change", - "rename": [ - "modem", - "2", - "operator" - ] - }, - "gsm/modem/secondary/iccid": { - "template": "any_change", - "rename": [ - "modem", - "2", - "iccid" - ] - }, - "gsm/modem/secondary/state": { - "template": "any_change", - "field": "curr_state", - "rename": [ - "modem", - "2", - "state" - ] - }, - "gsm/modem/secondary/interface": { - "template": "any_change", - "rename": [ - "modem", - "2", - "wann_type" - ] - }, - "gsm/modem/secondary/rx_bytes": { - "template": "network_stat", - "rename": [ - "modem", - "2", - "rx_bytes" - ] - }, - "gsm/modem/secondary/tx_bytes": { - "template": "network_stat", - "rename": [ - "modem", - "2", - "tx_bytes" - ] - }, - "system/info/boot_time": { - "protocol": "http", - "rename": [ - "system", - "boot_time" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "absolute", - "threshold": 1000 - } - ] - }, - "system/info/firmware/version": { - "template": "any_change", - "rename": [ - "system", - "fw_id" - ] - }, - "system/info/hardware/revision": { - "template": "any_change", - "rename": [ - "system", - "hw_id" - ] - }, - "system/info/hardware/serial": { - "template": "any_change", - "rename": [ - "system", - "serial" - ] - }, - "system/info/hardware/board/revision": { - "template": "any_change", - "rename": [ - "system", - "board_revision" - ] - }, - "system/info/cpu/overall_utilisation": { - "template": "percent_5", - "rename": [ - "system", - "cpu_util" - ] - }, - "system/info/mem/util": { - "template": "percent_5", - "rename": [ - "system", - "mem_util" - ] - }, - "system/info/temperature": { - "protocol": "http", - "rename": [ - "system", - "temp" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "absolute", - "threshold": 2 - } - ] - }, - "system/info/power/under_voltage_detected": { - "template": "any_change", - "rename": [ - "system", - "power_undervoltage" - ] - }, - "system/info/power/arm_frequency_capped": { - "template": "any_change", - "rename": [ - "system", - "power_arm_frequency_capped" - ] - }, - "system/info/power/currently_throttled": { - "template": "any_change", - "rename": [ - "system", - "power_currently_throttled" - ] - }, - "system/info/power/under_voltage_occurred": { - "template": "any_change", - "rename": [ - "system", - "power_under_voltage_occurred" - ] - }, - "mcu/sys/mem/alloc": { - "protocol": "http", - "process": [ - { - "type": "TimeTrigger", - "duration": 60 - } - ] - }, - "mcu/power/temperature/internal": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "absolute", - "threshold": 1 - } - ] - }, - "mcu/env/temperature/core": { - "template": "percent_5" - }, - "mcu/env/humidity/core": { - "template": "percent_5" - }, - "mcu/power/battery/internal/vbat": { - "template": "absolute_10" - }, - "mcu/power/battery/internal/ibat": { - "template": "absolute_10" - }, - "mcu/power/charger/internal/vin": { - "template": "absolute_10" - }, - "mcu/power/charger/internal/vsys": { - "template": "absolute_10" - }, - "mcu/power/charger/internal/iin": { - "template": "absolute_10" - }, - "mcu/power/charger/internal/system/charger_enabled": { - "template": "any_change" - }, - "mcu/power/charger/internal/system/mppt_en_pin": { - "template": "any_change" - }, - "mcu/power/charger/internal/system/equalize_req": { - "template": "any_change" - }, - "mcu/power/charger/internal/system/drvcc_good": { - "template": "any_change" - }, - "mcu/power/charger/internal/system/cell_count_error": { - "template": "any_change" - }, - "mcu/power/charger/internal/system/ok_to_charge": { - "template": "any_change" - }, - "mcu/power/charger/internal/system/no_rt": { - "template": "any_change" - }, - "mcu/power/charger/internal/system/thermal_shutdown": { - "template": "any_change" - }, - "mcu/power/charger/internal/system/vin_ovlo": { - "template": "any_change" - }, - "mcu/power/charger/internal/system/vin_gt_vbat": { - "template": "any_change" - }, - "mcu/power/charger/internal/system/intvcc_gt_4p3v": { - "template": "any_change" - }, - "mcu/power/charger/internal/system/intvcc_gt_2p8v": { - "template": "any_change" - }, - "mcu/power/charger/internal/status/iin_limited": { - "template": "any_change" - }, - "mcu/power/charger/internal/status/uvcl_active": { - "template": "any_change" - }, - "mcu/power/charger/internal/status/cc_phase": { - "template": "any_change" - }, - "mcu/power/charger/internal/status/cv_phase": { - "template": "any_change" - }, - "mcu/power/charger/internal/state/bat_short": { - "template": "any_change" - }, - "mcu/power/charger/internal/state/bat_missing": { - "template": "any_change" - }, - "mcu/power/charger/internal/state/max_charge_time_fault": { - "template": "any_change" - }, - "mcu/power/charger/internal/state/c_over_x_term": { - "template": "any_change" - }, - "mcu/power/charger/internal/state/timer_term": { - "template": "any_change" - }, - "mcu/power/charger/internal/state/ntc_pause": { - "template": "any_change" - }, - "mcu/power/charger/internal/state/precharge": { - "template": "any_change" - }, - "mcu/power/charger/internal/state/cccv": { - "template": "any_change" - }, - "mcu/power/charger/internal/state/absorb": { - "template": "any_change" - }, - "mcu/power/charger/internal/state/equalize": { - "template": "any_change" - }, - "mcu/power/charger/internal/state/suspended": { - "template": "any_change" - }, - "wifi/clients/+/sessions/+/session_start": { - "protocol": "http", - "process": [] - }, - "wifi/clients/+/sessions/+/session_end": { - "protocol": "http", - "process": [] - }, - "wifi/clients/+/sessions/+/hostname": { - "template": "any_change" - }, - "wifi/clients/+/sessions/+/rx_bytes": { - "template": "network_stat" - }, - "wifi/clients/+/sessions/+/tx_bytes": { - "template": "network_stat" - }, - "wifi/clients/+/sessions/+/signal": { - "template": "percent_5" - }, - "wifi/hp/+/+/+/power": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 10, - "initial_val": 0 - } - ] - }, - "wifi/hp/+/+/+/channel": { - "template": "any_change" - }, - "wifi/hp/+/+/+/noise": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 10, - "initial_val": 0 - } - ] - }, - "wifi/hp/+/+/+/rx_bytes": { - "template": "network_stat" - }, - "wifi/hp/+/+/+/rx_packets": { - "template": "network_stat" - }, - "wifi/hp/+/+/+/rx_dropped": { - "template": "network_stat" - }, - "wifi/hp/+/+/+/rx_errors": { - "template": "network_stat" - }, - "wifi/hp/+/+/+/tx_bytes": { - "template": "network_stat" - }, - "wifi/hp/+/+/+/tx_packets": { - "template": "network_stat" - }, - "wifi/hp/+/+/+/tx_dropped": { - "template": "network_stat" - }, - "wifi/hp/+/+/+/tx_errors": { - "template": "network_stat" - }, - "wifi/hp/+/+/+/num_sta": { - "template": "percent_5" - }, - "wifi/num_sta": { - "template": "percent_5" - }, - "wifi/hp/+/num_sta": { - "template": "percent_5" - }, - "switch/system/mem": { - "template": "any_change" - }, - "switch/system/cpu": { - "template": "any_change" - }, - "switch/system/temp": { - "template": "any_change" - }, - "switch/system/power": { - "template": "any_change" - }, - "switch/system/curr_time": { - "template": "any_change" - } - } - }, - "system": { - "report_period": 60, - "usb3_enabled": true, - "alarms": [ - { - "time": "5:00", - "repeats": "daily", - "payload": { - "name": "daily reboot", - "type": "reboot" - } - } - ] - }, - "switch": { - "report_period": 10, - "host": "172.28.100.9", - "username": "$SWITCH_USERNAME", - "password": "$SWITCH_PASSWORD", - "type": "rtl8380m" - } -} diff --git a/src/configs/bigbox_tera.json b/src/configs/bigbox_tera.json deleted file mode 100644 index 20231d12..00000000 --- a/src/configs/bigbox_tera.json +++ /dev/null @@ -1,2075 +0,0 @@ -{ - "hal": { - "managers": { - "modemcard": {}, - "uci": {}, - "ubus": {}, - "wlan": { - "radios": { - "radio0": { - "type": "mac80211", - "path": "axi/1000110000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0" - }, - "radio1": { - "type": "mac80211", - "path": "axi/1000110000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0+1" - } - } - }, - "uart": { - "serial_ports": [ - { - "name": "uart0", - "path": "/dev/ttyAMA0" - } - ] - } - } - }, - "net": { - "report_period": 60, - "default": { - "backhauls": { - "network": { - "ipv4": { - "proto": "dhcp" - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - } - }, - "dns": { - "upstream_servers": [ - "8.8.8.8", - "1.1.1.1" - ], - "default_cache_size": 1000 - }, - "dhcp": { - "domains": [ - { - "name": "unifi", - "ip": "$UNIFI_IP" - }, - { - "name": "config.bigbox.home", - "ip": "172.28.8.1" - } - ] - }, - "network": [ - { - "name": "Loopback", - "id": "loopback", - "type": "local", - "interfaces": [ - "lo" - ], - "ipv4": { - "proto": "static", - "ip_address": "127.0.0.1", - "netmask": "255.0.0.0" - }, - "dns_server": { - "local_server": true, - "default_hosts": [] - } - }, - { - "name": "Admin Network", - "id": "adm", - "type": "local", - "is_bridge": true, - "interfaces": [ - "eth0.8" - ], - "ipv4": { - "proto": "static", - "ip_address": "172.28.8.1", - "netmask": "255.255.255.0" - }, - "dhcp_server": { - "range_skip": "10", - "range_extent": "240", - "lease_time": "12h" - }, - "dns_server": { - "local_server": true, - "default_hosts": [ - "ads" - ] - }, - "firewall": { - "zone": "lan" - } - }, - { - "name": "Jangala Network", - "id": "jan", - "type": "local", - "is_bridge": true, - "interfaces": [ - "eth0.32" - ], - "ipv4": { - "proto": "static", - "ip_address": "172.28.32.1", - "netmask": "255.255.255.0" - }, - "dhcp_server": { - "range_skip": "10", - "range_extent": "240", - "lease_time": "12h" - }, - "dns_server": { - "local_server": true, - "default_hosts": [ - "ads", - "adult" - ] - }, - "firewall": { - "zone": "lan_rst" - }, - "shaping": { - "egress": { - "qdisc": "htb", - "filters": [ - { - "id": "per_ip_filter", - "kind": "u32", - "hash_key": "dest_ip", - "target_class_template": "per_ip_shapers" - } - ], - "class_template": { - "id": "per_ip_shapers", - "kind": "per_dest_ip", - "classes": [ - { - "qdisc": "htb", - "config": { - "rate": "2mbit", - "ceil": "8mbit", - "burst": "500k" - }, - "classes": [ - { - "qdisc": "fq_codel" - } - ] - } - ] - } - }, - "ingress": { - "qdisc": "htb", - "filters": [ - { - "id": "per_ip_filter", - "kind": "u32", - "hash_key": "src_ip", - "target_class_template": "per_ip_shapers" - } - ], - "class_template": { - "id": "per_ip_shapers", - "kind": "per_src_ip", - "classes": [ - { - "qdisc": "htb", - "config": { - "rate": "1.5mbit", - "ceil": "6mbit", - "burst": "225k" - }, - "classes": [ - { - "qdisc": "fq_codel" - } - ] - } - ] - } - } - } - }, - { - "name": "Internal Network", - "id": "int", - "type": "local", - "is_bridge": true, - "interfaces": [ - "eth0.100" - ], - "ipv4": { - "proto": "static", - "ip_address": "172.28.100.1", - "netmask": "255.255.255.0" - }, - "dhcp_server": { - "range_skip": "10", - "range_extent": "240", - "lease_time": "12h" - }, - "dns_server": {}, - "firewall": { - "zone": "lan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - } - }, - { - "name": "Primary Modem", - "id": "mdm0", - "type": "backhaul", - "modem_id": "primary", - "ipv4": { - "enabled": true, - "proto": "dhcp", - "peerdns": 0 - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - }, - { - "name": "Secondary Modem", - "id": "mdm1", - "type": "backhaul", - "modem_id": "secondary", - "ipv4": { - "enabled": true, - "proto": "dhcp" - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - }, - { - "name": "Wired Internet", - "id": "wan", - "type": "backhaul", - "interfaces": [ - "eth0.4" - ], - "ipv4": { - "enabled": true, - "proto": "dhcp", - "peerdns": 0 - }, - "firewall": { - "zone": "wan" - }, - "shaping": { - "egress": { - "qdisc": "fq_codel" - }, - "ingress": { - "qdisc": "fq_codel" - } - }, - "multiwan": { - "dynamic_weight": true, - "metric": 1 - } - } - ], - "multiwan": { - "globals": { - "mmx_mask": "0x3F00" - }, - "rules": {}, - "default_strategy": "dynamic_weight", - "health_checks": { - "down": 2, - "up": 1, - "initial_state": "offline", - "track_method": "ping", - "track_ip": [ - "1.1.1.1", - "8.8.8.8" - ], - "timeout": 2, - "interval": 1 - } - }, - "firewall": { - "defaults": { - "syn_flood": "1", - "input": "ACCEPT", - "forward": "REJECT", - "output": "ACCEPT", - "disable_ipv6": "1" - }, - "zones": [ - { - "description": "Local Networks", - "name": "lan", - "config": { - "name": "lan", - "input": "ACCEPT", - "output": "ACCEPT", - "forward": "ACCEPT" - }, - "forwarding": [ - { - "dest": "wan" - } - ] - }, - { - "description": "Local Networks - restricted", - "name": "lan_rst", - "config": { - "name": "lan_rst", - "input": "REJECT", - "output": "ACCEPT", - "forward": "REJECT" - }, - "forwarding": [ - { - "dest": "wan" - } - ] - }, - { - "description": "Internet", - "name": "wan", - "config": { - "name": "wan", - "input": "REJECT", - "output": "ACCEPT", - "forward": "REJECT", - "masq": 1, - "mtu_fix": 1 - } - } - ], - "rules": [ - { - "description": "Allows for DHCP renewal from upstream routers", - "config": { - "name": "Allow-DHCP-Renew", - "src": "wan", - "proto": "udp", - "dest_port": "68", - "target": "ACCEPT" - } - }, - { - "description": "Responds to pings from the Internet", - "config": { - "name": "Allow-Ping", - "src": "wan", - "proto": "icmp", - "icmp_type": "echo-request", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-IGMP", - "src": "wan", - "proto": "igmp", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-IPSec-ESP", - "src": "wan", - "dest": "lan", - "proto": "esp", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-ISAKMP", - "src": "wan", - "dest": "lan", - "proto": "udp", - "dest_port": "500", - "target": "ACCEPT" - } - }, - { - "description": "", - "config": { - "name": "Allow-IPSec-ESP (RST)", - "src": "wan", - "dest": "lan_rst", - "proto": "esp", - "target": "ACCEPT" - } - }, - { - "name": "", - "config": { - "name": "Allow-ISAKMP (RST)", - "src": "wan", - "dest": "lan_rst", - "proto": "udp", - "dest_port": "500", - "target": "ACCEPT" - } - }, - { - "name": "", - "config": { - "name": "Allow DHCP request (RST)", - "src": "lan_rst", - "proto": "udp", - "src_port": "67-68", - "dest_port": "67-68", - "target": "ACCEPT" - } - }, - { - "name": "", - "config": { - "name": "Allow DNS queries (RST)", - "src": "lan_rst", - "proto": "tcp udp", - "dest_port": "53", - "target": "ACCEPT" - } - } - ] - }, - "static_route": [ - { - "target": "192.168.100.1", - "interface": "wan" - } - ] - }, - "wifi": { - "report_period": 10, - "radios": [ - { - "name": "radio0", - "band": "2g", - "channel": "auto", - "channels": [ - "1", - "6", - "11" - ], - "country": "GB", - "htmode": "HE20", - "txpower": "20", - "disabled": false - }, - { - "name": "radio1", - "band": "5g", - "channel": "36", - "country": "GB", - "htmode": "HE80", - "txpower": "23", - "disabled": false - } - ], - "ssids": [ - { - "name": "Jangala", - "mode": "access_point", - "encryption": "none", - "network": "jan", - "radios": [ - "radio0", - "radio1" - ] - }, - { - "mainflux_path": "config/mainflux", - "encryption": "psk2", - "mode": "access_point", - "radios": [ - "radio0", - "radio1" - ] - } - ], - "band_steering": { - "log_level": 2, - "globals": { - "rrm_mode": "PAT", - "kicking": { - "kick_mode": "both", - "bandwidth_threshold": 0, - "kicking_threshold": 15, - "evals_before_kick": 3 - }, - "stations": { - "use_station_count": false, - "max_station_diff": 1 - }, - "neighbor_reports": { - "dyn_report_num": 2, - "disassoc_report_len": 6 - }, - "legacy": { - "eval_probe_req": true, - "eval_assoc_req": false, - "eval_auth_req": false, - "min_probe_count": 2, - "deny_auth_reason": 1, - "deny_assoc_reason": 17 - } - }, - "timings": { - "updates": { - "client": 10, - "chan_util": 5, - "beacon_reports": 5, - "hostapd": 10, - "tcp_con": 10 - }, - "cleanup": { - "client": 15, - "probe": 40, - "ap": 460 - }, - "inactive_client_kickoff": 60 - }, - "networking": { - "method": "tcp+umdns", - "ip": "0.0.0.0", - "port": 1026, - "broadcast_port": 1025, - "enable_encryption": false - }, - "bands": { - "2g": { - "initial_score": 90, - "rssi_scoring": { - "center": "-69", - "weight": 2, - "good_threshold": "-66", - "good_reward": 12, - "bad_threshold": "-75", - "bad_penalty": "-15" - }, - "chan_util_scoring": { - "good_threshold": 140, - "good_reward": 5, - "bad_threshold": 170, - "bad_penalty": "-20" - }, - "support_bonuses": { - "vht": 5, - "ht": 5 - } - }, - "5g": { - "initial_score": 95, - "rssi_scoring": { - "center": "-64", - "weight": 4, - "good_threshold": "-62", - "good_reward": 10, - "bad_threshold": "-70", - "bad_penalty": "-28" - }, - "chan_util_scoring": { - "good_threshold": 140, - "good_reward": 5, - "bad_threshold": 170, - "bad_penalty": "-20" - }, - "support_bonuses": { - "vht": 2, - "ht": 2 - } - } - } - } - }, - "gsm": { - "modems": { - "default": { - "enabled": true - }, - "known": [ - { - "name": "primary", - "id_field": "device", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb5/5-1/5-1.1", - "autoconnect": true - }, - { - "name": "secondary", - "id_field": "device", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb4/4-1/4-1.2", - "autoconnect": true - } - ] - } - }, - "bridge": { - "sources": [ - { - "name": "pico_mcu", - "type": "uart", - "capability_id": "uart0", - "options": { - "baudrate": 115200 - } - } - ], - "connections": [ - { - "source": "pico_mcu", - "topic": "power/#", - "direction": "in", - "local_prefix": "cm", - "remote_prefix": "mcu" - } - ] - }, - "metrics": { - "cloud_url": "$CLOUD_URL", - "publish_cache": { - "enabled": true, - "period": 60 - }, - "collections": { - "net/adm/rx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/adm/rx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/adm/rx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/adm/rx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/adm/tx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/adm/tx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/adm/tx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/adm/tx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/jan/rx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/jan/rx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/jan/rx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/jan/rx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/jan/tx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/jan/tx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/jan/tx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/jan/tx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm0/rx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm0/rx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm0/rx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm0/rx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm0/tx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm0/tx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm0/tx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm0/tx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm1/rx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm1/rx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm1/rx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm1/rx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm1/tx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm1/tx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm1/tx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/mdm1/tx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/wan/rx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/wan/rx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/wan/rx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/wan/rx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/wan/tx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/wan/tx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/wan/tx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "net/wan/tx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "gsm/modem/primary/firmware": { - "protocol": "http", - "rename": [ - "modem", - "1", - "fw_version" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/primary/band": { - "protocol": "http", - "rename": [ - "modem", - "1", - "band" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/primary/access-tech": { - "protocol": "http", - "rename": [ - "modem", - "1", - "access_tech" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/primary/access-family": { - "protocol": "http", - "rename": [ - "modem", - "1", - "access_fam" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/primary/imei": { - "protocol": "http", - "rename": [ - "modem", - "1", - "imei" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/primary/sim": { - "protocol": "http", - "rename": [ - "modem", - "1", - "sim" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/primary/signal/rssi": { - "protocol": "http", - "rename": [ - "modem", - "1", - "rssi" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5 - } - ] - }, - "gsm/modem/primary/signal/rsrp": { - "protocol": "http", - "rename": [ - "modem", - "1", - "rsrp" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5 - } - ] - }, - "gsm/modem/primary/signal/rsrq": { - "protocol": "http", - "rename": [ - "modem", - "1", - "rsrq" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5 - } - ] - }, - "gsm/modem/primary/bars": { - "protocol": "http", - "rename": [ - "modem", - "1", - "bars" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/primary/operator": { - "protocol": "http", - "rename": [ - "modem", - "1", - "operator" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/primary/iccid": { - "protocol": "http", - "rename": [ - "modem", - "1", - "iccid" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/primary/state": { - "protocol": "http", - "field": "curr_state", - "rename": [ - "modem", - "1", - "state" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/primary/interface": { - "protocol": "http", - "rename": [ - "modem", - "1", - "wann_type" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/primary/rx_bytes": { - "protocol": "http", - "rename": [ - "modem", - "1", - "rx_bytes" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "gsm/modem/primary/tx_bytes": { - "protocol": "http", - "rename": [ - "modem", - "1", - "tx_bytes" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "gsm/modem/secondary/firmware": { - "protocol": "http", - "rename": [ - "modem", - "2", - "fw_version" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/secondary/band": { - "protocol": "http", - "rename": [ - "modem", - "2", - "band" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/secondary/access-tech": { - "protocol": "http", - "rename": [ - "modem", - "2", - "access_tech" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/secondary/access-family": { - "protocol": "http", - "rename": [ - "modem", - "2", - "access_fam" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/secondary/imei": { - "protocol": "http", - "rename": [ - "modem", - "2", - "imei" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/secondary/sim": { - "protocol": "http", - "rename": [ - "modem", - "2", - "sim" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/secondary/signal/rssi": { - "protocol": "http", - "rename": [ - "modem", - "2", - "rssi" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5 - } - ] - }, - "gsm/modem/secondary/signal/rsrp": { - "protocol": "http", - "rename": [ - "modem", - "2", - "rsrp" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5 - } - ] - }, - "gsm/modem/secondary/signal/rsrq": { - "protocol": "http", - "rename": [ - "modem", - "2", - "rsrq" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5 - } - ] - }, - "gsm/modem/secondary/bars": { - "protocol": "http", - "rename": [ - "modem", - "2", - "bars" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/secondary/operator": { - "protocol": "http", - "rename": [ - "modem", - "2", - "operator" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/secondary/iccid": { - "protocol": "http", - "rename": [ - "modem", - "2", - "iccid" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/secondary/state": { - "protocol": "http", - "field": "curr_state", - "rename": [ - "modem", - "2", - "state" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/secondary/interface": { - "protocol": "http", - "rename": [ - "modem", - "2", - "wann_type" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "gsm/modem/secondary/rx_bytes": { - "protocol": "http", - "rename": [ - "modem", - "2", - "rx_bytes" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "gsm/modem/secondary/tx_bytes": { - "protocol": "http", - "rename": [ - "modem", - "2", - "tx_bytes" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "system/info/boot_time": { - "protocol": "http", - "rename": [ - "system", - "boot_time" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "system/info/firmware/version": { - "protocol": "http", - "rename": [ - "system", - "fw_id" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "system/info/hardware/revision": { - "protocol": "http", - "rename": [ - "system", - "hw_id" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "system/info/hardware/serial": { - "protocol": "http", - "rename": [ - "system", - "serial" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "system/info/hardware/board/revision": { - "protocol": "http", - "rename": [ - "system", - "board_revision" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "system/info/cpu/overall_utilisation": { - "protocol": "http", - "rename": [ - "system", - "cpu_util" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5 - } - ] - }, - "system/info/mem/util": { - "protocol": "http", - "rename": [ - "system", - "mem_util" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5 - } - ] - }, - "system/info/temperature": { - "protocol": "http", - "rename": [ - "system", - "temp" - ], - "process": [ - { - "type": "DiffTrigger", - "diff_method": "absolute", - "threshold": 2 - } - ] - }, - "wifi/clients/+/sessions/+/session_start": { - "protocol": "http", - "process": [] - }, - "wifi/clients/+/sessions/+/session_end": { - "protocol": "http", - "process": [] - }, - "wifi/clients/+/sessions/+/rx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "wifi/clients/+/sessions/+/tx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "wifi/clients/+/sessions/+/signal": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5 - } - ] - }, - "wifi/hp/+/+/+/power": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 10, - "initial_val": 0 - } - ] - }, - "wifi/hp/+/+/+/channel": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "any-change" - } - ] - }, - "wifi/hp/+/+/+/noise": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 10, - "initial_val": 0 - } - ] - }, - "wifi/hp/+/+/+/rx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "wifi/hp/+/+/+/rx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "wifi/hp/+/+/+/rx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "wifi/hp/+/+/+/rx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "wifi/hp/+/+/+/tx_bytes": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "wifi/hp/+/+/+/tx_packets": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "wifi/hp/+/+/+/tx_dropped": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "wifi/hp/+/+/+/tx_errors": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": 5, - "initial_val": 0 - }, - { - "type": "DeltaValue" - } - ] - }, - "wifi/hp/+/+/+/num_sta": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": "5", - "initial_val": 0 - } - ] - }, - "wifi/num_sta": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": "5", - "initial_val": 0 - } - ] - }, - "wifi/hp/+/num_sta": { - "protocol": "http", - "process": [ - { - "type": "DiffTrigger", - "diff_method": "percent", - "threshold": "5", - "initial_val": 0 - } - ] - } - } - }, - "system": { - "report_period": 60, - "usb3_enabled": true, - "alarms": [ - { - "time": "5:00", - "repeats": "daily", - "payload": { - "name": "daily reboot", - "type": "reboot" - } - } - ] - }, - "switch": { - "report_period": 10, - "host": "172.28.100.9", - "username": "admin", - "password": "admin", - "type": "rtl8380m" - } -} diff --git a/src/configs/bigbox_v1-pr1.json b/src/configs/bigbox_v1-pr1.json deleted file mode 100644 index b1eb836a..00000000 --- a/src/configs/bigbox_v1-pr1.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "network": { - "networks": [ - { - "name": "Wired Internet", - "id": "wan", - "interface": "eth1", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Primary Modem", - "id": "mwan0", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Secondary Modem", - "id": "mwan1", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Admin Network", - "id": "adm", - "interface": "eth0", - "vlan": "1", - "protocol": "static", - "ipaddr": "172.28.1.1/24" - }, - { - "name": "Guest Network", - "id": "jng", - "interface": "eth0", - "vlan": "2", - "protocol": "static", - "ipaddr": "172.28.2.1/24" - } - ] - }, - "wifi": { - "radios": [ - { - "id": "radio1", - "type": "mac80211", - "path": "platform/ahb/18100000.wmac", - "band": "2g", - "channel": "auto", - "channels": ["1", "6", "11"], - "country": "GB", - "htmode": "HT20", - "txpower": "20" - } - ], - "ssids": [ - { - "name": "GetBox-PmWBH", - "encryption": "psk2", - "mode": "access_point", - "password": "shiny-huge-valet", - "network": "jng", - "radios": ["radio1"] - }, - { - "name": "GetBox-PmWBH-admin", - "encryption": "psk2", - "mode": "access_point", - "password": "shiny-huge-valet", - "network": "adm", - "radios": ["radio1"] - } - ] - }, - "gsm": { - "modems": { - "defaults": { - "target_state": "autoconnect" - }, - "devices": { - "internal_primary": { - "builtin": true, - "id_field": "device", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-2", - "permanent_connector": "builtin", - "permanent_connector_slot": "1", - "target_state": "autoconnect" - }, - "internal_secondary": { - "builtin": true, - "id_field": "device", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00200000.usb/xhci-hcd.0/usb1/1-1", - "permanent_connector": "builtin", - "permanent_connector_slot": "2", - "target_state": "disabled" - }, - "modem1": { - "id_field": "imei", - "imei": "123456789054321", - "sim_detect": "state_cycle", - "target_state": "autoconnect" - } - } - }, - "sims": { - "default": { - "autoconnect": true - } - }, - "connectors": { - "devices": { - "internal": { - "builtin": true, - "output_slots": ["internal_primary", "internal_secondary"], - "id_field": "specific USB identifier here if more than one switcher connected?", - "device": "if attached on usb something like /sys/devices/platform/scb/fd500000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0/usb1/1-1/1-1.2", - "enabled": true, - "always_connected": true, - "type": "bbv1_internal" - } - } - } - }, - "service2": { - "paramX": "valueX", - "paramY": "valueY" - }, - "switch": { - "report_period": 10, - "host": "172.28.100.9", - "username": "admin", - "password": "admin", - "type": "rtl8380m" - } -} diff --git a/src/configs/config.json b/src/configs/config.json deleted file mode 100644 index 1397aef9..00000000 --- a/src/configs/config.json +++ /dev/null @@ -1,141 +0,0 @@ -{ - "network": { - "networks": [ - { - "name": "Wired Internet", - "id": "wan", - "interface": "eth1", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Primary Modem", - "id": "mwan0", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Secondary Modem", - "id": "mwan1", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Admin Network", - "id": "adm", - "interface": "eth0", - "vlan": "1", - "protocol": "static", - "ipaddr": "172.28.1.1/24" - }, - { - "name": "Guest Network", - "id": "jng", - "interface": "eth0", - "vlan": "2", - "protocol": "static", - "ipaddr": "172.28.2.1/24" - } - ] - }, - "wifi": { - "radios": [ - { - "id": "radio1", - "type": "mac80211", - "path": "platform/ahb/18100000.wmac", - "band": "2g", - "channel": "auto", - "channels": ["1", "6", "11"], - "country": "GB", - "htmode": "HT20", - "txpower": "20" - } - ], - "ssids": [ - { - "name": "GetBox-PmWBH", - "encryption": "psk2", - "mode": "access_point", - "password": "shiny-huge-valet", - "network": "jng", - "radios": ["radio1"] - }, - { - "name": "GetBox-PmWBH-admin", - "encryption": "psk2", - "mode": "access_point", - "password": "shiny-huge-valet", - "network": "adm", - "radios": ["radio1"] - } - ] - }, - "gsm": { - "modems": { - "default": { - "enabled": true, - "autoconnect": true - }, - "known": { - "primary": { - "id_field": "device", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb4/4-1", - "enabled": true, - "autoconnect": true - }, - "secondary": { - "id_field": "device", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-2", - "enabled": true, - "autoconnect": true - }, - "modem1": { - "id_field": "imei", - "imei": "123456789054321", - "enabled": true, - "autoconnect": true - } - } - }, - "sims": { - }, - "connectors": { - "known": { - "builtin": { - "id_field": "specific USB identifier here if more than one switcher connected?", - "device": "if attached on usb something like /sys/devices/platform/scb/fd500000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0/usb1/1-1/1-1.2", - "enabled": true, - "always_connected": true, - "type": "bbv1_internal" - } - } - } - }, - "service2": { - "paramX": "valueX", - "paramY": "valueY" - }, - "hub": { - "ws_host": "172.19.0.2", - "ws_port": "9003", - "connections": [ - // { - // "type": "http", - // "url": "http://cloud.dev.janga.la/http/channels/{channel}/messages", - // "mainflux_id":"{ID}", - // "mainflux_key":"{KEY}", - // "mainflux_datachannel":"{DATACHANNEL_ID}" - // }, - { - "type": "fd", - "path_read": "/dev/ttyAMA0", - "path_send": "/dev/ttyAMA0" - } - ] - }, - "testdevice":{ - "fd_path": "/dev/ttyAMA0", - "interval": 10 - } -} \ No newline at end of file diff --git a/src/configs/rich_bb_test.json b/src/configs/rich_bb_test.json deleted file mode 100644 index 81d24b54..00000000 --- a/src/configs/rich_bb_test.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "network": { - "networks": [ - { - "name": "Wired Internet", - "id": "wan", - "interface": "eth1", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Primary Modem", - "id": "mwan0", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Secondary Modem", - "id": "mwan1", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Admin Network", - "id": "adm", - "interface": "eth0", - "vlan": "1", - "protocol": "static", - "ipaddr": "172.28.1.1/24" - }, - { - "name": "Guest Network", - "id": "jng", - "interface": "eth0", - "vlan": "2", - "protocol": "static", - "ipaddr": "172.28.2.1/24" - } - ] - }, - "wifi": { - "radios": [ - { - "id": "radio0", - "type": "mac80211", - "path": "platform/axi/1001100000.mmc/mmc_host/mmc1/mmc1:0001/mmc1:0001:1", - "band": "2g", - "channel": "auto", - "channels": ["1", "6", "11"], - "country": "GB", - "htmode": "HT20", - "txpower": "20" - } - ], - "ssids": [ - { - "name": "OpenWart", - "encryption": "none", - "mode": "ap", - "network": "lan", - "radios": ["radio0"] - } - ] - }, - "gsm": { - "modems": { - "default": { - "enabled": true, - "autoconnect": true - }, - "known": { - "primary": { - "id_field": "device", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb4/4-1", - "enabled": true, - "autoconnect": true, - "sim_detect": "state_cycle" - }, - "secondary": { - "id_field": "device", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-2", - "enabled": true, - "autoconnect": true, - "sim_detect": "state_cycle" - }, - "modem1": { - "id_field": "imei", - "imei": "123456789054321", - "enabled": true, - "autoconnect": true, - "sim_detect": "state_cycle" - } - } - }, - "sims": { - }, - "connectors": { - } - }, - "service2": { - "paramX": "valueX", - "paramY": "valueY" - } -} \ No newline at end of file diff --git a/src/configs/ryan-bb-dev.json b/src/configs/ryan-bb-dev.json deleted file mode 100644 index da0f3d3e..00000000 --- a/src/configs/ryan-bb-dev.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "network": { - "networks": [ - { - "name": "Wired Internet", - "id": "wan", - "interface": "eth1", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Primary Modem", - "id": "mwan0", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Secondary Modem", - "id": "mwan1", - "protocol": "dhcp", - "ipv6": true - }, - { - "name": "Admin Network", - "id": "adm", - "interface": "eth0", - "vlan": "1", - "protocol": "static", - "ipaddr": "172.28.1.1/24" - }, - { - "name": "Guest Network", - "id": "jng", - "interface": "eth0", - "vlan": "2", - "protocol": "static", - "ipaddr": "172.28.2.1/24" - } - ] - }, - "gsm": { - "modems": { - "default": { - "enabled": true - }, - "known": { - "primary": { - "id_field": "device", - "device": "/sys/devices/platform/scb/fd500000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0/usb1/1-1/1-1.2", - "enabled": true, - "autoconnect": true - }, - "secondary": { - "id_field": "device", - "device": "/sys/devices/platform/scb/fd500000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0/usb1/1-1/1-1.4", - "enabled": true, - "autoconnect": true - }, - "modem1": { - "id_field": "imei", - "imei": "123456789054321", - "enabled": true, - "autoconnect": true - }, - "ryan-tera-secondary": { - "id_field": "imei", - "imei": "867929068986514", - "enabled": true, - "autoconnect": true - }, - "ryan-tera-primary": { - "id_field": "imei", - "imei": "868371052887573", - "enabled": true, - "autoconnect": true - } - } - }, - "sims": { - "default": { - "autoconnect": true - } - } - }, - "metrics": { - "publish_cache": { - "period": 60 - }, - "collections": { - "hal/capability/modem/+": { - "field": "index", - "protocol": "log", - "process": [] - }, - "hal/capability/modem/+/info/nas/lte/rssi": { - "protocol": "log", - "process": [] - }, - "hal/capability/modem/+/info/nas/lte/rsrp": { - "protocol": "log", - "process": [] - }, - "system/info/cpu/overall_utilisation": { - "protocol": "log", - "process": [] - }, - "system/info/memory/+": { - "protocol": "log", - "process": [] - }, - "system/info/temperature": { - "protocol": "log", - "process": [] - }, - "system/info/uptime": { - "protocol": "log", - "process": [] - }, - "system/info/device/+": { - "protocol": "log", - "process": [] - }, - "system/info/firmware/version": { - "protocol": "log", - "process": [] - } - } - }, - "system": { - "report_period": 1, - "usb3_enabled": false, - "alarms": [ - { - "time": "5:00", - "repeats": "daily", - "payload": { - "name": "daily reboot", - "type": "reboot" - } - } - ] - } -} - diff --git a/src/devicecode/authz.lua b/src/devicecode/authz.lua new file mode 100644 index 00000000..93ee885c --- /dev/null +++ b/src/devicecode/authz.lua @@ -0,0 +1,312 @@ +-- devicecode/authz.lua +-- +-- Topic-aware principal and authoriser helpers. +-- +-- Design intent for this step: +-- * keep enforcement in bus.lua +-- * keep policy definition here +-- * remain simple: admin may do everything, but via a rule engine +-- * default deny for missing/unknown principals + +local trie = require 'trie' + +local M = {} + +local Authorizer = {} +Authorizer.__index = Authorizer + +-------------------------------------------------------------------------------- +-- trie.literal interoperability +-------------------------------------------------------------------------------- + +local LIT_MT = getmetatable(trie.literal('x')) + +local function is_lit(tok) + return type(tok) == 'table' and getmetatable(tok) == LIT_MT +end + +local function unwrap_token(tok) + if is_lit(tok) then + return tok.v, true + end + return tok, false +end + +-------------------------------------------------------------------------------- +-- Small helpers +-------------------------------------------------------------------------------- + +local function copy_roles(roles) + local out = {} + if type(roles) ~= 'table' then + return out + end + for i = 1, #roles do + if type(roles[i]) == 'string' and roles[i] ~= '' then + out[#out + 1] = roles[i] + end + end + return out +end + +local function has_role(principal, wanted) + local roles = principal and principal.roles + if type(roles) ~= 'table' then + return false + end + for i = 1, #roles do + if roles[i] == wanted then + return true + end + end + return false +end + +local function action_matches(rule_action, action) + if rule_action == '*' then return true end + if type(rule_action) == 'string' then + return rule_action == action + end + if type(rule_action) == 'table' then + for i = 1, #rule_action do + if rule_action[i] == '*' or rule_action[i] == action then + return true + end + end + end + return false +end + +local function role_matches(rule_roles, principal) + if rule_roles == '*' then return true end + if type(rule_roles) ~= 'table' then return false end + for i = 1, #rule_roles do + if has_role(principal, rule_roles[i]) then + return true + end + end + return false +end + +local function principal_matches(sel, principal) + if sel == '*' then return true end + if type(principal) ~= 'table' then return false end + + if type(sel) == 'function' then + return not not sel(principal) + end + + if type(sel) ~= 'table' then + return false + end + + if sel.kind ~= nil and sel.kind ~= principal.kind then + return false + end + + if sel.id ~= nil and sel.id ~= principal.id then + return false + end + + if sel.roles ~= nil and not role_matches(sel.roles, principal) then + return false + end + + return true +end + +local function topic_matches(pattern, topic, s_wild, m_wild, pi, ti) + pi = pi or 1 + ti = ti or 1 + + local pn = #pattern + local tn = #topic + + while true do + if pi > pn then + return ti > tn + end + + local p_raw, p_lit = unwrap_token(pattern[pi]) + + if not p_lit and p_raw == m_wild then + return true + end + + if ti > tn then + return false + end + + local t_raw = unwrap_token(topic[ti]) + + if (not p_lit) and p_raw == s_wild then + pi = pi + 1 + ti = ti + 1 + elseif p_raw == t_raw then + pi = pi + 1 + ti = ti + 1 + else + return false + end + end +end + +local function rule_topic_matches(rule_topic, topic, s_wild, m_wild) + if rule_topic == '*' or rule_topic == nil then + return true + end + if type(rule_topic) ~= 'table' or type(topic) ~= 'table' then + return false + end + return topic_matches(rule_topic, topic, s_wild, m_wild) +end + +local function effect_of(rule) + if rule and rule.effect == 'deny' then + return 'deny' + end + return 'allow' +end + +-------------------------------------------------------------------------------- +-- Principal constructors +-------------------------------------------------------------------------------- + +---@param name string +---@param opts? { roles?: string[] } +---@return table +function M.service_principal(name, opts) + opts = opts or {} + local roles = copy_roles(opts.roles or { 'admin' }) + return { + kind = 'service', + id = tostring(name), + roles = roles, + } +end + +---@param id string +---@param opts? { roles?: string[] } +---@return table +function M.user_principal(id, opts) + opts = opts or {} + return { + kind = 'user', + id = tostring(id), + roles = copy_roles(opts.roles or {}), + } +end + +---@param id string +---@param opts? { roles?: string[] } +---@return table +function M.peer_principal(id, opts) + opts = opts or {} + return { + kind = 'peer', + id = tostring(id), + roles = copy_roles(opts.roles or {}), + } +end + +-------------------------------------------------------------------------------- +-- Rule helpers +-------------------------------------------------------------------------------- + +---@param spec? table +---@return table +function M.rule(spec) + spec = spec or {} + return { + principal = spec.principal or '*', + action = spec.action or '*', + topic = spec.topic or '*', + effect = spec.effect or 'allow', + reason = spec.reason, + } +end + +---@param roles string[]|string +---@param effect? '"allow"'|'"deny"' +---@return table[] +function M.rules_for_roles(roles, effect) + if type(roles) == 'string' then roles = { roles } end + return { + M.rule { + principal = { roles = roles }, + action = '*', + topic = '*', + effect = effect or 'allow', + }, + } +end + +-------------------------------------------------------------------------------- +-- Default rules +-------------------------------------------------------------------------------- + +local function default_rules() + -- For now: + -- * admin may do anything + -- * everything else is denied by default + -- + -- The point is to get the matching machinery in place now, while keeping + -- policy broad until roles are defined properly. + return M.rules_for_roles({ 'admin' }, 'allow') +end + +-------------------------------------------------------------------------------- +-- Authoriser +-------------------------------------------------------------------------------- + +---@param ctx table +---@param rule table +---@param s_wild string|number +---@param m_wild string|number +---@return boolean +local function rule_matches(ctx, rule, s_wild, m_wild) + return principal_matches(rule.principal, ctx.principal) + and action_matches(rule.action, ctx.action) + and rule_topic_matches(rule.topic, ctx.topic, s_wild, m_wild) +end + +--- Authorisation hook for bus.lua. +--- ctx = { bus, principal, action, topic, extra } +---@param ctx table +---@return boolean|nil ok +---@return string|nil reason +function Authorizer:allow(ctx) + local p = ctx and ctx.principal or nil + if type(p) ~= 'table' then + return false, 'missing_principal' + end + + local bus = ctx and ctx.bus or nil + local s_wild = (bus and bus._s_wild) or '+' + local m_wild = (bus and bus._m_wild) or '#' + + local rules = self.rules or {} + for i = 1, #rules do + local rule = rules[i] + if rule_matches(ctx, rule, s_wild, m_wild) then + if effect_of(rule) == 'deny' then + return false, rule.reason or 'forbidden' + end + return true, nil + end + end + + return false, 'forbidden' +end + +---@param opts? { rules?: table[] } +---@return table +function M.new(opts) + opts = opts or {} + return setmetatable({ + rules = opts.rules or default_rules(), + opts = opts, + }, Authorizer) +end + +return M diff --git a/src/devicecode/blob_source.lua b/src/devicecode/blob_source.lua new file mode 100644 index 00000000..2cf8fb44 --- /dev/null +++ b/src/devicecode/blob_source.lua @@ -0,0 +1,234 @@ +---@module 'devicecode.blob_source' + +local fibers = require 'fibers' +local op = require 'fibers.op' +local resource = require 'devicecode.support.resource' + +---@class BlobSource +local BlobSource = {} +BlobSource.__index = BlobSource + +---@class BlobSink +local BlobSink = {} +BlobSink.__index = BlobSink + +---------------------------------------------------------------------- +-- String source +---------------------------------------------------------------------- + +---@class StringSource : BlobSource +---@field _data string +---@field _off integer +local StringSource = setmetatable({}, { __index = BlobSource }) +StringSource.__index = StringSource + +---@param max_bytes integer|nil +---@return Op -- (chunk:string|nil, err:string|nil) +function StringSource:read_chunk_op(max_bytes) + return op.guard(function () + if self._off >= #self._data then + return op.always(nil, nil) -- EOF + end + + local n = max_bytes or (#self._data - self._off) + if type(n) ~= 'number' or n <= 0 then + return op.always(nil, 'invalid max_bytes') + end + + local chunk = self._data:sub(self._off + 1, self._off + n) + self._off = self._off + #chunk + return op.always(chunk, nil) + end) +end + +function StringSource:close_op() + return op.always(true, nil) +end + +function StringSource:terminate(_) + return true, nil +end + +---------------------------------------------------------------------- +-- Stream source +---------------------------------------------------------------------- + +---@class StreamSource : BlobSource +---@field _stream Stream +local StreamSource = setmetatable({}, { __index = BlobSource }) +StreamSource.__index = StreamSource + +---@param max_bytes integer|nil +---@return Op -- (chunk:string|nil, err:string|nil) +function StreamSource:read_chunk_op(max_bytes) + local n = max_bytes or 65536 + return self._stream:read_some_op(n):wrap(function (chunk, err) + return chunk, err + end) +end + +function StreamSource:close_op() + if not self._stream or type(self._stream.close_op) ~= 'function' then + return op.always(true, nil) + end + return self._stream:close_op() +end + +function StreamSource:terminate(reason) + return resource.terminate(self._stream, reason) +end + +---------------------------------------------------------------------- +-- Memory sink +---------------------------------------------------------------------- + +---@class MemorySink : BlobSink +---@field _parts string[] +local MemorySink = setmetatable({}, { __index = BlobSink }) +MemorySink.__index = MemorySink + +---@param chunk string +---@return Op -- (ok:boolean|nil, err:string|nil) +function MemorySink:write_chunk_op(chunk) + return op.guard(function () + if type(chunk) ~= 'string' then + return op.always(nil, 'chunk must be a string') + end + self._parts[#self._parts + 1] = chunk + return op.always(true, nil) + end) +end + +function MemorySink:close_op() + return op.always(true, nil) +end + +function MemorySink:terminate(_) + return true, nil +end + +function MemorySink:result() + return table.concat(self._parts) +end + +---------------------------------------------------------------------- +-- Stream sink +---------------------------------------------------------------------- + +---@class StreamSink : BlobSink +---@field _stream Stream +local StreamSink = setmetatable({}, { __index = BlobSink }) +StreamSink.__index = StreamSink + +---@param chunk string +---@return Op -- (ok:boolean|nil, err:string|nil) +function StreamSink:write_chunk_op(chunk) + return self._stream:write_op(chunk):wrap(function (n, err) + if n == nil then + return nil, err + end + return true, nil + end) +end + +function StreamSink:close_op() + if not self._stream or type(self._stream.close_op) ~= 'function' then + return op.always(true, nil) + end + return self._stream:close_op() +end + +function StreamSink:terminate(reason) + return resource.terminate(self._stream, reason) +end + +---------------------------------------------------------------------- +-- Structured copy +---------------------------------------------------------------------- + +---@param source BlobSource +---@param sink BlobSink +---@param opts? { chunk_size?: integer, close_source?: boolean, close_sink?: boolean } +---@return Op -- yields st, rep, bytes_or_primary via run_scope_op +local function copy_op(source, sink, opts) + opts = opts or {} + local chunk_size = opts.chunk_size or 65536 + local close_source = opts.close_source ~= false + local close_sink = opts.close_sink ~= false + + return fibers.run_scope_op(function (scope) + local bytes = 0 + + scope:finally(function (_, status, primary) + local reason = primary or status or 'blob copy closed' + if close_source then + resource.terminate_checked(source, reason, 'blob copy source cleanup failed') + end + if close_sink then + resource.terminate_checked(sink, reason, 'blob copy sink cleanup failed') + end + end) + + while true do + local chunk, rerr = fibers.perform(source:read_chunk_op(chunk_size)) + if rerr ~= nil then + error(rerr, 0) + end + if chunk == nil then + return bytes + end + + local ok, werr = fibers.perform(sink:write_chunk_op(chunk)) + if ok == nil then + error(werr or 'write failed', 0) + end + + bytes = bytes + #chunk + end + end) +end + +---------------------------------------------------------------------- +-- Constructors +---------------------------------------------------------------------- + +local M = {} + +---@param data string +---@return BlobSource +function M.from_string(data) + assert(type(data) == 'string', 'from_string: data must be string') + return setmetatable({ + _data = data, + _off = 0, + }, StringSource) +end + +---@param stream Stream +---@return BlobSource +function M.from_stream(stream) + return setmetatable({ + _stream = stream, + }, StreamSource) +end + +---@return BlobSink +function M.to_memory() + return setmetatable({ + _parts = {}, + }, MemorySink) +end + +---@param stream Stream +---@return BlobSink +function M.to_stream(stream) + return setmetatable({ + _stream = stream, + }, StreamSink) +end + +M.copy_op = copy_op +M.BlobSource = BlobSource +M.BlobSink = BlobSink + +return M diff --git a/src/devicecode/main.lua b/src/devicecode/main.lua new file mode 100644 index 00000000..b85b9e61 --- /dev/null +++ b/src/devicecode/main.lua @@ -0,0 +1,313 @@ +-- devicecode/main.lua +-- +-- Main runtime entrypoint logic. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local authz = require 'devicecode.authz' +local busmod = require 'bus' + +local safe = require 'coxpcall' + +local EXIT_GRACE_PERIOD = 60.0 + +local M = {} + +local function require_env(name) + local v = os.getenv(name) + if not v or v == '' then + error(('missing required environment variable %s'):format(name), 2) + end + return v +end + +local function parse_csv(s) + local out = {} + for part in tostring(s):gmatch('[^,%s]+') do + out[#out + 1] = part + end + return out +end + +local function assert_unique_services(names) + local seen = {} + for i = 1, #names do + local name = names[i] + if seen[name] then + error(('DEVICECODE_SERVICES contains duplicate service name %s'):format(tostring(name)), 2) + end + seen[name] = true + end +end + +local function move_to_front(list, wanted) + local out = {} + for i = 1, #list do + if list[i] == wanted then + out[#out + 1] = list[i] + end + end + for i = 1, #list do + if list[i] ~= wanted then + out[#out + 1] = list[i] + end + end + return out +end + +local function cleanup_child_scope(child, reason) + if not child then return end + child:cancel(reason or 'cleanup') +end + +local function spawn_service(child, bus, name, mod, env, extra_opts) + return child:spawn(function() + local conn = bus:connect({ + principal = authz.service_principal(name), + }) + + local function connect_as(principal) + return bus:connect({ + principal = principal, + }) + end + + mod.start(conn, { + name = name, + env = env, + connect = connect_as, + services = extra_opts and extra_opts.services or nil, + run_http = extra_opts and extra_opts.run_http or nil, + verify_login = extra_opts and extra_opts.verify_login or nil, + }) + + error(('service returned unexpectedly: %s'):format(tostring(name)), 0) + end) +end + +local function build_bus() + return busmod.new({ + q_length = 10, + full = 'drop_oldest', + s_wild = '+', + m_wild = '#', + authoriser = authz.new(), + }) +end + +local function now() + return fibers.now() +end + +local function retain_main_state(conn, status, fields) + local payload = { + status = status, + t = now(), + } + if fields then + for k, v in pairs(fields) do + payload[k] = v + end + end + conn:retain({ 'obs', 'state', 'main' }, payload) +end + +local function retain_service_state(conn, name, status, fields) + local payload = { + service = name, + status = status, + t = now(), + } + if fields then + for k, v in pairs(fields) do + payload[k] = v + end + end + conn:retain({ 'obs', 'state', 'service', name }, payload) +end + +local function load_service(service_loader, name) + local ok, mod = safe.pcall(service_loader, name) + if not ok then + return nil, mod + end + if type(mod) ~= 'table' or type(mod.start) ~= 'function' then + return nil, 'service module must export start(conn, opts)' + end + return mod, nil +end + +local function fail_boot(main_conn, service_name, what, err, extra) + if service_name then + retain_service_state(main_conn, service_name, 'failed', { + what = what, + err = tostring(err), + }) + end + + local payload = { + what = what, + err = tostring(err), + } + if service_name then + payload.service = service_name + end + if extra then + for k, v in pairs(extra) do + payload[k] = v + end + end + + retain_main_state(main_conn, 'failed', payload) + main_conn:publish({ 'obs', 'log', 'main', 'error' }, payload) + + if service_name then + error(('boot failed for service %s: %s'):format(tostring(service_name), tostring(err)), 0) + else + error(('boot failed: %s'):format(tostring(err)), 0) + end +end + +function M.run(scope, params) + params = params or {} + + local env = params.env or (os.getenv('DEVICECODE_ENV') or 'dev') + local exit_grace_period = params.exit_grace_period or EXIT_GRACE_PERIOD + + local service_names = parse_csv(params.services_csv or require_env('DEVICECODE_SERVICES')) + if #service_names == 0 then + error('DEVICECODE_SERVICES must contain at least one service name', 2) + end + + assert_unique_services(service_names) + + -- Start monitor first if it is present. + service_names = move_to_front(service_names, 'monitor') + + local bus = params.bus or build_bus() + local service_loader = params.service_loader or function(name) + return require('services.' .. name) + end + local service_opts = params.service_opts or {} + + local main_conn = bus:connect({ + principal = authz.service_principal('main'), + }) + + retain_main_state(main_conn, 'starting', { + env = env, + services = service_names, + }) + + main_conn:publish({ 'obs', 'log', 'main', 'info' }, { + what = 'starting', + env = env, + }) + + local services = {} + + for i = 1, #service_names do + local name = service_names[i] + + retain_service_state(main_conn, name, 'starting') + main_conn:publish({ 'obs', 'event', 'main', 'spawn' }, { service = name, t = now() }) + + local mod, lerr = load_service(service_loader, name) + if not mod then + fail_boot(main_conn, name, 'load_failed', lerr) + end + + local child, cerr = scope:child() + if not child then + fail_boot(main_conn, name, 'child_scope_failed', cerr) + end + + local ok_spawn, serr = spawn_service(child, bus, name, mod, env, service_opts[name]) + if not ok_spawn then + cleanup_child_scope(child, 'spawn_failed') + fail_boot(main_conn, name, 'spawn_failed', serr) + end + + retain_service_state(main_conn, name, 'running') + services[#services + 1] = { name = name, scope = child } + end + + retain_main_state(main_conn, 'running', { + env = env, + services = service_names, + }) + + scope:spawn(function() + local ev = nil + + for i = 1, #services do + local rec = services[i] + local one = rec.scope:not_ok_op():wrap(function(st, primary) + return rec, st, primary + end) + ev = ev and op.choice(ev, one) or one + end + + if not ev then + retain_main_state(main_conn, 'failed', { + what = 'no_services_started', + }) + scope:cancel('no_services_started') + return + end + + local rec, _, _ = fibers.perform(ev) + local svc = rec.name + + local jst, report, jprimary = fibers.perform(rec.scope:join_op()) + + retain_service_state(main_conn, svc, jst, { + primary = tostring(jprimary), + report = report, + }) + + retain_main_state(main_conn, 'failed', { + what = 'service_not_ok', + service = svc, + status = jst, + primary = tostring(jprimary), + }) + + main_conn:publish({ 'obs', 'event', 'main', 'service_exit' }, { + service = svc, + status = jst, + primary = tostring(jprimary), + report = report, + t = now(), + }) + + main_conn:publish({ 'obs', 'log', 'main', (jst == 'failed') and 'error' or 'warn' }, { + what = 'service_not_ok', + service = svc, + status = jst, + primary = tostring(jprimary), + }) + + sleep.sleep(exit_grace_period) + + scope:cancel(('service_not_ok:%s'):format(tostring(svc))) + end) + + main_conn:publish({ 'obs', 'log', 'main', 'info' }, { + what = 'services_spawned', + n = #services, + }) + + local n = 0 + while true do + n = n + 1 + main_conn:publish({ 'obs', 'event', 'main', 'tick' }, { + n = n, + t = now(), + }) + sleep.sleep(10.0) + end +end + +return M diff --git a/src/devicecode/service_base.lua b/src/devicecode/service_base.lua new file mode 100644 index 00000000..d4057a83 --- /dev/null +++ b/src/devicecode/service_base.lua @@ -0,0 +1,441 @@ +-- devicecode/service_base.lua +-- +-- Small service scaffold: +-- * obs helpers (legacy + v1) +-- * svc//status retained +-- * svc//meta retained +-- * svc//announce retained +-- * service run_id +-- * convenience lifecycle helpers +-- * best-effort retained-topic cleanup on scope exit +-- +---@module 'devicecode.service_base' + +local fibers = require 'fibers' +local runtime = require 'fibers.runtime' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' + +local tablex = require 'shared.table' + +local ok_uuid, uuid = pcall(require, 'uuid') + +local M = {} + +local LOG_LEVELS = { trace = true, debug = true, info = true, warn = true, error = true, fatal = true } + +local function normalise_log_level(level) + level = tostring(level or 'info'):lower() + if level == 'warning' then level = 'warn' end + if LOG_LEVELS[level] then return level end + return 'info' +end + + +local function t(...) return { ... } end + +local function wall() + return os.date('%Y-%m-%d %H:%M:%S') +end + +local function topic_to_string(topic) + if type(topic) ~= 'table' then return tostring(topic) end + local parts = {} + for i = 1, #topic do parts[#parts + 1] = tostring(topic[i]) end + return table.concat(parts, '/') +end + +local function new_run_id() + if ok_uuid and uuid and type(uuid.new) == 'function' then + return tostring(uuid.new()) + end + return ('run-%d-%d'):format(os.time(), math.random(1, 1000000)) +end + +local shallow_copy = tablex.shallow_copy + +local function merge_payload(base, extra) + local out = shallow_copy(base) + if type(extra) == 'table' then + for k, v in pairs(extra) do + out[k] = v + end + end + return out +end + +local function sorted_keys(t) + local keys = {} + for k in pairs(t or {}) do keys[#keys + 1] = k end + table.sort(keys, function(a, b) return tostring(a) < tostring(b) end) + return keys +end + +local function stable_status_value(v, key) + if key == 'ts' or key == 'at' then return '' end + if type(v) ~= 'table' then return tostring(v) end + local out = { '{' } + for _, k in ipairs(sorted_keys(v)) do + if k ~= 'ts' and k ~= 'at' then + out[#out + 1] = tostring(k) + out[#out + 1] = '=' + out[#out + 1] = stable_status_value(v[k], k) + out[#out + 1] = ';' + end + end + out[#out + 1] = '}' + return table.concat(out) +end + +local function default_ready_predicate(payload, opts) + if type(payload) ~= 'table' then return false end + if payload.ready == true then return true end + if opts and opts.accept_running_without_ready and payload.state == 'running' and payload.ready == nil then + return true + end + return false +end + +---@param conn any +---@param opts? { name?: string, env?: string, meta?: table, announce?: table } +---@return ServiceBase +function M.new(conn, opts) + opts = opts or {} + + ---@class ServiceBase + local svc = {} + + svc.conn = conn + svc.name = opts.name or 'service' + svc.env = opts.env or (os.getenv('DEVICECODE_ENV') or 'dev') + svc.run_id = new_run_id() + + -- Track retained topics so we can clean them up on scope exit. + svc._retained_topics = {} + svc._announce_published = false + svc._meta_published = false + svc._lifecycle_state = nil + svc._lifecycle_extra = nil + svc._status_semantic_key = nil + + function svc:now() return runtime.now() end + function svc:wall() return wall() end + function svc:t(...) return t(...) end + function svc:topic_to_string(topic) return topic_to_string(topic) end + + ---------------------------------------------------------------------- + -- Service topics + ---------------------------------------------------------------------- + + function svc:service_topic(kind) + return t('svc', self.name, kind) + end + + function svc:status_topic() + return self:service_topic('status') + end + + function svc:meta_topic() + return self:service_topic('meta') + end + + function svc:announce_topic() + return self:service_topic('announce') + end + + ---------------------------------------------------------------------- + -- Observability topics + ---------------------------------------------------------------------- + + function svc:obs_log_legacy_topic(level) + return t('obs', 'log', self.name, level) + end + + function svc:obs_event_legacy_topic(name) + return t('obs', 'event', self.name, name) + end + + function svc:obs_state_legacy_topic(name) + return t('obs', 'state', self.name, name) + end + + function svc:obs_event_topic(name) + return t('obs', 'v1', self.name, 'event', name) + end + + function svc:obs_metric_topic(name) + return t('obs', 'v1', self.name, 'metric', name) + end + + function svc:obs_counter_topic(name) + return t('obs', 'v1', self.name, 'counter', name) + end + + ---------------------------------------------------------------------- + -- Internal retained helpers + ---------------------------------------------------------------------- + + function svc:_track_retained(topic) + self._retained_topics[topic_to_string(topic)] = topic + end + + function svc:_retain(topic, payload) + self.conn:retain(topic, payload) + self:_track_retained(topic) + end + + function svc:_publish_dual(legacy_topic, v1_topic, payload) + self.conn:publish(legacy_topic, payload) + self.conn:publish(v1_topic, payload) + end + + function svc:_retain_dual(legacy_topic, v1_topic, payload) + self.conn:retain(legacy_topic, payload) + self.conn:retain(v1_topic, payload) + self:_track_retained(legacy_topic) + self:_track_retained(v1_topic) + end + + function svc:base_payload(extra) + return merge_payload({ + service = self.name, + env = self.env, + run_id = self.run_id, + ts = self:now(), + at = self:wall(), + }, extra) + end + + ---------------------------------------------------------------------- + -- Observability helpers + ---------------------------------------------------------------------- + + function svc:log(level, what, payload) + level = normalise_log_level(level) + + if payload == nil and type(what) == 'table' then + payload = what + what = payload.what + elseif payload == nil then + payload = { message = tostring(what or ''), summary = tostring(what or '') } + what = nil + elseif type(payload) ~= 'table' then + payload = { message = tostring(payload), summary = tostring(payload) } + end + + local out = self:base_payload(payload) + if what ~= nil and out.what == nil then out.what = tostring(what) end + if out.summary == nil and type(out.message) == 'string' then out.summary = out.message end + out.level = level + + self.conn:publish(self:obs_log_legacy_topic(level), out) + self.conn:publish(self:obs_event_topic('log'), out) + return out + end + + function svc:obs_log(level, payload) + return self:log(level, payload) + end + + function svc:trace(what, payload) return self:log('trace', what, payload) end + function svc:debug(what, payload) return self:log('debug', what, payload) end + function svc:info(what, payload) return self:log('info', what, payload) end + function svc:warn(what, payload) return self:log('warn', what, payload) end + function svc:error(what, payload) return self:log('error', what, payload) end + function svc:fatal(what, payload) return self:log('fatal', what, payload) end + + function svc:obs_event(name, payload) + self:_publish_dual( + self:obs_event_legacy_topic(name), + self:obs_event_topic(name), + payload + ) + end + + function svc:obs_state(name, payload) + -- Legacy retained state + v1 metric bridge for compatibility. + self:_retain_dual( + self:obs_state_legacy_topic(name), + self:obs_metric_topic(name), + payload + ) + end + + function svc:obs_metric(name, payload) + self:_retain(self:obs_metric_topic(name), payload) + end + + function svc:obs_counter(name, payload) + self.conn:publish(self:obs_counter_topic(name), payload) + end + + + local function retain_status_if_semantically_changed(self, payload) + local key = stable_status_value(payload, nil) + if self._status_semantic_key == key then return false end + self._status_semantic_key = key + self:_retain(self:status_topic(), payload) + self:obs_state('status', payload) + return true + end + + ---------------------------------------------------------------------- + -- Service identity / lifecycle + ---------------------------------------------------------------------- + + function svc:meta(extra) + local payload = self:base_payload(extra) + self._meta_published = true + self:_retain(self:meta_topic(), payload) + return payload + end + + function svc:announce(extra) + local payload = self:base_payload(extra) + self._announce_published = true + self:_retain(self:announce_topic(), payload) + return payload + end + + function svc:status(state, extra) + local payload = self:base_payload(merge_payload({ state = state }, extra)) + retain_status_if_semantically_changed(self, payload) + return payload + end + + function svc:lifecycle(state, extra) + local payload = self:base_payload(merge_payload({ state = state }, extra)) + local prev_state = self._lifecycle_state + local prev_ready = self._lifecycle_extra and self._lifecycle_extra.ready + self._lifecycle_state = state + self._lifecycle_extra = shallow_copy(extra) + local changed = retain_status_if_semantically_changed(self, payload) + if changed then + local level = 'debug' + if state == 'failed' then level = 'error' + elseif state == 'degraded' then level = 'warn' + elseif payload.ready == true and (prev_state ~= state or prev_ready ~= true) then level = 'info' end + self:log(level, 'service_state_changed', { + state = state, + ready = payload.ready, + previous_state = prev_state, + reason = payload.reason, + }) + end + return payload + end + + function svc:starting(extra) + return self:lifecycle('starting', merge_payload({ ready = false }, extra)) + end + + function svc:running(extra) + return self:lifecycle('running', merge_payload({ ready = false }, extra)) + end + + function svc:set_ready(is_ready, extra) + if is_ready then + return self:lifecycle('running', merge_payload({ ready = true }, extra)) + end + return self:lifecycle('running', merge_payload({ ready = false }, extra)) + end + + function svc:ready(extra) + return self:set_ready(true, extra) + end + + function svc:degraded(extra) + return self:lifecycle('degraded', merge_payload({ ready = false }, extra)) + end + + function svc:failed(reason, extra) + return self:lifecycle('failed', merge_payload({ reason = reason, ready = false }, extra)) + end + + function svc:stopped(extra) + return self:lifecycle('stopped', merge_payload({ ready = false }, extra)) + end + + ---------------------------------------------------------------------- + -- Convenience helpers + ---------------------------------------------------------------------- + + function svc:spawn_heartbeat(period_s, event_name) + period_s = period_s or 30.0 + event_name = event_name or 'tick' + + fibers.spawn(function () + local n = 0 + while true do + n = n + 1 + self:obs_event(event_name, { n = n, ts = self:now() }) + sleep.sleep(period_s) + end + end) + end + + -- Initial retained identity/public presence. + svc:meta(opts.meta) + svc:announce(opts.announce) + + local scope = fibers.current_scope and fibers.current_scope() or nil + if scope and scope.finally then + scope:finally(function () + for _, topic in pairs(svc._retained_topics) do + svc.conn:unretain(topic) + end + end) + end + + return svc +end + +--- Wait until a service reports ready/running/degraded via svc//status. +--- This is a compatibility helper for code that wants a simple blocking wait. +--- +---@param conn any +---@param service_name string +---@param opts? { timeout?: number, accept_running_without_ready?: boolean, ready_predicate?: fun(payload:any, opts:table|nil):boolean } +---@return table|nil payload +---@return string|nil err +function M.wait_service_ready(conn, service_name, opts) + opts = opts or {} + + local sub = conn:subscribe(t('svc', service_name, 'status')) + local timeout = opts.timeout + local ready_predicate = opts.ready_predicate or default_ready_predicate + + while true do + local which, a, b + + if timeout then + which, a, b = fibers.perform(op.named_choice{ + status = sub:recv_op(), + timeout = sleep.sleep_op(timeout), + }) + else + which, a, b = 'status', fibers.perform(sub:recv_op()), nil + end + + if which == 'timeout' then + sub:unsubscribe() + return nil, 'timeout' + end + + local msg, err = a, b + if not msg then + sub:unsubscribe() + return nil, tostring(err or 'subscription closed') + end + + if ready_predicate(msg.payload, opts) then + sub:unsubscribe() + return msg.payload, nil + end + end +end + +M.default_ready_predicate = default_ready_predicate +M.normalise_log_level = normalise_log_level + +return M diff --git a/src/devicecode/signal_bridge.lua b/src/devicecode/signal_bridge.lua new file mode 100644 index 00000000..e7f7538f --- /dev/null +++ b/src/devicecode/signal_bridge.lua @@ -0,0 +1,184 @@ +local sleep = require 'fibers.sleep' + +local M = {} + +local SIGNAL_NUMBERS = { + TERM = 15, + INT = 2, +} + +local function normalise_name(name) + name = tostring(name or ''):upper():gsub('^SIG', '') + return name +end + +local function load_posix_backend() + local ok_sig, psig = pcall(require, 'posix.signal') + if not ok_sig or type(psig) ~= 'table' or type(psig.signal) ~= 'function' then + return nil + end + + local ok_unistd, unistd = pcall(require, 'posix.unistd') + if not ok_unistd or type(unistd) ~= 'table' then + unistd = {} + end + + local function signum(name) + return psig['SIG' .. name] or SIGNAL_NUMBERS[name] + end + + local function set_handler(name, handler) + local old, err, eno = psig.signal(signum(name), handler) + if not old and err then + return nil, tostring(err) .. (eno and (':' .. tostring(eno)) or '') + end + return old or true + end + + local function restore(name, old) + if old == true then + old = 'default' + end + pcall(psig.signal, signum(name), old) + end + + local function hard_exit(name) + local sig = signum(name) + pcall(psig.signal, sig, 'default') + if type(psig.kill) == 'function' and type(unistd.getpid) == 'function' then + pcall(psig.kill, unistd.getpid(), sig) + end + os.exit(128 + (sig or 0)) + end + + return { + name = 'posix.signal', + set_handler = set_handler, + restore = restore, + hard_exit = hard_exit, + } +end + +local function load_nixio_backend() + local ok, nixio = pcall(require, 'nixio') + if not ok or type(nixio) ~= 'table' or type(nixio.signal) ~= 'function' then + return nil + end + + local function signum(name) + return nixio['SIG' .. name] or SIGNAL_NUMBERS[name] + end + + local function set_handler(name, handler) + -- The documented nixio API only guarantees 'ign' and 'dfl'. Some + -- builds may accept Lua callbacks; use that only when it succeeds. + local ok2, old_or_err, eno = pcall(nixio.signal, signum(name), handler) + if not ok2 or old_or_err == nil or old_or_err == false then + return nil, tostring(old_or_err) .. (eno and (':' .. tostring(eno)) or '') + end + return old_or_err or true + end + + local function restore(name, old) + if old == true then + old = 'dfl' + end + pcall(nixio.signal, signum(name), old) + end + + local function hard_exit(name) + local sig = signum(name) + pcall(nixio.signal, sig, 'dfl') + if type(nixio.kill) == 'function' and type(nixio.getpid) == 'function' then + pcall(nixio.kill, nixio.getpid(), sig) + end + os.exit(128 + (sig or 0)) + end + + return { + name = 'nixio.signal', + set_handler = set_handler, + restore = restore, + hard_exit = hard_exit, + } +end + +local function choose_backend() + return load_posix_backend() or load_nixio_backend() +end + +---Install a TERM/INT bridge from process signals to root-scope cancellation. +---The signal callback records intent only; a normal fiber performs cancellation. +---A second signal restores the default disposition and re-signals this process. +---@param scope Scope +---@param signals table|nil +---@return boolean ok +---@return string? err +function M.install(scope, signals) + assert(scope and type(scope.spawn) == 'function', 'signal_bridge.install: scope required') + + signals = signals or { TERM = true, INT = true } + + local backend = choose_backend() + if not backend then + return false, 'no supported signal backend' + end + + local pending + local seen = {} + local installed = {} + + for name, enabled in pairs(signals) do + name = normalise_name(name) + if enabled then + local function handler() + if seen[name] then + backend.hard_exit(name) + return + end + seen[name] = true + pending = name + end + + if jit and jit.off then + pcall(jit.off, handler, true) + end + + local old, err = backend.set_handler(name, handler) + if not old then + for installed_name, old_handler in pairs(installed) do + backend.restore(installed_name, old_handler) + end + return false, 'failed to install signal handler for ' .. name .. ': ' .. tostring(err) + end + installed[name] = old + end + end + + scope:finally(function() + for name, old in pairs(installed) do + backend.restore(name, old) + end + end) + + local spawned, spawn_err = scope:spawn(function() + while true do + if pending then + scope:cancel('signal:' .. tostring(pending)) + return + end + sleep.sleep(0.1) + end + end) + + if not spawned then + for name, old in pairs(installed) do + backend.restore(name, old) + end + return false, 'failed to start signal watcher: ' .. tostring(spawn_err) + end + + return true, backend.name +end + +return M diff --git a/src/devicecode/support/bus_cleanup.lua b/src/devicecode/support/bus_cleanup.lua new file mode 100644 index 00000000..0160a23c --- /dev/null +++ b/src/devicecode/support/bus_cleanup.lua @@ -0,0 +1,218 @@ +-- devicecode/support/bus_cleanup.lua +-- +-- Immediate local-bus cleanup and publication helpers. +-- +-- Fabric treats the in-process bus methods named here as immediate and +-- non-yielding. These wrappers make that assumption explicit at call sites and +-- keep finalisers/coordinators away from Op-based close or receive paths. +-- +-- This module must not call fibers.perform, op.perform_raw, sleep, close_op, or +-- any other readiness wait. + +local M = {} + +local function stringify(err, fallback) + if err == nil then + return fallback or 'bus cleanup failed' + end + return tostring(err) +end + +local function labelled_error(label, err) + local msg = stringify(err, label or 'bus cleanup failed') + if label ~= nil and msg ~= tostring(label) then + return tostring(label) .. ': ' .. msg + end + return msg +end + +local function normalise_result(a, b, fallback) + if a == false then + return nil, labelled_error(fallback, b) + end + + -- Most bus methods return true on success. Some legacy immediate cleanup + -- methods are idempotent and return no values; accept nil,nil as success. + if a == nil and b ~= nil then + return nil, labelled_error(fallback, b) + end + + return true, nil +end + +local function invoke(label, target, method, ...) + if target == nil then + return true, nil + end + + local fn + local self + + if type(target) == 'function' and method == nil then + fn = target + self = nil + elseif type(target) == 'table' and type(method) == 'string' then + fn = target[method] + self = target + elseif type(method) == 'function' then + fn = method + self = target + else + return nil, (label or 'bus_cleanup') .. ': invalid target/method' + end + + if type(fn) ~= 'function' then + return nil, (label or 'bus_cleanup') .. ': method not available' + end + + if type(target) == 'table' and type(target.close_op) == 'function' + and (method == 'close' or method == 'unsubscribe' or method == 'unbind' or method == 'unwatch') + and type(target[method]) ~= 'function' + then + return nil, (label or 'bus_cleanup') .. ': close_op-only resources are not immediate bus cleanup' + end + + local ok, a, b = pcall(function (...) + if self ~= nil then + return fn(self, ...) + end + return fn(...) + end, ...) + + if not ok then + return nil, (label or 'bus_cleanup') .. ': ' .. stringify(a, 'raised') + end + + return normalise_result(a, b, label or 'bus_cleanup') +end + +function M.call_now(label, target, method, ...) + return invoke(label, target, method, ...) +end + +function M.checked(label, target, method, ...) + local ok, err = invoke(label, target, method, ...) + if ok ~= true then + error(err or label or 'bus cleanup failed', 2) + end + return true, nil +end + +function M.publish(conn, topic, payload, opts) + return invoke('bus publish failed', conn, 'publish', topic, payload, opts) +end + +function M.retain(conn, topic, payload, opts) + return invoke('bus retain failed', conn, 'retain', topic, payload, opts) +end + +function M.unretain(conn, topic, opts) + return invoke('bus unretain failed', conn, 'unretain', topic, opts) +end + +function M.subscribe(conn, topic, opts) + if conn == nil or type(conn.subscribe) ~= 'function' then + return nil, 'bus subscribe failed: method not available' + end + + local ok, sub_or_err = pcall(function () + return conn:subscribe(topic, opts) + end) + + if not ok then + return nil, 'bus subscribe failed: ' .. stringify(sub_or_err, 'raised') + end + + if sub_or_err == nil then + return nil, 'bus subscribe failed' + end + + return sub_or_err, nil +end + +function M.bind(conn, topic, opts) + if conn == nil or type(conn.bind) ~= 'function' then + return nil, 'bus bind failed: method not available' + end + + local ok, ep_or_err = pcall(function () + return conn:bind(topic, opts) + end) + + if not ok then + return nil, 'bus bind failed: ' .. stringify(ep_or_err, 'raised') + end + + if ep_or_err == nil then + return nil, 'bus bind failed' + end + + return ep_or_err, nil +end + +function M.watch_retained(conn, topic, opts) + if conn == nil or type(conn.watch_retained) ~= 'function' then + return nil, 'bus watch_retained failed: method not available' + end + + local ok, watch_or_err = pcall(function () + return conn:watch_retained(topic, opts) + end) + + if not ok then + return nil, 'bus watch_retained failed: ' .. stringify(watch_or_err, 'raised') + end + + if watch_or_err == nil then + return nil, 'bus watch_retained failed' + end + + return watch_or_err, nil +end + +function M.unsubscribe(conn, sub) + if conn ~= nil and type(conn.unsubscribe) == 'function' then + return invoke('bus unsubscribe failed', conn, 'unsubscribe', sub) + end + return invoke('bus unsubscribe failed', sub, 'unsubscribe') +end + +function M.unwatch_retained(conn, watch) + if conn ~= nil and type(conn.unwatch_retained) == 'function' then + return invoke('bus unwatch_retained failed', conn, 'unwatch_retained', watch) + end + return invoke('bus unwatch_retained failed', watch, 'unwatch') +end + +function M.unbind(conn, ep) + if conn ~= nil and type(conn.unbind) == 'function' then + return invoke('bus unbind failed', conn, 'unbind', ep) + end + return invoke('bus unbind failed', ep, 'unbind') +end + +function M.disconnect(conn) + return invoke('bus disconnect failed', conn, 'disconnect') +end + +function M.close_feed(feed) + return invoke('bus feed close failed', feed, 'close') +end + +function M.reply(req, value) + local ok, err = invoke('bus reply failed', req, 'reply', value) + if ok ~= true then + return nil, err + end + return true, nil +end + +function M.fail(req, reason) + local ok, err = invoke('bus fail failed', req, 'fail', reason) + if ok ~= true then + return nil, err + end + return true, nil +end + +return M diff --git a/src/devicecode/support/capability_dependencies.lua b/src/devicecode/support/capability_dependencies.lua new file mode 100644 index 00000000..33b61d3d --- /dev/null +++ b/src/devicecode/support/capability_dependencies.lua @@ -0,0 +1,611 @@ +-- devicecode/support/capability_dependencies.lua +-- +-- Coordinator-facing capability dependency tracker. +-- +-- This helper observes capability status topics and turns them into local, +-- non-yielding facts for service coordinators. It deliberately does not +-- start work, retry calls, degrade services, or perform policy decisions. +-- Services should treat it as a small observable model: inspect facts, wait for +-- changed_op() or event_source(), and then admit their own scoped work. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local pulse = require 'fibers.pulse' +local cap_sdk = require 'services.hal.sdk.cap' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local queue = require 'devicecode.support.queue' +local dep_failure = require 'devicecode.support.dependency_failure' +local tablex = require 'shared.table' + +local M = {} +local Dependencies = {} +Dependencies.__index = Dependencies + +local DEFAULT_ID = 'main' + +local AVAILABLE_STATUS = { + available = true, + running = true, +} + +local function copy(v) + return tablex.deep_copy(v) +end + +local function now_fn(opts) + if opts and type(opts.now) == 'function' then return opts.now end + if type(os.clock) == 'function' then return os.clock end + return function () return 0 end +end + +local function is_non_empty_string(v) + return type(v) == 'string' and v ~= '' +end + +local function assert_key(key) + if not is_non_empty_string(key) then + error('capability_dependencies: key must be a non-empty string', 3) + end + return key +end + +local function status_available(status) + return AVAILABLE_STATUS[tostring(status or '')] == true +end + +local function format_reason(reason) + if reason == nil then return nil end + if type(reason) == 'table' then + if reason.err ~= nil then return tostring(reason.err) end + if reason.detail ~= nil then return tostring(reason.detail) end + if reason.reason ~= nil and reason.reason ~= reason then return format_reason(reason.reason) end + if reason.code ~= nil then return tostring(reason.code) end + return 'route_missing' + end + return tostring(reason) +end + +local function normalise_status_reason(payload) + if type(payload) ~= 'table' then return nil end + + local reason = payload.reason + if reason == nil then reason = payload.err end + if reason == nil then reason = payload.error end + if reason == nil then reason = payload.detail end + + return format_reason(reason) +end + +local function normalise_status_payload(payload) + if type(payload) == 'table' then + local status + if payload.state ~= nil then + status = tostring(payload.state) + elseif payload.status ~= nil then + status = tostring(payload.status) + elseif payload.available == true then + status = 'available' + elseif payload.available == false then + status = 'unavailable' + else + status = 'unavailable' + end + + local available + if payload.available ~= nil then + available = payload.available == true + else + available = status_available(status) + end + + return status, available, normalise_status_reason(payload) + end + + if payload == nil then return 'unavailable', false, nil end + if payload == true then return 'available', true, nil end + if payload == false then return 'unavailable', false, nil end + + local status = tostring(payload) + return status, status_available(status), nil +end + +local function topic_for_spec(spec) + local id = spec.id or DEFAULT_ID + if spec.raw_kind == 'host' then + return { 'raw', 'host', spec.source, 'cap', spec.class, id, 'status' } + elseif spec.raw_kind == 'member' then + return { 'raw', 'member', spec.source, 'cap', spec.class, id, 'status' } + end + return { 'cap', spec.class, id, 'status' } +end + +local function ref_for(conn, spec) + if spec.ref ~= nil then return spec.ref end + if conn == nil then return nil end + local id = spec.id or DEFAULT_ID + if spec.raw_kind == 'host' then + return cap_sdk.new_raw_host_cap_ref(conn, spec.source, spec.class, id) + elseif spec.raw_kind == 'member' then + return cap_sdk.new_raw_member_cap_ref(conn, spec.source, spec.class, id) + end + return cap_sdk.new_curated_cap_ref(conn, spec.class, id) +end + +local function subscribe_for(conn, dep, opts) + if dep.watch_status == false then return nil, nil end + + -- A supplied capability ref may be a test double or a non-bus-backed ref. + -- Prefer its own status subscription hook even when there is no bus + -- connection. A bus connection is needed only for the fallback topic + -- subscription path below. + if dep.ref ~= nil and type(dep.ref.get_status_sub) == 'function' then + local ok, sub, sub_err = pcall(function () + return dep.ref:get_status_sub({ + queue_len = dep.queue_len or opts.status_queue_len or opts.queue_len or 8, + full = dep.full or opts.status_full or opts.full or 'drop_oldest', + }) + end) + if ok and sub ~= nil then return sub, nil end + if not ok then return nil, tostring(sub or 'status subscription failed') end + return nil, tostring(sub_err or 'status subscription failed') + end + + if conn ~= nil and dep.topic ~= nil then + return bus_cleanup.subscribe(conn, dep.topic, { + queue_len = dep.queue_len or opts.status_queue_len or opts.queue_len or 8, + full = dep.full or opts.status_full or opts.full or 'drop_oldest', + }) + end + + return nil, nil +end + +local function watch_failure_policy(dep) + if dep.watch_failure ~= nil then return dep.watch_failure end + return dep.required and 'fail' or 'unavailable' +end + +local function dependency_watch_failed_error(dep, err) + return 'dependency_status_watch_failed:' .. tostring(dep.key) .. ':' .. tostring(err or 'status subscription failed') +end + +local function record_watch_failed(dep, err, now) + dep.observed_status = 'watch_failed' + dep.observed_reason = tostring(err or 'status subscription failed') + dep.effective_status = 'watch_failed' + dep.available = false + dep.reason = dep.observed_reason + dep.updated_at = now +end + +local function has_no_route(v) + return dep_failure.is_no_route(v) +end + +local function dep_public_snapshot(dep) + return { + key = dep.key, + class = dep.class, + id = dep.id, + required = dep.required, + + observed_status = dep.observed_status, + observed_reason = dep.observed_reason, + + status = dep.effective_status, + available = dep.available == true, + reason = dep.reason, + + route_missing = dep.route_missing == true, + + updated_at = dep.updated_at, + last_error = copy(dep.last_error), + } +end + +local function ordered_specs(specs) + local out = {} + for i = 1, #(specs or {}) do out[#out + 1] = specs[i] end + return out +end + +local function build_dep(conn, spec, opts, now) + if type(spec) ~= 'table' then + return nil, 'capability_dependencies.open: each spec must be a table' + end + if not is_non_empty_string(spec.key) then + return nil, 'capability_dependencies.open: spec.key must be a non-empty string' + end + if spec.ref == nil and not is_non_empty_string(spec.class) then + return nil, 'capability_dependencies.open: spec.class must be a non-empty string when ref is not supplied' + end + if (spec.raw_kind == 'host' or spec.raw_kind == 'member') and not is_non_empty_string(spec.source) then + return nil, 'capability_dependencies.open: raw capability specs require source' + end + + local dep = { + key = spec.key, + class = spec.class, + id = spec.id or DEFAULT_ID, + raw_kind = spec.raw_kind, + source = spec.source, + required = spec.required ~= false, + watch_status = spec.watch_status ~= false, + watch_failure = spec.watch_failure, + queue_len = spec.queue_len, + full = spec.full, + ref = ref_for(conn, spec), + topic = spec.topic, + observed_status = nil, + observed_available = false, + effective_status = nil, + available = false, + route_missing = false, + observed_reason = nil, + reason = nil, + last_error = nil, + updated_at = nil, + payload = nil, + } + + if dep.topic == nil and spec.class ~= nil then dep.topic = topic_for_spec(dep) end + dep.configured = dep.ref ~= nil + dep.observed_status = dep.configured and (spec.initial_status or 'configured') or 'not_configured' + dep.effective_status = dep.observed_status + dep.available = status_available(dep.effective_status) + dep.observed_available = dep.available + dep.updated_at = now() + + local sub, err = subscribe_for(conn, dep, opts) + if err ~= nil then + if watch_failure_policy(dep) == 'fail' then + return nil, dependency_watch_failed_error(dep, err) + end + record_watch_failed(dep, err, now()) + else + dep.sub = sub + end + + return dep, nil +end + +local function recompute(dep) + local observed_status = dep.observed_status or (dep.configured and 'configured' or 'not_configured') + local observed_available = dep.observed_available == true + + if dep.route_missing and observed_available then + dep.effective_status = 'route_missing' + dep.available = false + dep.reason = dep.reason or 'no_route' + return + end + + dep.effective_status = observed_status + dep.available = observed_available + + if dep.available then + dep.reason = nil + else + dep.reason = dep.observed_reason + end +end + +local function signal_if_changed(self, before) + local after = self:snapshot() + if tablex.deep_equal(before, after) then return false, self:version() end + return true, self._pulse:signal() +end + +local function record_status(self, key, payload, meta) + if self._closed then return nil, self._closed_reason or 'closed' end + local dep = self._deps[assert_key(key)] + if not dep then return nil, 'unknown_dependency' end + + local status, available, observed_reason = normalise_status_payload(payload) + local payload_copy = copy(payload) + local will_clear_route_missing = available == true and dep.route_missing == true + if dep.observed_status == status + and dep.observed_available == (available == true) + and dep.observed_reason == observed_reason + and tablex.deep_equal(dep.payload, payload_copy) + and not will_clear_route_missing + then + return dep.effective_status, dep.available == true, false, self:version() + end + + local before = self:snapshot() + + dep.observed_status = status + dep.observed_available = available == true + dep.observed_reason = observed_reason + dep.payload = payload_copy + dep.updated_at = (meta and meta.at) or self._now() + + if available == true then + dep.route_missing = false + dep.last_error = nil + dep.reason = nil + elseif dep.route_missing then + -- Explicit unavailability supersedes a previous local no_route inference. + dep.route_missing = false + dep.reason = nil + end + + recompute(dep) + local changed, version = signal_if_changed(self, before) + return dep.effective_status, dep.available == true, changed, version +end + +local function mark_route_missing(self, key, reason) + if self._closed then return nil, self._closed_reason or 'closed' end + local dep = self._deps[assert_key(key)] + if not dep then return nil, 'unknown_dependency' end + + local before = self:snapshot() + dep.route_missing = dep.observed_available == true + dep.last_error = copy(reason or 'no_route') + dep.reason = dep.route_missing and (format_reason(reason) or 'no_route') or nil + dep.updated_at = self._now() + recompute(dep) + local changed, version = signal_if_changed(self, before) + return true, version, changed +end + +local function event_from_msg(self, key, msg, err) + if msg == nil then + local status = record_status(self, key, { state = 'unavailable', reason = err or 'status_closed' }) + local dep = self._deps[assert_key(key)] + if dep ~= nil then dep.sub = nil end + return { + kind = self.closed_kind, + key = key, + status = status, + available = self:available(key), + reason = err or 'status_closed', + dependency = self:dependency(key), + } + end + + local status, available, changed, version = record_status(self, key, msg.payload, { msg = msg }) + return { + kind = self.changed_kind, + key = key, + status = status, + available = available == true, + changed = changed == true, + version = version, + dependency = self:dependency(key), + payload = msg.payload, + msg = msg, + } +end + +local function recv_op_for(self, key) + local dep = self._deps[assert_key(key)] + if not dep or not dep.sub then return op.never() end + return dep.sub:recv_op():wrap(function (msg, err) + return event_from_msg(self, key, msg, err) + end) +end + +local function try_recv_for(self, key) + local dep = self._deps[assert_key(key)] + if not dep then return nil, 'unknown_dependency' end + if not dep.sub then return nil, 'not_watched' end + local msg, err = queue.try_recv_now(dep.sub) + if msg ~= nil then return event_from_msg(self, key, msg, nil), nil end + if err == 'not_ready' then return nil, 'not_ready' end + return event_from_msg(self, key, nil, err), nil +end + +local function try_recv_any(self) + for i = 1, #self._order do + local key = self._order[i] + if self._deps[key].sub ~= nil then + local ev, err = try_recv_for(self, key) + if ev ~= nil then return ev, nil end + if err ~= 'not_ready' and err ~= 'not_watched' then return nil, err end + end + end + return nil, 'not_ready' +end + +local function recv_any_op(self) + local arms = {} + for i = 1, #self._order do + local key = self._order[i] + if self._deps[key].sub ~= nil then + arms[key] = recv_op_for(self, key) + end + end + if next(arms) == nil then return op.never() end + return fibers.named_choice(arms):wrap(function (_key, ev) + return ev + end) +end + +local function has_active_source(self) + if self._closed then return false end + for i = 1, #self._order do + if self._deps[self._order[i]].sub ~= nil then return true end + end + return false +end + +function M.status_available(status) + return status_available(status) +end + +function M.is_no_route(...) + for i = 1, select('#', ...) do + local value = select(i, ...) + if has_no_route(value) then return true end + end + return false +end + +function M.classify_call_failure(reply, err) + return dep_failure.classify_call_failure(reply, err) +end + +function M.open(conn, specs, opts) + local supplied_opts = opts or {} + local open_opts = {} + for k, v in pairs(supplied_opts) do open_opts[k] = v end + if type(specs) ~= 'table' then + return nil, 'capability_dependencies.open: specs must be a table' + end + local now = now_fn(open_opts) + + local deps = {} + local order = {} + local mgr = setmetatable({ + conn = conn, + _deps = deps, + _order = order, + _pulse = pulse.new(0), + _closed = false, + _closed_reason = nil, + _now = now, + _open_opts = open_opts, + changed_kind = open_opts.changed_kind or 'capability_dependency_changed', + closed_kind = open_opts.closed_kind or 'capability_dependency_closed', + }, Dependencies) + + for _, spec in ipairs(ordered_specs(specs)) do + local dep, err = build_dep(conn, spec, open_opts, now) + if not dep then + mgr:terminate('open_failed') + return nil, err + end + if deps[dep.key] ~= nil then + mgr:terminate('open_failed') + return nil, 'capability_dependencies.open: duplicate key ' .. tostring(dep.key) + end + deps[dep.key] = dep + order[#order + 1] = dep.key + end + + return mgr, nil +end + +function Dependencies:version() + return self._pulse:version() +end + +function Dependencies:ref(key) + local dep = self._deps[assert_key(key)] + return dep and dep.ref or nil +end + +function Dependencies:dependency(key) + local dep = self._deps[assert_key(key)] + return dep and dep_public_snapshot(dep) or nil +end + +function Dependencies:snapshot() + local out = {} + for i = 1, #self._order do + local key = self._order[i] + out[key] = dep_public_snapshot(self._deps[key]) + end + return out +end + +function Dependencies:add(spec) + if self._closed then return nil, self._closed_reason or 'closed' end + local dep, err = build_dep(self.conn, spec, self._open_opts or {}, self._now) + if not dep then return nil, err end + if self._deps[dep.key] ~= nil then + return nil, 'capability_dependencies.add: duplicate key ' .. tostring(dep.key) + end + + local before = self:snapshot() + self._deps[dep.key] = dep + self._order[#self._order + 1] = dep.key + local changed, version = signal_if_changed(self, before) + return true, nil, self:dependency(dep.key), changed, version +end + +function Dependencies:ensure(spec) + if type(spec) ~= 'table' then return nil, 'capability_dependencies.ensure: spec must be a table' end + local key = assert_key(spec.key) + if self._deps[key] ~= nil then return true, nil, self:dependency(key), false, self:version() end + return self:add(spec) +end + +-- status() is diagnostic. Coordinators should use available() for +-- admission decisions because an observed status such as 'running' may still +-- carry available=false, or may be overridden locally by route_missing. +function Dependencies:status(key) + local dep = self._deps[assert_key(key)] + return dep and dep.effective_status or 'not_configured' +end + +function Dependencies:available(key) + local dep = self._deps[assert_key(key)] + return dep and dep.available == true or false +end + + +function Dependencies:changed_op(last_seen) + if type(last_seen) ~= 'number' or last_seen < 0 or last_seen % 1 ~= 0 then + error('capability_dependencies.changed_op: last_seen must be a non-negative integer', 2) + end + return self._pulse:changed_op(last_seen):wrap(function (version, reason) + if version == nil then + return nil, nil, reason or self._closed_reason or 'closed' + end + return version, self:snapshot(), nil + end) +end + +function Dependencies:event_source(opts) + opts = opts or {} + local name = opts.name or 'capability_dependencies' + return { + name = name, + enabled = function () return has_active_source(self) end, + try_now = function () + while true do + local ev = try_recv_any(self) + if ev == nil then return nil end + if ev.changed ~= false or ev.kind == self.closed_kind then return ev end + end + end, + recv_op = function () return recv_any_op(self) end, + } +end + +function Dependencies:classify_call_failure(key, reply, err) + local class, reason = M.classify_call_failure(reply, err) + if class == 'route_missing' then + local ok, version, changed = mark_route_missing(self, key, reason or 'no_route') + return 'route_missing', reason or 'no_route', self:dependency(key), ok, version, changed + end + return class, reason, self:dependency(key), false, nil, false +end + +function Dependencies:terminate(reason) + if self._closed then return true, nil end + self._closed = true + self._closed_reason = reason or 'closed' + for i = 1, #self._order do + local dep = self._deps[self._order[i]] + if dep.sub ~= nil then + bus_cleanup.unsubscribe(self.conn, dep.sub) + dep.sub = nil + end + end + self._pulse:close(self._closed_reason) + return true, nil +end + +M.Dependencies = Dependencies +M._test = { + normalise_status_payload = normalise_status_payload, + topic_for_spec = topic_for_spec, + has_no_route = has_no_route, +} + +return M diff --git a/src/devicecode/support/config_watch.lua b/src/devicecode/support/config_watch.lua new file mode 100644 index 00000000..61ab97de --- /dev/null +++ b/src/devicecode/support/config_watch.lua @@ -0,0 +1,125 @@ +-- devicecode/support/config_watch.lua +-- +-- Shared retained cfg/ watcher for service shells. +-- +-- The helper owns only the local retained subscription and event-shaping +-- mechanics. Service modules still own validation, normalisation, generation +-- policy and effects. +-- +-- lua-bus Subscription creation replays matching retained messages before live +-- publications. That replay is the service bootstrap mechanism: a service that +-- starts after cfg/ was retained must still see the current config as +-- its first config_changed event. Do not replace this in service shells with a +-- retained-view plus subscription pair; that would reintroduce ordering races. + +local bus_cleanup = require 'devicecode.support.bus_cleanup' + +local M = {} +local Watch = {} +Watch.__index = Watch + +local function cfg_topic(service) + return { 'cfg', service } +end + +local function payload_of(msg) + return msg and msg.payload or msg +end + +local function data_of(payload) + if type(payload) == 'table' and payload.data ~= nil then + return payload.data + end + return payload +end + +local function rev_of(payload, fallback) + if type(payload) == 'table' and type(payload.rev) == 'number' then + return payload.rev + end + return fallback +end + +local function event_from_msg(self, msg, err) + if msg == nil then + return { + kind = self.closed_kind, + service = self.service, + err = err, + } + end + + local payload = payload_of(msg) + self.generation = rev_of(payload, self.generation + 1) + + return { + kind = self.changed_kind, + service = self.service, + generation = self.generation, + rev = rev_of(payload, nil), + raw = data_of(payload), + record = payload, + msg = msg, + } +end + +function M.open(conn, service, opts) + opts = opts or {} + if type(service) ~= 'string' or service == '' then + return nil, 'config_watch.open: service must be a non-empty string' + end + + local topic = opts.topic or cfg_topic(service) + -- conn:subscribe() replays retained cfg messages. Keep this as the single + -- bootstrap path used by modern service shells. + local sub, err = bus_cleanup.subscribe(conn, topic, { + queue_len = opts.queue_len or opts.config_queue_len or 4, + full = opts.full or 'reject_newest', + }) + if not sub then return nil, err or 'config subscribe failed' end + + return setmetatable({ + conn = conn, + service = service, + topic = topic, + sub = sub, + generation = opts.initial_generation or 0, + changed_kind = opts.changed_kind or 'config_changed', + closed_kind = opts.closed_kind or 'config_closed', + }, Watch), nil +end + +function Watch:recv_op() + return self.sub:recv_op():wrap(function (msg, err) + return event_from_msg(self, msg, err) + end) +end + +function Watch:try_recv_now() + local queue = require 'devicecode.support.queue' + local msg, err = queue.try_recv_now(self.sub) + if msg ~= nil then return event_from_msg(self, msg, nil) end + if err ~= 'not_ready' then return event_from_msg(self, nil, err) end + return nil +end + +function Watch:close() + if self.sub then + local sub = self.sub + self.sub = nil + return bus_cleanup.unsubscribe(self.conn, sub) + end + return true, nil +end + +function M.topic(service) + return cfg_topic(service) +end + +M._test = { + data_of = data_of, + rev_of = rev_of, + payload_of = payload_of, +} + +return M diff --git a/src/devicecode/support/contracts.lua b/src/devicecode/support/contracts.lua new file mode 100644 index 00000000..62ea4792 --- /dev/null +++ b/src/devicecode/support/contracts.lua @@ -0,0 +1,76 @@ +-- devicecode/support/contracts.lua +-- +-- Application/runtime contracts for fibers, bus and finaliser-safe resources. + +local validate = require 'shared.validate' + +local M = {} + +local function fail(name, msg, level) + error((name or 'value') .. ' ' .. msg, (level or 1) + 1) +end + +function M.is_op(v) + return type(v) == 'table' and type(v.perform) == 'function' +end + +function M.require_op(v, name, level) + if not M.is_op(v) then fail(name, 'must be an Op', (level or 1) + 1) end + return v +end + +function M.is_mailbox_rx(v) + return type(v) == 'table' and type(v.recv_op) == 'function' +end + +function M.require_rx(v, name, level) + if not M.is_mailbox_rx(v) then fail(name or 'rx', 'must provide recv_op()', (level or 1) + 1) end + return v +end + +function M.is_mailbox_tx(v) + return type(v) == 'table' and type(v.send_op) == 'function' +end + +function M.require_tx(v, name, level) + if not M.is_mailbox_tx(v) then fail(name or 'tx', 'must provide send_op(value)', (level or 1) + 1) end + return v +end + +function M.has_terminate(v) + return type(v) == 'table' and type(v.terminate) == 'function' +end + +function M.require_terminate(v, name, level) + if not M.has_terminate(v) then fail(name, 'must provide terminate(reason)', (level or 1) + 1) end + return v +end + +function M.has_close_op(v) + return type(v) == 'table' and type(v.close_op) == 'function' +end + +function M.require_close_op(v, name, level) + if not M.has_close_op(v) then fail(name, 'must provide close_op()', (level or 1) + 1) end + return v +end + +function M.has_stream_contract(v) + return type(v) == 'table' + and type(v.read_some_op) == 'function' + and type(v.write_all_op) == 'function' + and (type(v.terminate) == 'function' or type(v.close_op) == 'function') +end + +function M.require_stream_contract(v, name, level) + if not M.has_stream_contract(v) then fail(name, 'must provide the stream contract', (level or 1) + 1) end + return v +end + +M.require_table = validate.table +M.require_function = validate.function_ +M.require_positive_integer = validate.positive_integer +M.require_non_negative_integer = validate.non_negative_integer +M.require_positive_number = validate.positive_number + +return M diff --git a/src/devicecode/support/dependency_failure.lua b/src/devicecode/support/dependency_failure.lua new file mode 100644 index 00000000..76f152d5 --- /dev/null +++ b/src/devicecode/support/dependency_failure.lua @@ -0,0 +1,96 @@ +-- devicecode/support/dependency_failure.lua +-- +-- Strict helpers for canonical dependency failures. +-- +-- Edges normalise messy backend/bus failures into this explicit shape: +-- { kind = 'dependency_failure', err = 'no_route', dependency_key = '', ... } +-- Core helpers consume that shape and a small set of direct bus call failures; +-- they do not search arbitrary reports, children, primary wrappers, or strings +-- containing diagnostic text. + +local tablex = require 'shared.table' + +local M = {} + +local function copy(v) return tablex.deep_copy(v) end + +local function non_empty(v) + return type(v) == 'string' and v ~= '' +end + +local function direct_no_route(v) + if v == 'no_route' then return true end + if type(v) ~= 'table' then return false end + return v.err == 'no_route' + or v.detail == 'no_route' + or v.reason == 'no_route' + or v.code == 'no_route' +end + +local function result_no_route(v) + return type(v) == 'table' and direct_no_route(v.result) +end + +function M.is(v) + return type(v) == 'table' and v.kind == 'dependency_failure' +end + +function M.key(v) + if M.is(v) and non_empty(v.dependency_key) then return v.dependency_key end + return nil +end + +function M.is_no_route_value(v) + return direct_no_route(v) or result_no_route(v) +end + +function M.is_no_route(...) + for i = 1, select('#', ...) do + local v = select(i, ...) + if M.is(v) then + return v.err == 'no_route' or v.code == 'no_route' + end + if M.is_no_route_value(v) then return true end + end + return false +end + +function M.no_route(dependency_key, detail, extra) + local out = { + kind = 'dependency_failure', + err = 'no_route', + dependency_key = dependency_key, + detail = copy(detail), + } + if type(extra) == 'table' then + for k, v in pairs(extra) do + if out[k] == nil then out[k] = copy(v) end + end + end + return out +end + +function M.from_no_route(dependency_key, reason, extra) + if not non_empty(dependency_key) then return nil end + if not M.is_no_route(reason) then return nil end + if M.is(reason) then + local out = copy(reason) + out.kind = 'dependency_failure' + out.err = 'no_route' + out.dependency_key = out.dependency_key or dependency_key + if type(extra) == 'table' then + for k, v in pairs(extra) do + if out[k] == nil then out[k] = copy(v) end + end + end + return out + end + return M.no_route(dependency_key, reason, extra) +end + +function M.classify_call_failure(reply, err) + if M.is_no_route(reply, err) then return 'route_missing', err or reply or 'no_route' end + return 'failure', err or reply +end + +return M diff --git a/src/devicecode/support/dependency_slot.lua b/src/devicecode/support/dependency_slot.lua new file mode 100644 index 00000000..ef3ad933 --- /dev/null +++ b/src/devicecode/support/dependency_slot.lua @@ -0,0 +1,68 @@ +-- devicecode/support/dependency_slot.lua +-- +-- Small owner-side helper for replacing, terminating and projecting a single +-- capability dependency manager field. It owns boilerplate only; service +-- coordinators still own admission policy. + +local cap_deps = require 'devicecode.support.capability_dependencies' + +local M = {} + +local function field_name(field) + if type(field) ~= 'string' or field == '' then + error('dependency_slot: field must be a non-empty string', 3) + end + return field +end + +function M.terminate(owner, field, reason) + field = field_name(field) + local deps = owner and owner[field] or nil + if deps and type(deps.terminate) == 'function' then + deps:terminate(reason or (field .. '_closed')) + end + if owner then owner[field] = nil end + return true, nil +end + +function M.open(owner, field, conn, specs, opts) + field = field_name(field) + M.terminate(owner, field, (opts and opts.replace_reason) or (field .. '_replaced')) + if not specs or #specs == 0 then return true, nil, nil end + local deps, err = cap_deps.open(conn, specs, opts or {}) + if not deps then return nil, err end + owner[field] = deps + return true, nil, deps +end + +function M.replace(owner, field, conn, specs, opts) + return M.open(owner, field, conn, specs, opts) +end + +function M.snapshot(owner, field) + local deps = owner and owner[field] or nil + if deps and type(deps.snapshot) == 'function' then return deps:snapshot() end + return {} +end + +function M.event_source(owner, field, opts) + local deps = owner and owner[field] or nil + if deps and type(deps.event_source) == 'function' then return deps:event_source(opts or {}) end + return nil +end + +function M.available(owner, field, key) + local deps = owner and owner[field] or nil + return deps ~= nil and deps:available(key) == true +end + +function M.all_available(owner, field, specs) + local deps = owner and owner[field] or nil + if deps == nil then return true end + for _, spec in ipairs(specs or {}) do + if deps:available(spec.key) ~= true then return false end + end + return true +end + +return M diff --git a/src/devicecode/support/model.lua b/src/devicecode/support/model.lua new file mode 100644 index 00000000..07dc5caf --- /dev/null +++ b/src/devicecode/support/model.lua @@ -0,0 +1,131 @@ +-- devicecode/support/model.lua +-- +-- Pulse-backed observable state model. Models are state only: they expose +-- snapshot/version/change Ops and immediate termination; they do not perform +-- Ops, publish, call services or own backend work. + +local pulse = require 'fibers.pulse' +local tablex = require 'shared.table' + +local M = {} +local Model = {} +Model.__index = Model + +local function default_copy(v) + return tablex.deep_copy(v) +end + +local function default_equals(a, b) + return tablex.deep_equal(a, b) +end + +local function assert_integer(n, name, level) + if type(n) ~= 'number' or n < 0 or n % 1 ~= 0 then + error(name .. ' must be a non-negative integer', (level or 1) + 1) + end +end + +function Model:version() + return self._pulse:version() +end + +function Model:is_closed() + return self._closed +end + +function Model:is_terminated() + return self._closed +end + +function Model:why() + return self._closed_reason +end + +function Model:termination_reason() + return self._closed_reason +end + +function Model:snapshot() + return self._copy(self._snapshot) +end + +function Model:set_snapshot(next_snapshot) + if self._closed then + return nil, self._closed_reason or 'closed' + end + + local copied = self._copy(next_snapshot) + if self._equals(self._snapshot, copied) then + return false, self:version() + end + + self._snapshot = copied + local v = self._pulse:signal() + return true, v +end + +function Model:update(f) + if type(f) ~= 'function' then + error((self._label or 'model') .. ':update expects a function', 2) + end + if self._closed then + return nil, self._closed_reason or 'closed' + end + + local current = self:snapshot() + local next_snapshot = f(current) + if next_snapshot == nil then next_snapshot = current end + return self:set_snapshot(next_snapshot) +end + +function Model:changed_op(last_seen) + assert_integer(last_seen, (self._label or 'model') .. '.changed_op: last_seen', 2) + + return self._pulse:changed_op(last_seen):wrap(function (version, reason) + if version == nil then + return nil, nil, reason or self._closed_reason or 'closed' + end + return version, self:snapshot(), nil + end) +end + +function Model:terminate(reason) + if self._closed then + if self._closed_reason == nil and reason ~= nil then + self._closed_reason = reason + end + return true + end + + self._closed = true + self._closed_reason = reason or 'closed' + self._pulse:close(self._closed_reason) + return true +end + +function M.new(initial, opts) + opts = opts or {} + if opts.copy ~= nil and type(opts.copy) ~= 'function' then + error('model.new: opts.copy must be a function', 2) + end + if opts.equals ~= nil and type(opts.equals) ~= 'function' then + error('model.new: opts.equals must be a function', 2) + end + + local copy_fn = opts.copy or default_copy + return setmetatable({ + _snapshot = copy_fn(initial or {}), + _pulse = pulse.new(0), + _copy = copy_fn, + _equals = opts.equals or default_equals, + _label = opts.label or 'model', + _closed = false, + _closed_reason = nil, + }, Model) +end + +M.Model = Model +M.default_copy = default_copy +M.default_equals = default_equals + +return M diff --git a/src/devicecode/support/priority_event.lua b/src/devicecode/support/priority_event.lua new file mode 100644 index 00000000..d8fd486d --- /dev/null +++ b/src/devicecode/support/priority_event.lua @@ -0,0 +1,191 @@ +-- devicecode/support/priority_event.lua +-- +-- Deterministic semantic event selection for the few places where readiness +-- order is not enough. +-- +-- Fibers choice combinators select a ready operation. They are deliberately not +-- priority mechanisms. This helper keeps that contract: callers provide an +-- explicit, non-yielding selector for already-ready semantic events, then a +-- normal unordered wait used only as a wake-up source. After any wake-up, the +-- selector is run again before the coordinator commits to an event. +-- +-- Contract: +-- * select_now must be non-yielding. +-- * store_wake, when supplied, must be non-yielding. +-- * wait_op returns an Op and may use ordinary choice/named_choice. +-- * the event returned by wait_op should be stored by store_wake; the helper +-- will then re-run select_now and return whichever ready event has semantic +-- priority. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local validate = require 'shared.validate' + +local M = {} + +local function require_function(v, name, level) + return validate.function_(v, name, (level or 1) + 1) +end + +local function require_table(v, name, level) + return validate.table(v, name, (level or 1) + 1) +end + +--- Build an Op that selects ready semantic events before and after blocking. +--- +--- Shape: +--- select_now() -> ev|nil +--- wait_op() -> Op +--- store_wake(...) -- optional, stores the wait result for later selection +--- +--- If no event is ready, wait_op() is performed. Its result is only treated as +--- a wake-up. If store_wake is supplied, it is called with the wake result. The +--- helper then calls select_now() again and returns that selected event. +--- +---@param spec table +---@return Op +function M.next_op(spec) + spec = require_table(spec, 'priority_event.next_op: spec', 2) + + local select_now = require_function(spec.select_now, 'priority_event.next_op: select_now', 2) + local wait_op = require_function(spec.wait_op, 'priority_event.next_op: wait_op', 2) + local store_wake = spec.store_wake + if store_wake ~= nil then + require_function(store_wake, 'priority_event.next_op: store_wake', 2) + end + + local label = spec.label or 'priority_event.next_op' + local allow_no_event = not not spec.allow_no_event + + return op.guard(function () + local ev = select_now() + if ev ~= nil then + return op.always(ev) + end + + return wait_op():wrap(function (...) + if store_wake ~= nil then + store_wake(...) + end + + local selected = select_now() + if selected ~= nil then + return selected + end + + if allow_no_event then + return nil + end + + error(label .. ': wake produced no selectable event', 0) + end) + end) +end + +--- Store a consumed wake-up event by source name. +--- +--- This is suitable for named_choice arms that return one semantic event: +--- fibers.named_choice{ a = rx_a:recv_op():wrap(map_a), ... } +--- +--- The key prepended by named_choice becomes the pending bucket. +--- +---@param pending table +---@return function +function M.store_named_event(pending) + pending = require_table(pending, 'priority_event.store_named_event: pending', 2) + + return function (name, ev) + if name ~= nil and ev ~= nil then + pending[name] = ev + end + end +end + +--- Take a pending event, if present. +---@param pending table +---@param name any +---@return any ev +function M.take_pending(pending, name) + if type(pending) ~= 'table' then + return nil + end + + local ev = pending[name] + if ev ~= nil then + pending[name] = nil + end + return ev +end + +--- Convenience helper for queue-like event sources. +--- +--- sources are checked in array order. Each source has: +--- name = pending bucket / named_choice arm key +--- try_now = function() -> ev|nil +--- recv_op = function() -> Op returning ev +--- enabled = optional function() -> boolean +--- +--- The helper stores a consumed blocking wake under its source name, then +--- re-runs the priority selector. The caller owns the source-specific mapping +--- from queue item/closure to semantic event. +--- +---@param spec table +---@return Op +function M.sources_op(spec) + spec = require_table(spec, 'priority_event.sources_op: spec', 2) + + local sources = require_table(spec.sources, 'priority_event.sources_op: sources', 2) + local pending = spec.pending or {} + local label = spec.label or 'priority_event.sources_op' + + local function select_now() + for _, source in ipairs(sources) do + local name = source.name + if name == nil then + error(label .. ': source missing name', 0) + end + + if source.enabled == nil or source.enabled() then + local ev = M.take_pending(pending, name) + if ev ~= nil then + return ev + end + + if source.try_now ~= nil then + local ready = source.try_now() + if ready ~= nil then + return ready + end + end + end + end + + return nil + end + + local function wait_op() + local arms = {} + + for _, source in ipairs(sources) do + if source.enabled == nil or source.enabled() then + require_function(source.recv_op, label .. ': source.recv_op', 0) + arms[source.name] = source.recv_op() + end + end + + if next(arms) == nil then + return op.always('__closed__', { kind = 'priority_event_no_sources' }) + end + + return fibers.named_choice(arms) + end + + return M.next_op { + label = label, + select_now = select_now, + wait_op = wait_op, + store_wake = M.store_named_event(pending), + } +end + +return M diff --git a/src/devicecode/support/queue.lua b/src/devicecode/support/queue.lua new file mode 100644 index 00000000..8adc23a8 --- /dev/null +++ b/src/devicecode/support/queue.lua @@ -0,0 +1,128 @@ +-- devicecode/support/queue.lua +-- +-- Public queue helpers for service/coordinator code. +-- +-- These helpers deliberately use the public Op interface: +-- +-- op:or_else(function() ... end) +-- +-- They do not inspect mailbox internals or primitive internals. +-- +-- Contract: +-- * "now" means no readiness wait. +-- * "now" does not mean cancellation-atomic. +-- * calls still use scope-aware fibers.perform. +-- * fallback thunks must not yield. + +local fibers = require 'fibers' + +local unpack = rawget(table, 'unpack') or _G.unpack +local pack = rawget(table, 'pack') or function (...) + return { n = select('#', ...), ... } +end + +local M = {} + +--- Perform an Op as an immediate attempt. +--- +--- If ev is not ready, returns the supplied fallback values. +--- This must not be used to hide a wait: the fallback is selected by or_else. +--- +---@param ev Op +---@param ... any fallback values +---@return any ... +function M.try_now(ev, ...) + local fallback = pack(...) + + return fibers.perform(ev:or_else(function () + return unpack(fallback, 1, fallback.n) + end)) +end + +--- Try to send to a mailbox now. +--- +--- Return shapes follow mailbox semantics: +--- true, nil accepted +--- false, "full" rejected by bounded policy +--- nil, "closed" closed +--- nil, "would_block" would have had to wait +--- +---@param tx MailboxTx +---@param value any +---@return boolean|nil ok +---@return string|nil err +function M.try_send_now(tx, value) + local ok, err = M.try_now(tx:send_op(value), nil, 'would_block') + + if ok == true then + return true, nil + end + + if ok == false then + return false, err or 'full' + end + + return nil, err or 'closed' +end + +M.try_admit_now = M.try_send_now + +--- Required immediate admission. +--- +--- Suitable for coordinator/reporting paths where a completion event must not +--- silently disappear. The caller should fail the observing scope or apply an +--- explicit degradation policy if this returns nil. +--- +---@param tx MailboxTx +---@param value any +---@param label? string +---@return boolean|nil ok +---@return string|nil err +function M.try_admit_required(tx, value, label) + local ok, err = M.try_admit_now(tx, value) + if ok == true then + return true, nil + end + + local prefix = label or 'queue_admission_failed' + return nil, prefix .. ': ' .. tostring(err or 'closed') +end + +--- Assert required immediate admission. +--- +--- This is useful inside reporter fibres: a report failure is then an observer +--- failure, not a silent dropped completion. +--- +---@param tx MailboxTx +---@param value any +---@param label? string +---@param level? integer +---@return true +function M.assert_admit_required(tx, value, label, level) + local ok, err = M.try_admit_required(tx, value, label) + if ok ~= true then + error(err or 'queue_admission_failed', (level or 1) + 1) + end + return true +end + +--- Try to receive from a mailbox now. +--- +--- Return shapes: +--- item, nil received an item +--- nil, "not_ready" would have had to wait +--- nil, reason closed and drained +--- +---@param rx MailboxRx +---@return any item +---@return string|nil err +function M.try_recv_now(rx) + return M.try_now(rx:recv_op():wrap(function (item) + if item == nil then + return nil, tostring((rx.why and rx:why()) or 'closed') + end + return item, nil + end), nil, 'not_ready') +end + +return M diff --git a/src/devicecode/support/request_owner.lua b/src/devicecode/support/request_owner.lua new file mode 100644 index 00000000..0c06dec4 --- /dev/null +++ b/src/devicecode/support/request_owner.lua @@ -0,0 +1,113 @@ +-- devicecode/support/request_owner.lua +-- +-- Small single-resolution owner for caller-visible request objects. +-- +-- A request owner belongs inside the scope that owns the request lifetime. It +-- separates visible resolution (reply/fail/finalise) from local abandonment. +-- This keeps client-close and stale-completion paths from accidentally sending +-- protocol-visible replies. + +local M = {} + +local RequestOwner = {} +RequestOwner.__index = RequestOwner + +local function default_reply(req, value) + if type(req) == 'table' and type(req.reply) == 'function' then + return req:reply(value) + end + return false, 'request has no reply method' +end + +local function default_fail(req, reason) + if type(req) == 'table' and type(req.fail) == 'function' then + return req:fail(reason) + end + return false, 'request has no fail method' +end + +function M.new(request, opts) + opts = opts or {} + if type(opts) ~= 'table' then + error('request_owner.new: opts must be a table or nil', 2) + end + + if opts.reply ~= nil and type(opts.reply) ~= 'function' then + error('request_owner.new: opts.reply must be a function', 2) + end + if opts.fail ~= nil and type(opts.fail) ~= 'function' then + error('request_owner.new: opts.fail must be a function', 2) + end + + return setmetatable({ + _request = request, + _done = false, + _reply = opts.reply or default_reply, + _fail = opts.fail or default_fail, + }, RequestOwner) +end + +function RequestOwner:request() + return self._request +end + +function RequestOwner:done() + return not not self._done +end + +function RequestOwner:reply_once(value) + if self._done then + return false, 'request already resolved' + end + + self._done = true + + return self._reply(self._request, value) +end + +function RequestOwner:fail_once(reason) + if self._done then + return false, 'request already resolved' + end + + self._done = true + return self._fail(self._request, reason) +end + +function RequestOwner:finalise_unresolved(reason) + if self._done then + return false, 'request already resolved' + end + return self:fail_once(reason or 'request finalised') +end + +function RequestOwner:abandon_unresolved(_reason) + if self._done then + return false, 'request already resolved' + end + + self._done = true + return true, nil +end + + +function RequestOwner:caller_cancel_op() + local req = self._request + if type(req) ~= 'table' or type(req.done_op) ~= 'function' then + return nil, 'request has no done_op' + end + + return req:done_op():wrap(function (status, _value, err) + if status == 'abandoned' and not self:done() then + local reason = err or 'caller_abandoned' + self:abandon_unresolved(reason) + return reason + end + + return false + end) +end + +M.RequestOwner = RequestOwner + +return M diff --git a/src/devicecode/support/resource.lua b/src/devicecode/support/resource.lua new file mode 100644 index 00000000..613173d9 --- /dev/null +++ b/src/devicecode/support/resource.lua @@ -0,0 +1,223 @@ +-- devicecode/support/resource.lua +-- +-- Canonical finaliser-safe resource helpers. +-- +-- Finaliser-owned resources must expose: +-- terminate(reason) immediate, idempotent, non-yielding cleanup +-- +-- Graceful cleanup belongs in resource-specific close_op() paths outside +-- finalisers. This module deliberately does not call legacy close methods, +-- close_op(), perform(), sleep(), or join operations. + +local M = {} + +local function stringify_error(err, fallback) + if err == nil then + return fallback or 'resource termination failed' + end + return tostring(err) +end + +local function normalise_terminate_result(a, b) + -- terminate() is expected to return true,nil on success. A nil,nil return is + -- accepted for idempotent immediate cleanup methods. + if a == nil and b == nil then + return true, nil + end + if a == true then + return true, nil + end + if a == false or a == nil then + return nil, stringify_error(b, 'resource termination failed') + end + + -- Non-boolean truthy values are accepted as success for Lua cleanup methods + -- that return the terminated object or another sentinel. + return true, nil +end + +--- Terminate a resource through its immediate finaliser-safe method. +--- +--- Accepted shapes: +--- nil -> true +--- table with :terminate(reason) -> terminate result +--- +--- Return shape: +--- true, nil success +--- nil, err termination failed or unsupported +--- +---@param obj any +---@param reason any +---@return boolean|nil ok +---@return string|nil err +function M.terminate(obj, reason) + if obj == nil then + return true, nil + end + + if type(obj) ~= 'table' or type(obj.terminate) ~= 'function' then + return nil, 'resource has no terminate(reason) method' + end + + local ok, a, b = pcall(function () + return obj:terminate(reason) + end) + + if not ok then + return nil, stringify_error(a, 'resource termination raised') + end + + return normalise_terminate_result(a, b) +end + +--- Terminate and raise on failure. Suitable for scope finalisers. +function M.terminate_checked(obj, reason, label) + local ok, err = M.terminate(obj, reason) + if ok ~= true then + error((label or 'resource termination failed') .. ': ' .. tostring(err), 2) + end + return true +end + +--- Install a standard finaliser for a finaliser-safe resource. +--- +--- The returned owner is handoff-friendly: callers that transfer ownership can +--- call :handoff(...) or :detach() on it, leaving the finaliser installed but +--- inert. +function M.finally(scope, obj, label) + if type(scope) ~= 'table' or type(scope.finally) ~= 'function' then + error('resource.finally: scope required', 2) + end + + local owner = M.owned(obj, { label = label }) + + scope:finally(function (_, status, primary) + owner:terminate_checked(primary or status or 'terminated', label) + end) + + return owner +end + +------------------------------------------------------------------------------- +-- Explicit ownership / handoff +------------------------------------------------------------------------------- + +local Owned = {} +Owned.__index = Owned + +local function terminate_owned_value(self, reason) + local value = self._value + if not self._owned then + return true, nil + end + + -- Disable ownership before cleanup so finalisers remain idempotent even if + -- the cleanup method reports failure. The caller still receives that failure. + self._owned = false + self._value = nil + + local terminate_fn = self._terminate + if terminate_fn then + local ok, a, b = pcall(function () + return terminate_fn(value, reason) + end) + if not ok then + return nil, stringify_error(a, 'resource termination raised') + end + return normalise_terminate_result(a, b) + end + + return M.terminate(value, reason) +end + +--- Create an owned finaliser-safe resource lease. Ownership is released only by +--- :handoff(), :detach(), or :terminate(). +--- +--- opts may be: +--- nil -> use resource.terminate(value, reason) +--- { terminate = function, label = ? } -> resource-specific immediate cleanup +function M.owned(value, opts) + local terminate_fn + local label + + if type(opts) == 'table' then + terminate_fn = opts.terminate + label = opts.label + if terminate_fn ~= nil and type(terminate_fn) ~= 'function' then + error('resource.owned: opts.terminate must be a function', 2) + end + elseif opts ~= nil then + error('resource.owned: opts must be a table or nil', 2) + end + + return setmetatable({ + _value = value, + _owned = true, + _terminate = terminate_fn, + _label = label, + }, Owned) +end + +function Owned:is_owned() + return self._owned == true +end + +function Owned:value() + return self._value +end + +function Owned:terminate(reason) + return terminate_owned_value(self, reason) +end + +function Owned:terminate_checked(reason, label) + local ok, err = self:terminate(reason) + if ok ~= true then + error((label or self._label or 'resource termination failed') .. ': ' .. tostring(err), 2) + end + return true +end + +function Owned:detach() + if not self._owned then + return nil, 'resource is not owned' + end + + local value = self._value + self._owned = false + self._value = nil + return value, nil +end + +--- Transfer ownership to another owner. receiver_install(value) is called before +--- this lease releases cleanup ownership. If receiver_install fails or raises, +--- this lease remains responsible for termination. +function Owned:handoff(receiver_install) + if not self._owned then + return nil, 'resource is not owned' + end + + local value = self._value + + if receiver_install ~= nil then + if type(receiver_install) ~= 'function' then + return nil, 'handoff receiver must be a function' + end + + local ok, a, b = pcall(receiver_install, value) + if not ok then + return nil, stringify_error(a, 'handoff receiver raised') + end + if a == false or a == nil then + return nil, stringify_error(b, 'handoff receiver rejected resource') + end + end + + self._owned = false + self._value = nil + return value, nil +end + +M.Owned = Owned + +return M diff --git a/src/devicecode/support/retained_domain.lua b/src/devicecode/support/retained_domain.lua new file mode 100644 index 00000000..6f0bfb8b --- /dev/null +++ b/src/devicecode/support/retained_domain.lua @@ -0,0 +1,87 @@ +-- devicecode/support/retained_domain.lua +-- +-- Tiny retained domain-state owner. This is intentionally not a model +-- framework: it is a non-yielding publication helper for older services while +-- they move towards the service-architecture.md coordinator/model style. +-- +-- Doctrine: +-- * retained state//... means this domain fact is currently asserted; +-- * unretain removes facts that are no longer asserted; +-- * clear() is suitable for scope finalisers and must never wait. + +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local retained_publish = require 'devicecode.support.retained_publish' +local tablex = require 'shared.table' + +local M = {} +local Domain = {} +Domain.__index = Domain + +local function copy(v) return tablex.deep_copy(v) end + +local function topic_key(topic) + local out = {} + for i = 1, #(topic or {}) do out[i] = tostring(topic[i]) end + return table.concat(out, '/') +end + +local function join_topic(prefix, rel) + local out = {} + for i = 1, #(prefix or {}) do out[#out + 1] = prefix[i] end + if type(rel) == 'table' then + for i = 1, #rel do out[#out + 1] = rel[i] end + elseif rel ~= nil then + out[#out + 1] = rel + end + return out +end + +function M.new(conn, opts) + opts = opts or {} + local prefix = copy(opts.prefix or {}) + return setmetatable({ + conn = conn, + prefix = prefix, + cache = {}, + owned = {}, + opts = opts.opts, + }, Domain) +end + +function Domain:topic(rel) + return join_topic(self.prefix, rel) +end + +function Domain:retain(rel, payload, opts) + local topic = self:topic(rel) + local key = topic_key(topic) + local ok, err, changed = retained_publish.retain_if_changed(self.conn, self.cache, key, topic, payload, opts or self.opts) + if ok == true then self.owned[key] = topic end + return ok, err, changed +end + +function Domain:unretain(rel, opts) + local topic = self:topic(rel) + local key = topic_key(topic) + local ok, err = bus_cleanup.unretain(self.conn, topic, opts or self.opts) + if ok ~= true then return nil, err, false end + self.owned[key] = nil + self.cache[key] = nil + return true, nil, true +end + +function Domain:clear(opts) + local keys = {} + for key in pairs(self.owned) do keys[#keys + 1] = key end + table.sort(keys) + for _, key in ipairs(keys) do + local topic = self.owned[key] + bus_cleanup.unretain(self.conn, topic, opts or self.opts) + self.owned[key] = nil + self.cache[key] = nil + end + return true, nil +end + +M.Domain = Domain +return M diff --git a/src/devicecode/support/retained_publish.lua b/src/devicecode/support/retained_publish.lua new file mode 100644 index 00000000..f5a72ba8 --- /dev/null +++ b/src/devicecode/support/retained_publish.lua @@ -0,0 +1,66 @@ +-- devicecode/support/retained_publish.lua +-- +-- Idempotent retained-state publication helpers. These helpers are for +-- retained state/meta/capability projections, not event streams: they suppress +-- unchanged retains so retained watchers are not woken for identical payloads. + +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local tablex = require 'shared.table' + +local M = {} + +local function key_of(key) + if key ~= nil then return key end + return '_' +end + +local function copy(v) return tablex.deep_copy(v) end + +function M.retain_if_changed(conn, cache, key, topic, payload, opts) + cache = cache or {} + local k = key_of(key) + if cache[k] ~= nil and tablex.deep_equal(cache[k], payload) then + return true, nil, false + end + local ok, err = bus_cleanup.retain(conn, topic, payload, opts) + if ok ~= true then return nil, err, false end + cache[k] = copy(payload) + return true, nil, true +end + +function M.unretain_if_present(conn, cache, key, topic, opts) + cache = cache or {} + local k = key_of(key) + if cache[k] == nil then return true, nil, false end + local ok, err = bus_cleanup.unretain(conn, topic, opts) + if ok ~= true then return nil, err, false end + cache[k] = nil + return true, nil, true +end + +function M.publish_map_changed(conn, cache, next_map, topic_fn, payload_fn, opts) + cache = cache or {} + local seen = {} + local changed = 0 + for id, rec in pairs(next_map or {}) do + seen[id] = true + local payload = payload_fn(rec, id) + local ok, err, did = M.retain_if_changed(conn, cache, id, topic_fn(id), payload, opts) + if ok ~= true then return nil, err, changed end + if did then changed = changed + 1 end + end + local removed = {} + for id in pairs(cache) do + if not seen[id] then removed[#removed + 1] = id end + end + for i = 1, #removed do + local id = removed[i] + local ok, err, did = M.unretain_if_present(conn, cache, id, topic_fn(id), opts) + if ok ~= true then return nil, err, changed end + if did then changed = changed + 1 end + end + return true, nil, changed +end + +M.copy = copy +return M diff --git a/src/devicecode/support/scoped_work.lua b/src/devicecode/support/scoped_work.lua new file mode 100644 index 00000000..e350d452 --- /dev/null +++ b/src/devicecode/support/scoped_work.lua @@ -0,0 +1,442 @@ +-- devicecode/support/scoped_work.lua +-- +-- Shared infrastructure for scoped Fabric/service work. +-- +-- Responsibilities: +-- * create a child scope under the lifetime owner +-- * run the worker inside that child scope +-- * signal body-ended from wrapper code, not user code +-- * ordinary early reaping waits for body-ended before join_op() +-- * store completion exactly once +-- * let reporters observe stored completion without joining +-- * require explicit delegation for non-lifetime reaping +-- +-- This module is infrastructure. It may use op.perform_raw for authorised +-- reaping so that outcome storage does not depend on the current scope staying +-- healthy. + +local safe = require 'coxpcall' + +local fibers = require 'fibers' +local op = require 'fibers.op' +local cond = require 'fibers.cond' +local tablex = require 'shared.table' + +local M = {} + +local copy_table = tablex.shallow_copy + +local function copy_value(v) + if type(v) == 'table' then + return copy_table(v) + end + return v +end + +local function copy_completion(ev) + if ev == nil then + return nil + end + + local out = copy_table(ev) + out.result = copy_value(ev.result) + out.report = copy_value(ev.report) + out.primary = copy_value(ev.primary) + return out +end + +local function scope_not_ready(scope, name) + if scope == nil then + return name .. ' is required' + end + + if type(scope.status) == 'function' then + local st, reason = scope:status() + if st ~= 'running' then + return ('%s is not running: %s%s'):format( + name, + tostring(st), + reason ~= nil and (': ' .. tostring(reason)) or '' + ) + end + end + + if type(scope.admission) == 'function' then + local admission, reason = scope:admission() + if admission ~= 'open' then + return ('%s admission is not open: %s%s'):format( + name, + tostring(admission), + reason ~= nil and (': ' .. tostring(reason)) or '' + ) + end + end + + return nil +end + +local function make_completion(identity, status, report, result, primary) + local ev = copy_table(identity) + + ev.status = status + ev.report = copy_value(report) + + if status == 'ok' then + ev.result = copy_value(result) + else + ev.primary = primary + end + + return ev +end + +local function validate_result(result) + if type(result) ~= 'table' then + error('scoped_work: worker must return one result table', 0) + end + return result +end + +--- Start scoped work. +--- +--- Required: +--- lifetime_scope : parent scope that owns the child lifetime +--- identity : table copied into the eventual completion event +--- identity.kind : string +--- run(scope) : worker body; must return one result table on success +--- +--- Optional: +--- reaper_scope : where the authorised reaper fibre lives +--- report_scope : where the reporter fibre lives +--- reaper_delegation : required when reaper_scope ~= lifetime_scope +--- setup(scope) : non-yielding setup hook before worker admission; +--- returns setup table or nil,err +--- setup table may include cancel_owned_now(reason), +--- an immediate, non-yielding cancellation hook for +--- setup-owned resources such as request owners +--- report(ev) : immediate reporter callback; should not yield +--- copy_result(result) : optional body-end snapshot hook for successful result +--- cancel_op : optional Op, or function(child, setup_result) -> Op; +--- if it wins before body-ended and returns neither nil +--- nor false, child scope is cancelled with that reason +--- +---@param spec table +---@return table|nil handle +---@return string|nil err +---@return table|nil setup_result +local function start_impl(spec, opts) + opts = opts or {} + local cleanup_on_start_failure = not not opts.cleanup_on_start_failure + if type(spec) ~= 'table' then + return nil, 'scoped_work.start: spec must be a table' + end + + local lifetime_scope = spec.lifetime_scope + local reaper_scope = spec.reaper_scope or lifetime_scope + local report_scope = spec.report_scope or reaper_scope + + local err + + err = scope_not_ready(lifetime_scope, 'lifetime_scope') + if err then return nil, err end + + err = scope_not_ready(reaper_scope, 'reaper_scope') + if err then return nil, err end + + if spec.report ~= nil then + if type(spec.report) ~= 'function' then + return nil, 'report must be a function when provided' + end + + err = scope_not_ready(report_scope, 'report_scope') + if err then return nil, err end + end + + if reaper_scope ~= lifetime_scope and spec.reaper_delegation == nil then + return nil, 'non-lifetime reaping requires explicit reaper_delegation' + end + + if type(spec.identity) ~= 'table' then + return nil, 'identity table required' + end + + if type(spec.identity.kind) ~= 'string' then + return nil, 'identity.kind string required' + end + + if type(spec.run) ~= 'function' then + return nil, 'run function required' + end + + if spec.setup ~= nil and type(spec.setup) ~= 'function' then + return nil, 'setup must be a function when provided' + end + + if spec.copy_result ~= nil and type(spec.copy_result) ~= 'function' then + return nil, 'copy_result must be a function when provided' + end + + if spec.cancel_op ~= nil and type(spec.cancel_op) ~= 'table' and type(spec.cancel_op) ~= 'function' then + return nil, 'cancel_op must be an Op or function when provided' + end + + local identity = copy_table(spec.identity) + local copy_result = spec.copy_result or copy_value + + local child, child_err = lifetime_scope:child() + if not child then + return nil, child_err or 'failed to create child scope' + end + + local setup_result = nil + + local body_done = cond.new() + local outcome_done = cond.new() + + local result + local failure_primary + local outcome + local reaped = false + local reported = false + + local function store_once(status, report, primary) + if reaped then + return false + end + + reaped = true + outcome = make_completion(identity, status, report, result, primary) + outcome_done:signal() + return true + end + + local function outcome_snapshot() + return copy_completion(outcome) + end + + local function cancel_child_now(reason) + reason = reason or 'cancelled' + + if setup_result and type(setup_result.cancel_owned_now) == 'function' then + local ok, err = setup_result.cancel_owned_now(reason) + if ok == false or ok == nil then + return nil, err or 'scoped_work_cancel_owned_failed' + end + end + + child:cancel(reason) + return true, nil + end + + local function outcome_op() + return op.guard(function () + if outcome ~= nil then + return op.always(copy_completion(outcome)) + end + + return outcome_done:wait_op():wrap(function () + return copy_completion(outcome) + end) + end) + end + + local function cancel_start_failure(reason) + -- No worker body has been admitted. Suppress reporter output for this + -- failed start, signal the body-ended barrier so an already-spawned + -- reaper can make progress, and cancel the empty child. + -- + -- If setup already transferred caller-visible resources into this helper, + -- give them their immediate cancellation path now. Parent join/finalisers + -- remain the structural cleanup fallback and must be idempotent. + -- + -- The normal public start() path deliberately does not join here: it is + -- coordinator-safe and must not hide synchronous cleanup waits. Parent + -- join remains the structural reaper of last resort. Setup-only callers + -- may request bounded eager cleanup via start_setup_checked(). + reported = true + if setup_result and type(setup_result.cancel_owned_now) == 'function' then + setup_result.cancel_owned_now(reason or 'scoped_work_start_failed') + end + body_done:signal() + child:cancel(reason or 'scoped_work_start_failed') + + if cleanup_on_start_failure then + op.perform_raw(child:join_op()) + end + end + + if spec.setup then + local ok_setup, a, b = safe.pcall(function () + return spec.setup(child) + end) + + if not ok_setup then + cancel_start_failure(a or 'setup_failed') + return nil, tostring(a or 'setup_failed') + end + + if a == nil then + local setup_err = b or 'setup_failed' + cancel_start_failure(setup_err) + return nil, tostring(setup_err) + end + + if type(a) ~= 'table' then + local setup_err = 'setup must return a table' + cancel_start_failure(setup_err) + return nil, setup_err + end + + setup_result = a + end + + local ok_reaper, reaper_spawn_err = reaper_scope:spawn(function () + -- Ordinary observation reaping must not close admission while the + -- worker body may still legitimately spawn child work. The helper, not + -- user code, signals this barrier. Start-failure paths signal it too so + -- the reaper cannot be left parked on an empty child scope. + op.perform_raw(body_done:wait_op()) + + local status, report, primary = op.perform_raw(child:join_op()) + if status == 'failed' and failure_primary ~= nil then + primary = copy_value(failure_primary) + end + store_once(status, report, primary) + end) + + if ok_reaper ~= true then + cancel_start_failure(reaper_spawn_err or 'reaper_spawn_failed') + return nil, reaper_spawn_err or 'reaper_spawn_failed' + end + + if spec.cancel_op ~= nil then + local cancel_op = spec.cancel_op + if type(cancel_op) == 'function' then + local ok_cancel_op, cop_or_err = safe.pcall(function () + return cancel_op(child, setup_result) + end) + if not ok_cancel_op then + cancel_start_failure(cop_or_err or 'cancel_op_setup_failed') + return nil, tostring(cop_or_err or 'cancel_op_setup_failed') + end + cancel_op = cop_or_err + end + + if cancel_op ~= nil then + local ok_cancel, cancel_spawn_err = reaper_scope:spawn(function () + local which, reason = fibers.perform(op.named_choice({ + cancel = cancel_op, + body_done = body_done:wait_op(), + })) + + if which == 'cancel' and reason ~= nil and reason ~= false then + local ok, cerr = cancel_child_now(reason) + if ok ~= true then error(cerr or 'scoped_work_cancel_failed', 0) end + end + end) + + if ok_cancel ~= true then + cancel_start_failure(cancel_spawn_err or 'cancel_watcher_spawn_failed') + return nil, cancel_spawn_err or 'cancel_watcher_spawn_failed' + end + end + end + + if spec.report then + local ok_reporter, reporter_spawn_err = report_scope:spawn(function () + -- Reporter is ordinary observing-scope code. If its scope is + -- cancelled before the outcome is available, it does not report. + local ev = fibers.perform(outcome_op()) + + if reported then + return + end + reported = true + + local ok, report_err = spec.report(copy_completion(ev)) + if ok ~= true then + error(report_err or 'scoped_work_report_failed', 0) + end + end) + + if ok_reporter ~= true then + cancel_start_failure(reporter_spawn_err or 'reporter_spawn_failed') + return nil, reporter_spawn_err or 'reporter_spawn_failed' + end + end + + local ok_worker, worker_spawn_err = child:spawn(function () + local ok, ret = safe.pcall(function () + -- Cancellation may have happened after admission but before this worker + -- fibre first ran. Observe the child scope before entering user code so + -- pre-body cancellation cannot leak into HAL/Fabric work. + child:perform(op.always(true)) + + local raw = validate_result(spec.run(child, setup_result)) + return validate_result(copy_result(raw)) + end) + + if ok then + -- Snapshot the worker result before join/finalisers run. Finalisers + -- must not be able to mutate the eventual successful completion. + result = ret + elseif spec.preserve_error_primary == true then + failure_primary = copy_value(ret) + end + + -- This is wrapper-owned, not user-owned. + body_done:signal() + + if not ok then + error(ret, 0) + end + end) + + if ok_worker ~= true then + cancel_start_failure(worker_spawn_err or 'worker_spawn_failed') + return nil, worker_spawn_err or 'worker_spawn_failed' + end + + local handle = { + _identity = identity, + } + + function handle:cancel(reason) + return cancel_child_now(reason or 'cancelled') + end + + function handle:outcome_op() + return outcome_op() + end + + function handle:outcome() + return outcome_snapshot() + end + + function handle:identity() + return copy_table(identity) + end + + return handle, nil, setup_result +end + +--- Start scoped work in coordinator-safe mode. This function never calls +--- join_op() or perform_raw() synchronously on start-failure paths. +--- Failed starts may leave a cancelled attached child for the parent/reaper to +--- account for later. +function M.start(spec) + return start_impl(spec, { cleanup_on_start_failure = false }) +end + +--- Start scoped work in setup-only mode. This preserves the stronger +--- no-attached-child-on-start-failure property by joining the just-created child +--- non-interruptibly if infrastructure admission fails. Do not call this from +--- strict coordinator branches. +function M.start_setup_checked(spec) + return start_impl(spec, { cleanup_on_start_failure = true }) +end + +M.make_completion = make_completion +M.copy_completion = copy_completion + +return M diff --git a/src/devicecode/support/service_events.lua b/src/devicecode/support/service_events.lua new file mode 100644 index 00000000..78024f7b --- /dev/null +++ b/src/devicecode/support/service_events.lua @@ -0,0 +1,178 @@ +-- devicecode/support/service_events.lua +-- +-- Small event-port helper for fibres services. +-- +-- Vocabulary: +-- source = the component/adapter that produced the event +-- source_id = the identity of that source instance +-- generation = generation/version boundary when applicable +-- event = immutable coordinator input +-- port = immediate, non-blocking event admission surface +-- +-- A port is deliberately not a callback into a parent. It only constructs and +-- admits events to a queue using the public queue helpers. Coordinators remain +-- the sole owners of their state. + +local queue = require 'devicecode.support.queue' +local tablex = require 'shared.table' + +local M = {} + +local copy_table = tablex.shallow_copy + +local normalise_event + +local Port = {} +Port.__index = Port + +local TargetPort = {} +TargetPort.__index = TargetPort + +function TargetPort:event(ev, attrs) + local out = normalise_event(ev, attrs) + for k, v in pairs(self._identity or {}) do + if out[k] == nil then out[k] = v end + end + if self._mark_route_events then + out._service_event = true + end + return out +end + +function TargetPort:emit_required(ev, attrs, label) + if type(attrs) == 'string' and label == nil then + label = attrs + attrs = nil + end + return self._target:emit_required( + self:event(ev, attrs), + label or self._label or 'service_event_admission_failed' + ) +end + +function TargetPort:emit_now(ev, attrs) + if type(self._target.emit_now) == 'function' then + return self._target:emit_now(self:event(ev, attrs)) + end + return self:emit_required(ev, attrs, self._label or 'service_event_admission_failed') +end + +function TargetPort:assert_emit_required(ev, attrs, label) + local ok, err = self:emit_required(ev, attrs, label) + if ok ~= true then error(err or 'service_event_admission_failed', 2) end + return true +end + +function TargetPort:identity() + return copy_table(self._identity) +end + + +function normalise_event(ev, attrs) + if type(ev) == 'string' then + local out = copy_table(attrs) + out.kind = ev + return out + end + if type(ev) ~= 'table' then + error('service_events: event table or kind string required', 3) + end + local out = copy_table(ev) + for k, v in pairs(attrs or {}) do + if out[k] == nil then out[k] = v end + end + return out +end + +function Port:event(ev, attrs) + local out = normalise_event(ev, attrs) + for k, v in pairs(self._identity or {}) do + if out[k] == nil then out[k] = v end + end + if self._mark_route_events then + out._service_event = true + end + return out +end + +function Port:emit_now(ev, attrs) + return queue.try_admit_now(self._tx, self:event(ev, attrs)) +end + +function Port:emit_required(ev, attrs, label) + if type(attrs) == 'string' and label == nil then + label = attrs + attrs = nil + end + return queue.try_admit_required( + self._tx, + self:event(ev, attrs), + label or self._label or 'service_event_admission_failed' + ) +end + +function Port:assert_emit_required(ev, attrs, label) + local ok, err = self:emit_required(ev, attrs, label) + if ok ~= true then error(err or 'service_event_admission_failed', 2) end + return true +end + +function Port:tx() + return self._tx +end + +function Port:identity() + return copy_table(self._identity) +end + +function M.port(tx, identity, opts) + if tx == nil or type(tx.send_op) ~= 'function' then + error('service_events.port: mailbox tx required', 2) + end + opts = opts or {} + return setmetatable({ + _tx = tx, + _identity = copy_table(identity), + _label = opts.label, + _mark_route_events = not not opts.mark_route_events, + }, Port) +end + + +function M.wrap(target, identity, opts) + if target == nil then + error('service_events.wrap: target required', 2) + end + if type(target.emit_required) == 'function' then + opts = opts or {} + return setmetatable({ + _target = target, + _identity = copy_table(identity), + _label = opts.label, + _mark_route_events = not not opts.mark_route_events, + }, TargetPort) + end + return M.port(target, identity, opts) +end + +function M.reporter_for(target, identity, opts) + return M.reporter(M.wrap(target, identity, opts), opts and opts.label) +end + +function M.reporter(port, label) + if type(port) ~= 'table' or type(port.emit_required) ~= 'function' then + error('service_events.reporter: port required', 2) + end + return function (ev) + return port:emit_required(ev, label or 'service_event_report_failed') + end +end + +function M.is_route_event(ev) + return type(ev) == 'table' and ev._service_event == true +end + +M.Port = Port +M.TargetPort = TargetPort + +return M diff --git a/src/devicecode/support/wake_probe.lua b/src/devicecode/support/wake_probe.lua new file mode 100644 index 00000000..97cba8e3 --- /dev/null +++ b/src/devicecode/support/wake_probe.lua @@ -0,0 +1,161 @@ +-- devicecode/support/wake_probe.lua +-- +-- Lightweight wake-rate instrumentation for cooperative services. It is disabled +-- unless explicitly enabled via opts.wake_probe or DEVICECODE_WAKE_PROBE=1. +-- Probes aggregate in memory for a short interval, then publish one retained +-- obs/v1//metric/wake_probe payload. They do not retain event history. + +local runtime = require 'fibers.runtime' + +local M = {} +local Probe = {} +Probe.__index = Probe + +local function env_truthy(name) + local v = os.getenv(name) + if v == nil or v == '' then return false end + v = tostring(v):lower() + return not (v == '0' or v == 'false' or v == 'no' or v == 'off') +end + +local function number_opt(v, default) + local n = tonumber(v) + if n == nil then return default end + return n +end + +local function bool_opt(v, default) + if v == nil then return default end + if type(v) == 'boolean' then return v end + local s = tostring(v):lower() + if s == '1' or s == 'true' or s == 'yes' or s == 'on' then return true end + if s == '0' or s == 'false' or s == 'no' or s == 'off' then return false end + return default +end + +function M.enabled(opts) + opts = opts or {} + if opts.wake_probe ~= nil then return bool_opt(opts.wake_probe, false) end + if opts.wake_probe_enabled ~= nil then return bool_opt(opts.wake_probe_enabled, false) end + return env_truthy('DEVICECODE_WAKE_PROBE') +end + +local function event_detail(ev) + if type(ev) ~= 'table' then return tostring(ev) end + return tostring(ev.kind or ev.type or ev.what or ev.source or ev.event or 'table') +end + +local function add_top(out, key, count, elapsed) + out[#out + 1] = { + key = key, + count = count, + rate_per_s = elapsed > 0 and (count / elapsed) or count, + } +end + +local function sort_top(top) + table.sort(top, function (a, b) + if a.count ~= b.count then return a.count > b.count end + return tostring(a.key) < tostring(b.key) + end) +end + +function M.new(svc, opts) + opts = opts or {} + local enabled = M.enabled(opts) + local now = runtime.now() + return setmetatable({ + enabled = enabled, + svc = svc, + conn = opts.conn or (svc and svc.conn), + service = opts.service or (svc and svc.name) or opts.name or 'service', + name = opts.metric_name or opts.wake_probe_metric or 'wake_probe', + report_interval_s = number_opt(opts.report_interval_s or os.getenv('DEVICECODE_WAKE_PROBE_INTERVAL_S'), 10.0), + warn_rate_per_s = number_opt(opts.warn_rate_per_s or os.getenv('DEVICECODE_WAKE_PROBE_WARN_RATE'), 0), + max_top = math.max(1, math.floor(number_opt(opts.max_top or os.getenv('DEVICECODE_WAKE_PROBE_TOP'), 16))), + counts = {}, + total = 0, + since = now, + next_report = now + math.max(1.0, number_opt(opts.report_interval_s or os.getenv('DEVICECODE_WAKE_PROBE_INTERVAL_S'), 10.0)), + }, Probe) +end + +function Probe:active() + return self.enabled == true +end + +function Probe:record(source, detail, n) + if self.enabled ~= true then return false end + source = tostring(source or 'wake') + local key = detail ~= nil and (source .. ':' .. tostring(detail)) or source + n = tonumber(n) or 1 + if n <= 0 then return false end + self.counts[key] = (self.counts[key] or 0) + n + self.total = self.total + n + return true +end + +function Probe:record_event(source, ev) + if self.enabled ~= true then return false end + return self:record(source, event_detail(ev), 1) +end + +function Probe:record_choice(which, ev) + if self.enabled ~= true then return false end + return self:record(tostring(which or 'choice'), event_detail(ev), 1) +end + +function Probe:report_if_due(extra) + if self.enabled ~= true then return false end + local now = runtime.now() + if now < self.next_report then return false end + + local elapsed = now - self.since + if elapsed <= 0 then elapsed = 0.000001 end + + local top = {} + for key, count in pairs(self.counts) do add_top(top, key, count, elapsed) end + sort_top(top) + while #top > self.max_top do top[#top] = nil end + + local payload = { + service = self.service, + interval_s = elapsed, + total_wakes = self.total, + rate_per_s = self.total / elapsed, + top = top, + } + if type(extra) == 'table' then + for k, v in pairs(extra) do payload[k] = v end + end + + if self.svc and type(self.svc.obs_metric) == 'function' then + self.svc:obs_metric(self.name, payload) + elseif self.conn and type(self.conn.retain) == 'function' then + self.conn:retain({ 'obs', 'v1', self.service, 'metric', self.name }, payload) + elseif self.svc and type(self.svc.obs_log) == 'function' then + self.svc:obs_log('debug', payload) + end + + if self.warn_rate_per_s > 0 and payload.rate_per_s >= self.warn_rate_per_s + and self.svc and type(self.svc.obs_log) == 'function' then + local warn_payload = {} + for k, v in pairs(payload) do warn_payload[k] = v end + warn_payload.what = 'wake_probe_high_rate' + self.svc:obs_log('warn', warn_payload) + end + + self.counts = {} + self.total = 0 + self.since = now + self.next_report = now + math.max(1.0, self.report_interval_s) + return true +end + +function Probe:record_and_report(source, detail, extra) + self:record(source, detail, 1) + return self:report_if_due(extra) +end + +M.Probe = Probe +return M diff --git a/src/devices/bigbox-v1-cm.lua b/src/devices/bigbox-v1-cm.lua index 5e440a7c..0dd62664 100644 --- a/src/devices/bigbox-v1-cm.lua +++ b/src/devices/bigbox-v1-cm.lua @@ -13,6 +13,6 @@ return { "ui", "wifi", "mcu_bridge", - "switch" + "wired" } } diff --git a/src/devices/bigbox_v1-pr1.lua b/src/devices/bigbox_v1-pr1.lua index df2dac21..e68b5426 100644 --- a/src/devices/bigbox_v1-pr1.lua +++ b/src/devices/bigbox_v1-pr1.lua @@ -4,7 +4,7 @@ return { services = { "log", "config", - "switch", + "wired", -- ... other services specific to this device version } } diff --git a/src/gpio.lua b/src/gpio.lua deleted file mode 100644 index 19ce819e..00000000 --- a/src/gpio.lua +++ /dev/null @@ -1,162 +0,0 @@ -local file = require 'fibers.stream.file' -local sc = require 'fibers.utils.syscall' - -local pollio = require 'fibers.pollio' -pollio.install_poll_io_handler() - -local pin = {} -pin.__index = pin - -local gpio_base -local gpio_path = "/sys/class/gpio" - --- Set the GPIO base -local function set_gpio_base() - local f = io.popen("cat /sys/class/gpio/gpiochip*/base | head -n1", "r") - if not f then error("could not read GPIO base!") end - local base = f:read() - f:close() - base = tonumber(base) - if not base then return nil, 'could not determine gpio_base' end - gpio_base = base - return base, nil -end - --- Helper function to write to a GPIO file -local function write_gpio_file(path, value) - local f, err = file.open(path, "w") - if not f then return nil, err end - f:write(value) - f:close() - return true, nil -end - --- Create a new pin instance -local function new_pin(gpio_num) - local self = setmetatable({ - gpio_num = gpio_num, - gpio_path = gpio_path .. "/gpio" .. (gpio_base + gpio_num) - }, pin) - return self -end - --- Export a GPIO pin -function pin:export() - if sc.stat(self.gpio_path) then - return true, nil - end - return write_gpio_file(gpio_path .. "/export", gpio_base + self.gpio_num) -end - --- Unexport a GPIO pin -function pin:unexport() - return write_gpio_file(gpio_path .. "/unexport", gpio_base + self.gpio_num) -end - --- Set the direction of the GPIO pin to "in" -function pin:set_in() - return write_gpio_file(self.gpio_path .. "/direction", "in") -end - --- Set the direction of the GPIO pin to "out" -function pin:set_out() - return write_gpio_file(self.gpio_path .. "/direction", "out") -end - --- Write a high value to the GPIO pin -function pin:pull_up() - assert(os.execute("pinctrl set "..self.gpio_num.." pu")) - return true -end - --- Write a low value to the GPIO pin -function pin:pull_down() - assert(os.execute("pinctrl set "..self.gpio_num.." pd")) - return true -end - --- Set edge detection to none -function pin:pull_none() - assert(os.execute("pinctrl set "..self.gpio_num.." pn")) - return true -end - --- Write a high value to the GPIO pin -function pin:write_high() - return write_gpio_file(self.gpio_path .. "/value", "1") -end - --- Write a low value to the GPIO pin -function pin:write_low() - return write_gpio_file(self.gpio_path .. "/value", "0") -end - --- Set edge detection to none -function pin:edge_none() - return write_gpio_file(self.gpio_path .. "/edge", "none") -end - --- Set edge detection to rising edge -function pin:edge_rising() - return write_gpio_file(self.gpio_path .. "/edge", "rising") -end - --- Set edge detection to falling edge -function pin:edge_falling() - return write_gpio_file(self.gpio_path .. "/edge", "falling") -end - --- Set edge detection to both rising and falling edges -function pin:edge_both() - return write_gpio_file(self.gpio_path .. "/edge", "both") -end - --- Read the value from the GPIO pin -function pin:read() - local f, err = file.open(self.gpio_path .. "/value", "r") - if not f then return nil, err end - local value = f:read() - f:close() - return value, nil -end - --- Operation to watch for a change in the value of a GPIO pin. can be used in a non-deterministic op.choice() selector -function pin:watch_op() - local f = self.watch_file - if not f then - local err - f, err = file.open(self.gpio_path .. "/value", "r") - if not f then return nil, err end - self.watch_file = f - end - f:seek(sc.SEEK_SET, 0) - f:read() -- consume existing `pri` event, if any - local retval = pollio.fd_priority_op(f.io.fd):wrap(function () - f:seek(sc.SEEK_SET, 0) - local state, err = f:read() - f:close() - self.watch_file = nil - if err then return nil, err end - return state, nil - end) - return retval -end - --- Operation to watch for a change in the value of a GPIO pin -function pin:watch() - return self:watch_op():perform() -end - --- Initialize the GPIO base -local function initialize_gpio() - local base, err = set_gpio_base() - if not base then - return nil, err - end - return base, nil -end - -return { - initialize_gpio = initialize_gpio, - new_pin = new_pin, -} \ No newline at end of file diff --git a/src/lua-bus b/src/lua-bus deleted file mode 160000 index 89af71a1..00000000 --- a/src/lua-bus +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 89af71a1ab469a70c19c6df2cc65619dba6a6e24 diff --git a/src/lua-fibers b/src/lua-fibers deleted file mode 160000 index 5c297941..00000000 --- a/src/lua-fibers +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5c297941873fa29725c6b1d56ab5f36388c1849d diff --git a/src/lua-trie b/src/lua-trie deleted file mode 160000 index 28b3572b..00000000 --- a/src/lua-trie +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 28b3572b597bd24d0b71f0cfcd709f15bc4384d4 diff --git a/src/main.lua b/src/main.lua index a8f3d377..d5f0ee45 100644 --- a/src/main.lua +++ b/src/main.lua @@ -1,124 +1,52 @@ -package.path = "../?.lua;" .. package.path .. ";/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua" - -local fiber = require 'fibers.fiber' -local sleep = require 'fibers.sleep' -local context = require 'fibers.context' -local sc = require 'fibers.utils.syscall' -local bus = require 'bus' -local service = require 'service' -local log = require 'services.log' -log.outfile = '/tmp/logs.log' -require 'fibers.pollio'.install_poll_io_handler() -require 'fibers.alarm'.install_alarm_handler() - -local file_chunk = 0 -local lines = 0 - --- Copy the ubus_scripts -local status = os.execute("cp -r ./ubus_scripts/* /") - -if status ~= 0 then - log.error("Failed to copy ubus_scripts") -end - --- local hook_file = io.open("/tmp/hook_logs_" .. file_chunk .. ".log", "w") -local function hook(event, line) - local info = debug.getinfo(2) - local msg = string.format("%s:%d:%s\n", info.short_src, line, info.name or "") - if hook_file then - hook_file:write(msg) - hook_file:flush() - end - - lines = lines + 1 - if lines > 2000 then - if file_chunk - 1 >= 0 then - os.remove("/tmp/hook_logs_" .. (file_chunk - 1) .. ".log") - end - file_chunk = file_chunk + 1 - hook_file:close() - hook_file = io.open("/tmp/hook_logs_" .. file_chunk .. ".log", "w") - lines = 0 - end +-- main.lua +-- +-- Bootstrap entrypoint. +-- +-- Required env: +-- DEVICECODE_SERVICES comma-separated service module names (e.g. "hal,config,monitor") +-- DEVICECODE_STATE_DIR persisted state root directory +-- +-- Optional env: +-- DEVICECODE_ENV "dev" | "prod" (default: dev) + +local function add_path(prefix) + package.path = prefix .. '?.lua;' .. prefix .. '?/init.lua;' .. package.path end --- debug.sethook(hook, "l") -local function count_dir_items(path) - local count - local p = io.popen('ls -1 "' .. path .. '" | wc -l') - if p then - local output = p:read("*all") - count = tonumber(output) - p:close() - end - return count +local env = os.getenv('DEVICECODE_ENV') or 'dev' +add_path('/usr/lib/lua/') +add_path('./') +if env == 'prod' then + add_path('./lib/') +else + add_path('../vendor/lua-fibers/src/') + add_path('../vendor/lua-bus/src/') + add_path('../vendor/lua-trie/src/') end -local function count_zombies() - local count - local p = io.popen('ps | grep -c " Z "') - if p then - local output = p:read("*all") - count = tonumber(output) - p:close() - end - return count +local fibers = require 'fibers' +local mainmod = require 'devicecode.main' +local signal_bridge = require 'devicecode.signal_bridge' + +local ok, err = xpcall(function() + return fibers.run(function(scope) + local sig_ok, sig_err = signal_bridge.install(scope, { + TERM = true, + INT = true, + }) + if not sig_ok then + io.stderr:write('devicecode: signal bridge disabled: ' .. tostring(sig_err) .. '\n') + end + + return mainmod.run(scope, { + env = env, + }) + end) +end, tostring) + +if not ok then + if tostring(err):match('signal:TERM') or tostring(err):match('signal:INT') then + os.exit(0) + end + error(err, 0) end - -local function count_processes() - local p = io.popen('ps | wc -l') - if p then - local output = p:read("*all") - local count = tonumber(output) - p:close() - return count - 1 -- Subtract 1 for the header line - end -end - --- Get the device type/version from command line arguments or environment -local device_version = arg[1] or os.getenv("DEVICE") - -if not device_version then error("device version must be specified on command line or env variable") end - --- create the root context for the whole application -local bg_ctx = context.background() -local rootctx = context.with_value(bg_ctx, "device", device_version) - --- Load the device configuration -local device_config = require("devices/" .. rootctx:value("device")) - --- Initialise bus (current bus implementation doesn't take a context) -local bus = bus.new({ q_length = 100, m_wild = '#', s_wild = '+' }) - -local services = {} - -local function launch_services() - for _, service_name in ipairs(device_config.services) do - local svce = require("services/" .. service_name) - service.spawn(svce, bus, rootctx) - end -end - --- The main control fiber -fiber.spawn(function() - local pid = sc.getpid() - -- Launch all the services for the specific device - launch_services() - - -- Here we can add more code for the CLI or other controls - while true do - local base_open_fds = count_dir_items("/proc/" .. pid .. "/fd") - local base_zombies = count_zombies() - local processes = count_processes() - print("main fiber sleeping, zombies:", base_zombies, "open fds:", base_open_fds, "processes:", processes) - sleep.sleep(5) - -- CLI or other control logic goes here - end -end) - - -fiber.main() - - - - diff --git a/src/service.lua b/src/service.lua deleted file mode 100644 index 55b6e333..00000000 --- a/src/service.lua +++ /dev/null @@ -1,130 +0,0 @@ -local context = require "fibers.context" -local fiber = require 'fibers.fiber' -local bus = require "bus" -local new_msg = bus.new_msg -local log = require 'services.log' - -local STATE = { - INITIALISING = 'initialising', - ACTIVE = 'active', - DISABLED = 'disabled' -} -local FiberRegister = {} -FiberRegister.__index = FiberRegister - -function FiberRegister.new() - return setmetatable({size = 0, fibers = {}}, FiberRegister) -end - -function FiberRegister:add(name, status) - if self.fibers[name] == nil then - self.size = self.size + 1 - end - self.fibers[name] = status -end - -function FiberRegister:remove(name) - if self.fibers[name] ~= nil then - self.size = self.size - 1 - self.fibers[name] = nil - end -end - -function FiberRegister:is_empty() - return self.size == 0 -end - --- spawns a service, which involves creation of a child context, --- bus connection and a shutdown fiber --- (which should be adapted for update, reboot etc when system service is more built out) -local function spawn(service, bus, ctx) - local bus_connection = bus:connect() - if service.name == nil then - log.error("Service name is nil") - return - end - local val_ctx = context.with_value(ctx, "service_name", service.name) - local child_ctx, cancel_fn = context.with_cancel(val_ctx) - - local health_topic = { child_ctx:value("service_name"), 'health' } - - -- Non-blocking start function - if service.start == nil then - log.error("Service start function is nil") - return - end - service:start(child_ctx, bus_connection) - bus_connection:publish(new_msg( - health_topic, - { name = child_ctx:value("service_name"), state = STATE.ACTIVE }, - { retained = true } - )) - - -- Creates a shutdown fiber to handle any shutdown messages from the system service - -- Tracks all long running fibers under the current service before reporting an end to the service - fiber.spawn(function () - local system_events_sub = bus_connection:subscribe({ child_ctx:value("service_name"), 'control', 'shutdown' }) - local shutdown_event = system_events_sub:next_msg() - system_events_sub:unsubscribe() - - -- let every fiber know to end - cancel_fn(shutdown_event.payload.cause) - local service_fibers_status_sub = bus_connection:subscribe( - { child_ctx:value("service_name"), 'health', 'fibers', '+' } - ) - local active_fibers = FiberRegister.new() - - -- collect what fibers are active or initialising - local fiber_check_ctx = context.with_deadline(ctx, shutdown_event.payload.deadline) - while true do - local message, ctx_err = service_fibers_status_sub:next_msg_with_context_op(fiber_check_ctx):perform() - if ctx_err then break end - if message.payload.state == STATE.DISABLED then - active_fibers:remove(message.payload.name) - else - active_fibers:add(message.payload.name, message.payload.state) - end - end - - -- if no active or initialising fibers, we can safely end the service - if active_fibers:is_empty() then - bus_connection:publish(new_msg( - health_topic, - { name = child_ctx:value("service_name"), state = STATE.DISABLED }, - { retained = true } - )) - end - end) -end - -local function spawn_fiber(name, bus_connection, ctx, fn) - local child_ctx = context.with_value(ctx, "fiber_name", name) - child_ctx = context.with_cancel(child_ctx) - - local fiber_topic = { child_ctx:value("service_name"), 'health', 'fibers', child_ctx:value("fiber_name") } - - bus_connection:publish(new_msg( - fiber_topic, - { name = child_ctx:value("fiber_name"), state = STATE.INITIALISING }, - { retained = true } - )) - - fiber.spawn(function () - bus_connection:publish(new_msg( - fiber_topic, - { name = child_ctx:value("fiber_name"), state = STATE.ACTIVE }, - { retained = true } - )) - fn(child_ctx) - bus_connection:publish(new_msg( - fiber_topic, - { name = child_ctx:value("fiber_name"), state = STATE.DISABLED }, - { retained = true } - )) - end) -end - -return { - spawn = spawn, - spawn_fiber = spawn_fiber -} diff --git a/src/services/config.lua b/src/services/config.lua index b10d7e27..85ad62f6 100644 --- a/src/services/config.lua +++ b/src/services/config.lua @@ -1,42 +1,291 @@ -local file = require 'fibers.stream.file' -local fiber = require 'fibers.fiber' -local log = require 'services.log' -local json = require 'cjson.safe' -local new_msg = require('bus').new_msg - -local config_service = { - name = 'config' -} -config_service.__index = config_service - -local function publish_config(ctx, conn) - -- publish service configs - local config_file = assert(file.open("configs/" .. ctx:value("device") .. ".json")) - local raw_config = config_file:read_all_chars() - local config = json.decode(raw_config) - for k, v in pairs(config) do - log.trace("Config: publishing config for: ", k) - conn:publish(new_msg({ "config", k }, v, { retained = true })) - end - - -- publish mainflux configs - local mainflux_config_file = file.open("/data/configs/mainflux.cfg") - if not mainflux_config_file then - log.warn("Failed to open mainflux config file, cloud telemetry will not work") - return - end - local raw_mainflux_config = mainflux_config_file:read_all_chars() - local mainflux_config = json.decode(raw_mainflux_config) - log.trace("Config: publishing config for: mainflux") - conn:publish(new_msg({ "config", "mainflux" }, mainflux_config, { retained = true })) -end +-- services/config.lua +-- +-- Config service (strict protocol): +-- - discovers HAL via retained svc/hal/announce +-- - reads a single JSON blob from HAL (string) in strict shape: +-- { : { rev: int, data: table }, ... } +-- - publishes retained cfg/ with { rev=int, data=table } +-- - accepts updates only on: +-- cmd/config/set with payload { service = , data = table } +-- +-- Persisted state location (within HAL): +-- ns = "config" +-- key = "services" + +-- services/config.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local pulse = require 'fibers.pulse' + +local perform = fibers.perform +local named_choice = fibers.named_choice + +local base = require 'devicecode.service_base' +local codec = require 'services.config.codec' +local state = require 'services.config.state' + +local cap_sdk = require 'services.hal.sdk.cap' + +local M = {} + +function M.start(conn, opts) + opts = opts or {} + local svc = base.new(conn, { name = opts.name or 'config', env = opts.env }) + + local timings = opts.timings or {} + + local function numopt(name, default) + local v = timings[name] + return (type(v) == 'number') and v or default + end + + local hal_wait_timeout_s = numopt('hal_wait_timeout_s', 60) + local heartbeat_s = numopt('heartbeat_s', 30.0) + + local persist_debounce_s = numopt('persist_debounce_s', 0.25) + local persist_max_delay_s = numopt('persist_max_delay_s', 5.0) + local persist_retry_initial_s = numopt('persist_retry_initial_s', 1.0) + local persist_retry_max_s = numopt('persist_retry_max_s', 30.0) + + svc:obs_state('boot', { at = svc:wall(), ts = svc:now(), state = 'entered' }) + svc:obs_log('debug', 'service start() entered') + svc:status('starting') + + svc:spawn_heartbeat(heartbeat_s, 'tick') + + local config_cap_monitor = cap_sdk.new_cap_listener(svc.conn, 'fs', 'config') + local config_cap, ferr = config_cap_monitor:wait_for_cap({ + timeout = hal_wait_timeout_s + }) + + if ferr ~= "" then + svc:status('failed', { reason = 'fs cap monitor: ' .. ferr }) + svc:obs_log('error', { what = 'start_failed', err = 'fs cap monitor: ' .. tostring(ferr) }) + error('config: failed to discover fs capability: ' .. tostring(ferr), 0) + end + + if not config_cap then + svc:status('failed', { reason = 'fs cap not found' }) + svc:obs_log('error', { what = 'start_failed', err = 'fs cap not found' }) + error('config: failed to discover fs capability', 0) + end + + local STATE_KEY = os.getenv('CONFIG_TARGET') or error('CONFIG_TARGET env var must be set to load inital config') + + -- current[svc] = { rev=int, data=table } + local current = {} + + local function count_current() + local n = 0 + local keys = {} + for k in pairs(current or {}) do n = n + 1; keys[#keys + 1] = tostring(k) end + table.sort(keys) + return n, table.concat(keys, ',') + end + + local function log_config_summary(reason) + local n, keys = count_current() + if reason == 'loaded' then + svc:info('config_loaded', { + summary = string.format('config loaded target=%s services=%d', tostring(STATE_KEY), n), + reason = reason, + target = STATE_KEY, + services = n, + }) + else + svc:info('config_summary', { + summary = string.format('config summary target=%s services=%d', tostring(STATE_KEY), n), + reason = reason, + target = STATE_KEY, + services = n, + keys = keys ~= '' and keys or nil, + }) + end + end + + local function publish_all_retained() + return state.publish_all_retained(conn, svc, current) + end + + local function load_from_hal() + svc:obs_event('load_begin', { key = STATE_KEY }) + + local reply, read_err = config_cap:call_control( + 'read', + cap_sdk.args.new.FilesystemReadOpts(STATE_KEY .. '.json') + ) + if not reply then + svc:obs_log('error', { what = 'load_failed', err = tostring(read_err) }) + current = {} + publish_all_retained() + svc:obs_event('load_end', { ok = false, reason = 'call_failed' }) + return true + end + if reply.ok ~= true then + svc:obs_log('error', { what = 'load_failed', err = tostring(reply.reason or 'hal read failed') }) + current = {} + publish_all_retained() + svc:obs_event('load_end', { ok = false, reason = 'hal_error' }) + return true + end + + local blob = reply.reason + local decoded, derr = codec.decode_blob_strict(blob or '') + if not decoded then + svc:status('degraded', { reason = 'invalid config JSON', err = tostring(derr) }) + svc:obs_log('error', { what = 'decode_failed', err = tostring(derr) }) + current = {} + publish_all_retained() + svc:obs_event('load_end', { ok = false, reason = tostring(derr) }) + return true + end + + current = decoded + publish_all_retained() + log_config_summary('loaded') + svc:obs_event('load_end', { ok = true }) + return true + end + + load_from_hal() + + local state_cap_monitor = cap_sdk.new_cap_listener(svc.conn, 'fs', 'state') + local state_cap, serr = state_cap_monitor:wait_for_cap({ + timeout = hal_wait_timeout_s + }) + + if serr ~= "" then + svc:status('failed', { reason = 'state cap monitor: ' .. serr }) + svc:obs_log('error', { what = 'start_failed', err = 'state cap monitor: ' .. tostring(serr) }) + error('config: failed to discover state fs capability: ' .. tostring(serr), 0) + end + + if not state_cap then + svc:status('failed', { reason = 'state cap not found' }) + svc:obs_log('error', { what = 'start_failed', err = 'state cap not found' }) + error('config: failed to discover state fs capability', 0) + end + + -- Debounced persistence worker (coalesces writes). + local p = pulse.new() + local dirty = false + local flush_at = math.huge + local flush_deadline = math.huge + + local debounce_s = persist_debounce_s + local max_delay_s = persist_max_delay_s + + local retry_s = persist_retry_initial_s + local retry_max_s = persist_retry_max_s + + local function mark_dirty(reason) + local n = svc:now() + dirty = true + flush_at = n + debounce_s + if flush_deadline == math.huge then + flush_deadline = n + max_delay_s + end + p:signal() + svc:obs_event('persist_dirty', { reason = reason, at = svc:wall(), ts = svc:now() }) + end + + local function persist_snapshot(reason) + local blob, berr = codec.encode_blob(current) + if not blob then + svc:obs_log('error', { what = 'persist_encode_failed', err = tostring(berr) }) + svc:obs_event('persist_end', { ok = false, err = tostring(berr), phase = 'encode' }) + return nil, berr + end + + svc:obs_event('persist_begin', { key = STATE_KEY, reason = reason, bytes = #blob }) + + local reply, wr_err = state_cap:call_control( + 'write', + cap_sdk.args.new.FilesystemWriteOpts(STATE_KEY .. '.json', blob) + ) + if not reply then + svc:obs_log('error', { what = 'persist_failed', err = tostring(wr_err) }) + svc:obs_event('persist_end', { ok = false, err = tostring(wr_err) }) + return nil, wr_err + end + if reply.ok ~= true then + local e = tostring(reply.reason or 'hal write failed') + svc:obs_log('error', { what = 'persist_failed', err = e }) + svc:obs_event('persist_end', { ok = false, err = e }) + return nil, e + end + + svc:obs_event('persist_end', { ok = true }) + log_config_summary('persisted') + return true, nil + end + + fibers.spawn(function() + local seen = p:version() + while true do + if dirty then + local due = math.min(flush_at, flush_deadline) + local dt = due - svc:now() + if dt <= 0 then + local ok, err = persist_snapshot('debounced_flush') + if ok then + dirty = false + flush_at = math.huge + flush_deadline = math.huge + retry_s = 1.0 + svc:status('running') + else + local n = svc:now() + flush_at = n + retry_s + flush_deadline = math.min(flush_deadline, n + max_delay_s) + retry_s = math.min(retry_s * 2, retry_max_s) + svc:status('degraded', { reason = 'persist_failed', err = tostring(err) }) + end + else + local which, a, b = perform(named_choice { + changed = p:changed_op(seen), + timer = sleep.sleep_op(dt):wrap(function() return true end), + }) + if which == 'changed' then + local v, r = a, b + if v == nil and r ~= nil then return end + seen = v or seen + end + end + else + local v, r = perform(p:changed_op(seen)) + if v == nil and r ~= nil then return end + seen = v or seen + end + end + end) + + local set_ep = conn:bind({ 'cmd', 'config', 'set' }, { queue_len = 50 }) + svc:obs_log('debug', 'bound cmd/config/set') + + svc:status('running') + svc:obs_log('debug', 'service running') + + while true do + local req, err = perform(set_ep:recv_op()) + if not req then + svc:status('stopped', { reason = err }) + svc:obs_log('warn', { what = 'endpoint_ended', err = tostring(err) }) + return + end -function config_service:start(rootctx, conn) - log.trace("Starting Config Service") + local payload = req.payload + local service = type(payload) == 'table' and payload.service or nil + local ok, uerr = state.set_service(current, conn, svc, mark_dirty, service, payload, req) - fiber.spawn(function() - publish_config(rootctx, conn) - end) + if ok then + req:reply({ ok = true, persisted = false }) + else + req:fail(tostring(uerr)) + svc:obs_log('warn', { what = 'set_rejected', service = tostring(service), err = tostring(uerr) }) + end + end end -return config_service +return M diff --git a/src/services/config/codec.lua b/src/services/config/codec.lua new file mode 100644 index 00000000..92eafbbf --- /dev/null +++ b/src/services/config/codec.lua @@ -0,0 +1,107 @@ +-- services/config/codec.lua +-- +-- Pure-ish codec helpers for the config service. + +local cjson = require 'cjson.safe' + +local M = {} + +M.JSON_NULL = cjson.null + +function M.is_plain_table(x) + return type(x) == 'table' and getmetatable(x) == nil +end + +function M.strip_nulls(x, seen) + if x == M.JSON_NULL then return nil end + if type(x) ~= 'table' then return x end + if getmetatable(x) ~= nil then return x end + + seen = seen or {} + if seen[x] then return x end + seen[x] = true + + for k, v in pairs(x) do + local nv = M.strip_nulls(v, seen) + if nv == nil then + x[k] = nil + else + x[k] = nv + end + end + return x +end + +function M.deepcopy_plain(x, seen) + if type(x) ~= 'table' then + return x + end + if getmetatable(x) ~= nil then + error('deepcopy_plain: metatables are not supported', 2) + end + + seen = seen or {} + if seen[x] then + return seen[x] + end + + local out = {} + seen[x] = out + + for k, v in pairs(x) do + local nk = M.deepcopy_plain(k, seen) + local nv = M.deepcopy_plain(v, seen) + out[nk] = nv + end + + return out +end + +function M.decode_blob_strict(blob) + local decoded, jerr = cjson.decode(blob) + if decoded == nil then + return nil, 'json_decode_failed: ' .. tostring(jerr) + end + if not M.is_plain_table(decoded) then + return nil, 'invalid_shape: root must be a table' + end + + M.strip_nulls(decoded) + + local out = {} + for svc, rec in pairs(decoded) do + if type(svc) ~= 'string' or svc == '' then + return nil, 'invalid_shape: service key must be non-empty string' + end + if not M.is_plain_table(rec) then + return nil, 'invalid_shape: record must be a table for ' .. svc + end + if type(rec.rev) ~= 'number' then + return nil, 'invalid_shape: rev must be a number for ' .. svc + end + if not M.is_plain_table(rec.data) then + return nil, 'invalid_shape: data must be a table for ' .. svc + end + + if type(rec.data.schema) ~= 'string' or rec.data.schema == '' then + return nil, 'invalid_shape: data.schema must be a non-empty string for ' .. svc + end + + out[svc] = { + rev = math.floor(rec.rev), + data = M.deepcopy_plain(rec.data), + } + end + + return out, nil +end + +function M.encode_blob(current) + local encoded, err = cjson.encode(current) + if encoded == nil then + return nil, 'json_encode_failed: ' .. tostring(err) + end + return encoded, nil +end + +return M diff --git a/src/services/config/state.lua b/src/services/config/state.lua new file mode 100644 index 00000000..ba06d705 --- /dev/null +++ b/src/services/config/state.lua @@ -0,0 +1,71 @@ +-- services/config/state.lua +-- +-- State helpers for the config service. + +local codec = require 'services.config.codec' + +local M = {} + +local function copy_record(rec) + return { + rev = rec.rev, + data = codec.deepcopy_plain(rec.data), + } +end + +function M.publish_all_retained(conn, svc, current) + local n = 0 + for sname, rec in pairs(current) do + n = n + 1 + conn:retain({ 'cfg', sname }, copy_record(rec)) + end + if svc and svc.obs_event then + svc:obs_event('publish_all', { services = n }) + end +end + +function M.set_service(current, conn, svc, mark_dirty, service, payload, msg) + if type(service) ~= 'string' or service == '' then + return nil, 'invalid service' + end + if not codec.is_plain_table(payload) or not codec.is_plain_table(payload.data) then + return nil, 'payload must be { data = table }' + end + + local settings = payload.data + + if type(settings.schema) ~= 'string' or settings.schema == '' then + return nil, 'payload.data.schema must be a non-empty string' + end + + local old = current[service] + local next_rev = (old and type(old.rev) == 'number') and (math.floor(old.rev) + 1) or 1 + local stored_data = codec.deepcopy_plain(settings) + + current[service] = { + rev = next_rev, + data = stored_data, + } + + conn:retain({ 'cfg', service }, { + rev = next_rev, + data = codec.deepcopy_plain(stored_data), + }) + + if svc and svc.obs_event then + local origin = msg and msg.origin or nil + svc:obs_event('set_applied', { + service = service, + rev = next_rev, + origin_kind = origin and origin.kind or nil, + conn_id = origin and origin.conn_id or nil, + }) + end + if mark_dirty then + mark_dirty('set ' .. service) + end + + return true, nil +end + +return M diff --git a/src/services/device.lua b/src/services/device.lua new file mode 100644 index 00000000..7185b96b --- /dev/null +++ b/src/services/device.lua @@ -0,0 +1,45 @@ +-- services/device.lua +-- +-- Public Device service assembly entry point. + +local service = require 'services.device.service' +local config = require 'services.device.config' +local catalogue = require 'services.device.catalogue' +local model = require 'services.device.model' +local projection = require 'services.device.projection' +local publisher = require 'services.device.publisher' +local observer_manager = require 'services.device.observer_manager' +local observer = require 'services.device.observer' +local action_manager = require 'services.device.action_manager' +local action_worker = require 'services.device.action_worker' +local availability = require 'services.device.availability' +local topics = require 'services.device.topics' +local fabric_stage = require 'services.device.fabric_stage' +local backpressure = require 'services.device.backpressure' + +local M = { + service = service, + config = config, + catalogue = catalogue, + model = model, + projection = projection, + publisher = publisher, + observer_manager = observer_manager, + observer = observer, + action_manager = action_manager, + action_worker = action_worker, + availability = availability, + topics = topics, + fabric_stage = fabric_stage, + backpressure = backpressure, +} + +function M.start(conn, opts) + return service.start(conn, opts) +end + +function M.run(scope, params) + return service.run(scope, params) +end + +return M diff --git a/src/services/device/action_manager.lua b/src/services/device/action_manager.lua new file mode 100644 index 00000000..19bad0c2 --- /dev/null +++ b/src/services/device/action_manager.lua @@ -0,0 +1,318 @@ +-- services/device/action_manager.lua +-- +-- Generation-owned action endpoint binding and scoped action admission. + +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local service_events = require 'devicecode.support.service_events' +local publisher = require 'services.device.publisher' +local projection = require 'services.device.projection' +local action_worker = require 'services.device.action_worker' +local request_owner = require 'devicecode.support.request_owner' +local cap_deps_mod = require 'devicecode.support.capability_dependencies' +local dependency_mod = require 'services.device.dependencies' +local backpressure = require 'services.device.backpressure' + +local M = {} + +local function new_request_id(state) + state._action_seq = (state._action_seq or 0) + 1 + return 'act-' .. tostring(state._action_seq) +end + +local function action_identity(generation, component_id, action, request_id) + return { + kind = 'component_action_done', + generation = generation, + component = component_id, + action = action, + request_id = request_id, + } +end + + + +local function completion_reporter(state, identity, label) + local target = state.events_port or state.done_tx + if type(target) == 'table' + and (type(target.emit_required) == 'function' or type(target.send_op) == 'function') + then + return service_events.reporter_for(target, identity, { label = label }) + end + return function (ev) + return queue.try_admit_required(state.done_tx, ev, label) + end +end + +local function request_payload(req) + return type(req) == 'table' and req.payload or nil +end + +local function fail_public_request(req, reason) + if type(req) == 'table' and type(req.fail) == 'function' then + local ok, err = req:fail(reason) + if ok == true or (ok == nil and err == nil) then + -- Local bus Request:fail returns true; a few test doubles return nil. + -- In both cases, the request-level policy has been applied. + return true, nil + end + return nil, err or ('request fail rejected: ' .. tostring(reason)) + end + return nil, 'request has no fail method: ' .. tostring(reason) +end + +local function resolve_action_spec(component, action, req, base_spec) + local mod = type(component) == 'table' and component.module or nil + if type(mod) == 'table' and type(mod.action_spec) == 'function' then + local ok, spec, err = pcall(mod.action_spec, component, action, request_payload(req), base_spec) + if not ok then return nil, tostring(spec) end + if spec == nil then return nil, err or 'unknown_action' end + return spec, nil + end + return base_spec, nil +end + + +local function mapped_action_dependency_key(active, component_id, action) + local by_component = active and active.action_dependency_keys and active.action_dependency_keys[component_id] or nil + return by_component and by_component[action] or nil +end + +local function remember_action_dependency_key(active, component_id, action, key) + if key == nil then return end + active.action_dependency_keys = active.action_dependency_keys or {} + active.action_dependency_keys[component_id] = active.action_dependency_keys[component_id] or {} + active.action_dependency_keys[component_id][action] = key +end + +local function open_dynamic_dependency_manager(state, active, spec) + local deps, err = cap_deps_mod.open(state.conn, { spec }, { + changed_kind = 'device_dependency_changed', + closed_kind = 'device_dependency_closed', + queue_len = state.dependency_queue_len or 8, + full = 'drop_oldest', + }) + if not deps then return nil, err or 'device_dynamic_action_dependency_open_failed' end + active.action_deps = deps + state.action_deps = deps + return true, nil +end + +local function ensure_action_dependency(state, active, component_id, action, action_spec) + local mapped = mapped_action_dependency_key(active, component_id, action) + local spec, err = dependency_mod.action_dependency_spec(component_id, action, action_spec) + if err ~= nil then return nil, err end + if spec == nil then return mapped, nil end + + local key = spec.key + remember_action_dependency_key(active, component_id, action, key) + + if active.action_deps == nil then + local ok, oerr = open_dynamic_dependency_manager(state, active, spec) + if ok ~= true then return nil, oerr end + elseif type(active.action_deps.ensure) == 'function' then + local ok, eerr = active.action_deps:ensure(spec) + if ok ~= true then return nil, eerr or 'device_dynamic_action_dependency_add_failed' end + elseif active.action_deps:dependency(key) == nil then + return nil, 'device_action_dependency_manager_cannot_add:' .. tostring(key) + end + + if type(state.update_dependency_model) == 'function' then + state.update_dependency_model(state, active) + end + + return key, nil +end + +local function component_actions(component) + local out = { ['get-status'] = true } + for action in pairs(component.actions or {}) do + out[action] = true + end + return out +end + +function M.bind_generation(active, params) + active.action_eps = active.action_eps or {} + params = params or {} + + for component_id, component in pairs((active.catalogue and active.catalogue.components) or {}) do + for action in pairs(component_actions(component)) do + local key = component_id .. ':' .. action + local ep, err = publisher.bind_now(params.conn, projection.component_cap_topic(component_id, action), { + queue_len = params.action_queue_len or backpressure.policy.action_endpoints.default_len, + full = backpressure.policy.action_endpoints.full, + }) + if not ep then + -- Binding public endpoints is an immediate generation-owned effect. + -- If later admission fails, undo what has already been exposed; do + -- not wait for generation finalisation to make the failed start safe. + local ok_unbind, unbind_err = M.unbind_generation(active, params.conn) + if ok_unbind ~= true then + return nil, tostring(err or 'bind failed') .. '; rollback unbind failed: ' .. tostring(unbind_err) + end + return nil, err + end + active.action_eps[key] = { + key = key, + generation = active.generation, + component = component_id, + action = action, + ep = ep, + bound = true, + } + end + end + + return true, nil +end + +function M.unbind_generation(active, conn) + local eps = (active and active.action_eps) or {} + local cleared = {} + + for key, rec in pairs(eps) do + local ep = rec and rec.ep or nil + local has_immediate_unbind = (conn ~= nil and type(conn.unbind) == 'function') + or (type(ep) == 'table' and type(ep.unbind) == 'function') + + if rec and rec.bound == true or has_immediate_unbind then + local ok, err = publisher.unbind_now(conn, ep) + if ok ~= true then + return nil, err or ('action endpoint unbind failed: ' .. tostring(key)) + end + end + + cleared[#cleared + 1] = key + end + + -- Only forget records after all durable public endpoint cleanup has succeeded. + for i = 1, #cleared do + eps[cleared[i]] = nil + end + if active then active.action_eps = eps end + return true, nil +end + +function M.endpoint_sources(active) + local sources = {} + for key, rec in pairs((active and active.action_eps) or {}) do + sources[#sources + 1] = rec + end + table.sort(sources, function (a, b) return tostring(a.key) < tostring(b.key) end) + return sources +end + +function M.status_reply(state, req, component_id) + local rec = state.model:component_snapshot(component_id) + if not rec then + if type(req.fail) == 'function' then return req:fail('unknown_component') end + return nil, 'unknown_component' + end + + local payload = projection.component_payloads(component_id, rec, state.now()).component + if type(req.reply) == 'function' then return req:reply(payload) end + return nil, 'request has no reply method' +end + +function M.start_action(state, req, rec) + local active = state.active + if not active or rec.generation ~= active.generation then + return fail_public_request(req, 'stale_generation') + end + + local component = active.catalogue.components[rec.component] + if not component then + return fail_public_request(req, 'unknown_component') + end + + if rec.action == 'get-status' then + return M.status_reply(state, req, rec.component) + end + + local base_action_spec = component.actions and component.actions[rec.action] + if not base_action_spec then + return fail_public_request(req, 'unknown_action') + end + + local action_spec, spec_err = resolve_action_spec(component, rec.action, req, base_action_spec) + if not action_spec then + return fail_public_request(req, spec_err or 'unknown_action') + end + + local dep_key, dep_err = ensure_action_dependency(state, active, rec.component, rec.action, action_spec) + if dep_err ~= nil then + return fail_public_request(req, 'dependency_invalid:' .. tostring(dep_err)) + end + if dep_key ~= nil then + if not active.action_deps or active.action_deps:available(dep_key) ~= true then + return fail_public_request(req, 'dependency_unavailable:' .. tostring(dep_key)) + end + end + + local request_id = new_request_id(state) + local owner = request_owner.new(req) + + local handle, err = scoped_work.start { + lifetime_scope = active.action_root or active.scope, + reaper_scope = active.action_root or active.scope, + report_scope = state.scope, + identity = action_identity(active.generation, rec.component, rec.action, request_id), + + setup = function (scope) + -- Request ownership is created before scoped-work admission. The + -- action-scope finaliser remains the structural fallback, while the + -- cancel_owned_now hook resolves the caller-visible request immediately + -- when the coordinator cancels the scoped action before the worker body + -- has had a chance to run. Both paths are idempotent. + scope:finally(function (_, status, primary) + action_worker.finalise_owner(owner, status, primary) + end) + + return { + request_owner = owner, + cancel_owned_now = function (reason) + action_worker.finalise_owner(owner, 'cancelled', reason) + return true + end, + } + end, + + cancel_op = owner:caller_cancel_op(), + + run = function (scope, setup) + return action_worker.run(scope, { + conn = state.conn, + request = req, + request_owner = setup.request_owner, + request_id = request_id, + component_id = rec.component, + action = rec.action, + action_spec = action_spec, + timeout = state.action_timeout, + fabric_client = state.fabric_client, + open_source = state.open_source, + open_source_op = state.open_source_op, + terminate_source = state.terminate_source, + }) + end, + report = completion_reporter(state, action_identity(active.generation, rec.component, rec.action, request_id), 'device_action_done_report_failed'), + } + + if not handle then + owner:fail_once(err or 'action_start_failed') + return nil, err or 'action_start_failed' + end + + state.pending_actions[request_id] = { + generation = active.generation, + component = rec.component, + action = rec.action, + dependency_key = dep_key, + handle = handle, + } + + return true, nil +end + +return M diff --git a/src/services/device/action_worker.lua b/src/services/device/action_worker.lua new file mode 100644 index 00000000..242aea9b --- /dev/null +++ b/src/services/device/action_worker.lua @@ -0,0 +1,325 @@ +-- services/device/action_worker.lua +-- +-- Scoped component action execution. Action workers own caller-visible request +-- resolution and any temporary resources they create. + +local fibers = require 'fibers' +local cond = require 'fibers.cond' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local scope_mod = require 'fibers.scope' +local request_owner = require 'devicecode.support.request_owner' +local resource = require 'devicecode.support.resource' +local fabric_stage = require 'services.device.fabric_stage' + +local M = {} + +local function request_payload(req) + if type(req) ~= 'table' then return nil end + return req.payload +end + +local function default_call_op(conn, topic, payload, opts) + if type(conn) == 'table' and type(conn.call_op) == 'function' then + return conn:call_op(topic, payload, opts) + end + + return nil, 'connection does not support call_op' +end + +local function normalise_reply(reply) + if type(reply) == 'table' and reply.ok == false then + return nil, reply.reason or reply.err or 'action failed' + end + if type(reply) == 'table' and reply.ok == true and reply.reason ~= nil then + return reply.reason, nil + end + return reply, nil +end + +local function public_result(status, fields) + fields = fields or {} + fields.public_status = status + fields.ok = (status == 'succeeded') + if fields.err ~= nil and fields.error == nil then + fields.error = fields.err + end + if fields.error ~= nil and fields.err == nil then + fields.err = fields.error + end + return fields +end + + +local INTERNAL_PUBLIC_KEYS = { + source_owner = true, +} + +local function sanitise_public_payload(value, seen) + if type(value) ~= 'table' then return value end + seen = seen or {} + if seen[value] then return nil end + seen[value] = true + + local out = {} + for k, v in pairs(value) do + if not INTERNAL_PUBLIC_KEYS[k] and type(v) ~= 'function' then + out[k] = sanitise_public_payload(v, seen) + end + end + seen[value] = nil + return out +end + +local function public_reply_payload(result) + if type(result) == 'table' and result.reply_payload ~= nil then + return sanitise_public_payload(result.reply_payload) + end + return sanitise_public_payload(result) +end + +local function finalise_owner(owner, status, primary) + if owner:done() then return end + if status == 'cancelled' then + owner:finalise_unresolved(primary or 'cancelled') + elseif status == 'failed' then + owner:finalise_unresolved(primary or 'failed') + else + owner:finalise_unresolved(primary or 'terminated') + end +end + +local function run_rpc_op(ctx, owner) + local action = ctx.action_spec + local payload = request_payload(ctx.request) + local call_ev, cerr = default_call_op(ctx.conn, action.call_topic, payload, { + timeout = action.timeout or ctx.timeout, + deadline = ctx.deadline, + }) + + if not call_ev then + local reason = cerr or 'rpc_unavailable' + owner:fail_once(reason) + return op.always(public_result('unavailable', { err = reason })) + end + + return call_ev:wrap(function (reply, err) + if reply == nil then + local reason = err or 'rpc_failed' + owner:fail_once(reason) + return public_result('remote_failed', { err = reason }) + end + + local value, rerr = normalise_reply(reply) + if value == nil and rerr ~= nil then + owner:fail_once(rerr) + return public_result('remote_failed', { err = rerr }) + end + + owner:reply_once(value) + return public_result('succeeded', { value = value, reply_payload = value }) + end) +end + +local function is_op(v) + return type(v) == 'table' and getmetatable(v) == op.Op +end + +local function open_source_op(ctx) + local payload = request_payload(ctx.request) or {} + local opener = ctx.open_source_op + + if type(opener) == 'function' then + local ev, err = opener(payload, ctx) + if ev == nil then return op.always(nil, err or 'source_required') end + if not is_op(ev) then return op.always(ev, nil) end + return ev + end + + opener = ctx.open_source + if type(opener) == 'function' then + local source, err = opener(payload, ctx) + if is_op(source) then return source end + return op.always(source, err) + end + + if payload.source ~= nil then return op.always(payload.source, nil) end + if payload.artifact ~= nil then return op.always(payload.artifact, nil) end + return op.always(nil, 'source_required') +end + + +local function source_terminator(ctx) + local terminate = ctx.terminate_source + if terminate ~= nil and type(terminate) ~= 'function' then + error('terminate_source must be a function', 0) + end + return terminate +end + +local function run_fabric_stage_op(_, ctx, owner) + -- Fabric staging is a meaningful sub-lifetime of the action: it may open a + -- source, own termination responsibility, wait on a Fabric client, and hand ownership away. Keep + -- that lifetime visible as a child scope rather than hiding it in a guard or + -- in the parent action finaliser. + return fibers.run_scope_op(function (stage_scope) + local source, err = fibers.perform(open_source_op(ctx)) + if not source then + owner:fail_once(err or 'source_required') + return public_result('rejected', { err = err or 'source_required' }) + end + + local owned = resource.owned(source, { + terminate = source_terminator(ctx), + label = 'device fabric-stage source termination', + }) + stage_scope:finally(function (_, status, primary) + owned:terminate_checked( + primary or status or 'fabric_stage_terminated', + 'device fabric-stage source termination failed' + ) + end) + + local result, ferr = fibers.perform(fabric_stage.stage_source_op(stage_scope, { + client = ctx.fabric_client, + source_owner = owned, + component = ctx.component_id, + action = ctx.action, + link_id = ctx.action_spec.link_id, + target = ctx.action_spec.target, + chunk_size = ctx.action_spec.chunk_size, + artifact_store = ctx.action_spec.artifact_store, + request_payload = request_payload(ctx.request), + timeout = ctx.action_spec.timeout or ctx.timeout, + deadline = ctx.deadline, + })) + + if not result then + local reason = ferr or 'fabric_stage_failed' + owner:fail_once(reason) + return public_result('remote_failed', { err = reason }) + end + + local public_payload = public_reply_payload(result) + owner:reply_once(public_payload) + return public_result('succeeded', { + value = public_payload, + reply_payload = public_payload, + }) + end):wrap(function (st, _rep, result_or_primary) + if st == 'ok' then + return result_or_primary + end + + local reason = result_or_primary or st or 'fabric_stage_failed' + owner:fail_once(reason) + return public_result(st == 'cancelled' and 'cancelled' or 'failed', { err = reason }) + end) +end + +local function action_op(scope, ctx, owner) + local action_spec = ctx.action_spec + if action_spec.kind == 'fabric_stage' then + return run_fabric_stage_op(scope, ctx, owner) + end + return run_rpc_op(ctx, owner) +end + +local function perform_with_timeout(scope, ctx, ev) + local timeout = ctx.timeout + if ctx.action_spec and type(ctx.action_spec.timeout) == 'number' then + timeout = ctx.action_spec.timeout + end + + if timeout ~= nil then + if type(timeout) ~= 'number' then + error('action timeout must be a number', 0) + end + if timeout <= 0 then + scope:cancel('timeout') + return public_result('timed_out', { err = 'timeout' }) + end + + -- Timeout is an ownership decision, not just a competing result. A small + -- timer fibre cancels the action scope; cancellation then cascades by + -- parentage into any HAL/Fabric child scopes. This avoids aborting a child + -- run_scope_op with a generic losing-choice reason before the action scope + -- has been cancelled with the public timeout reason. + local done = cond.new() + local ok_timer, timer_err = fibers.spawn(function () + local timed_out = fibers.perform(fibers.boolean_choice( + sleep.sleep_op(timeout), + done:wait_op() + )) + if timed_out then + scope:cancel('timeout') + end + end) + if ok_timer ~= true then + -- The action body may first run after its owning action scope has + -- already been cancelled by generation replacement or service shutdown. + -- In that case timeout-helper admission being closed is not a worker + -- failure; preserve the scope cancellation reason so the request-owner + -- finaliser reports the real ownership decision. + local st, reason = scope:status() + if st == 'cancelled' then + error(scope_mod.cancelled(reason or timer_err or 'cancelled'), 0) + end + error(timer_err or 'action timeout timer start failed', 0) + end + + scope:finally(function () + done:signal() + end) + + local result = fibers.perform(ev) + done:signal() + return result + end + + return fibers.perform(ev) +end + +function M.run(scope, ctx) + ctx = ctx or {} + if fibers.current_scope and fibers.current_scope() ~= scope then + error('device action worker must run in its owning scope', 0) + end + local req = assert(ctx.request, 'device action worker requires request') + local action_spec = assert(ctx.action_spec, 'device action worker requires action_spec') + local owner = ctx.request_owner or request_owner.new(req, ctx.request_owner_opts) + + -- When action work is launched through action_manager, the request owner is + -- installed in scoped_work setup before the worker fibre is admitted. That + -- closes the ownership gap where a generation can be cancelled after + -- admission but before this worker body has run. Direct users of + -- action_worker.run still get the same finaliser here. + if ctx.request_owner == nil then + scope:finally(function (_, status, primary) + finalise_owner(owner, status, primary) + end) + end + + local result = perform_with_timeout(scope, ctx, action_op(scope, ctx, owner)) + result = result or public_result('failed', { err = 'action returned no result' }) + if result.public_status == 'timed_out' and not owner:done() then + owner:fail_once(result.err or result.error or 'timeout') + end + + return { + component = ctx.component_id, + action = ctx.action, + request_id = ctx.request_id, + ok = result.ok == true, + public_status = result.public_status or (result.ok == true and 'succeeded' or 'remote_failed'), + value = result.value or result.reply_payload, + reply_payload = result.reply_payload, + err = result.err or result.error, + error = result.error or result.err, + } +end + +M.finalise_owner = finalise_owner +M.sanitise_public_payload = sanitise_public_payload + +return M diff --git a/src/services/device/architecture_note.md b/src/services/device/architecture_note.md new file mode 100644 index 00000000..358fcad9 --- /dev/null +++ b/src/services/device/architecture_note.md @@ -0,0 +1,62 @@ +## Device + +### Role + +Device is the authoritative appliance composition service. + +It observes raw host, member, Fabric and HAL-facing truth, then publishes curated appliance component truth. + +Device is not a raw RPC router. It owns public component semantics. + +### Public surfaces + +Device publishes canonical appliance truth under: + +```text +state/device/identity +state/device/components +state/device/component/ +state/device/component//software +state/device/component//update +``` + +Device publishes curated component interfaces under: + +```text +cap/component//meta +cap/component//status +cap/component//rpc/ +``` + +Provider-native backing facts and interfaces remain under `raw/...`. Device capability metadata should include provenance references to backing raw topics and interfaces. + +### Lifetime shape + +```text +device service scope + coordinator + service-owned model + publisher + generation scope + component observers + component action bindings + fabric-stage action workers +``` + +### Rules + +The service owns the public model. + +A generation owns catalogue-dependent observers and action bindings. + +Observers own raw watches. + +Actions own caller-visible requests and blocking work. + +Device should not mirror every raw capability. It should curate stable appliance-level component surfaces. + +Fabric-stage actions use Fabric transfer targets such as: + +```text +updater/main +``` diff --git a/src/services/device/availability.lua b/src/services/device/availability.lua new file mode 100644 index 00000000..de831b0a --- /dev/null +++ b/src/services/device/availability.lua @@ -0,0 +1,167 @@ +-- services/device/availability.lua +-- Pure availability and health policy. +-- +-- This module consumes only catalogue/model snapshot data. It performs no Ops, +-- owns no resources, and has no service-side effects. + +local tablex = require 'shared.table' + +local M = {} + +local copy_value = tablex.deep_copy + +local function seen_any(states) + if type(states) ~= 'table' then return false end + for _, meta in pairs(states) do + if type(meta) == 'table' and meta.seen == true then return true end + end + return false +end + +local table_or_empty = tablex.table_or_empty + +local function normalise_policy(rec) + local p = table_or_empty(rec and rec.availability) + return { + missing_required = p.missing_required or 'unavailable', + malformed_required = p.malformed_required or 'degraded', + source_stale = p.source_stale or 'degraded', + source_down = p.source_down or 'unavailable', + no_observation = p.no_observation or 'unknown', + } +end + +local function status(availability, health, reason, details) + local available = availability == 'available' or availability == 'degraded' + return { + availability = availability, + available = available, + ready = availability == 'available', + health = health or (availability == 'available' and 'ok' or 'unknown'), + reason = reason, + details = details or {}, + } +end + +local function set_status(base, fields) + fields = fields or {} + local out = copy_value(base or {}) + for k, v in pairs(fields) do out[k] = v end + if out.availability == nil then + out.availability = out.available and (out.ready and 'available' or 'degraded') or 'unknown' + end + out.available = out.availability == 'available' or out.availability == 'degraded' + out.ready = out.availability == 'available' + return out +end + +function M.any_observation_seen(rec) + if type(rec) ~= 'table' then return false end + return seen_any(rec.fact_state) or seen_any(rec.event_state) +end + +function M.required_facts_ready(rec, required) + if type(required) ~= 'table' or #required == 0 then + return M.any_observation_seen(rec) + end + + local fact_state = type(rec) == 'table' and rec.fact_state or nil + for i = 1, #required do + local meta = fact_state and fact_state[required[i]] or nil + if not (type(meta) == 'table' and meta.seen == true) then + return false, required[i] + end + end + return true, nil +end + +local function health_from_fact(raw_health) + if raw_health == nil then return nil, nil end + if type(raw_health) == 'string' then + local h = raw_health:lower() + if h == 'ok' then return 'ok', nil end + if h == 'degraded' then return 'warning', 'health_warning' end + if h == 'failed' then return 'fault', 'health_fault' end + return raw_health, nil + end + if type(raw_health) == 'table' then + if raw_health.state ~= nil then return health_from_fact(tostring(raw_health.state)) end + end + return nil, nil +end + +local function has_only_events(rec) + return type(rec) == 'table' + and type(rec.events) == 'table' and next(rec.events) ~= nil + and not (type(rec.facts) == 'table' and next(rec.facts) ~= nil) +end + +local function base_component_status(rec) + if type(rec) ~= 'table' then + return status('unknown', 'unknown', 'unknown_component') + end + + local policy = normalise_policy(rec) + local details = {} + local source_err = rec.source_err + local source_up = rec.source_up == true + local seen = M.any_observation_seen(rec) + + if source_err ~= nil then + local reason = tostring(source_err) + if reason == 'stale' then + return status(policy.source_stale, 'warning', 'source_stale', { source_reason = reason }) + end + return status(policy.source_down, 'unknown', reason, { source_reason = reason }) + end + + if not source_up then + if seen then + return status('degraded', 'warning', 'source_not_confirmed') + end + if has_only_events(rec) then + return status('available', 'ok', nil) + end + return status(policy.no_observation, 'unknown', 'no_observation') + end + + local ready, missing = M.required_facts_ready(rec, rec.required_facts) + if not ready then + details.missing_fact = missing + return status(policy.missing_required, 'unknown', 'missing_required_fact', details) + end + + local health, health_reason = health_from_fact(rec.raw_facts and rec.raw_facts.health or nil) + if health == 'fault' then + return status('unavailable', 'fault', health_reason or 'health_fault') + elseif health == 'warning' or health == 'degraded' then + return status('degraded', 'warning', health_reason or 'health_warning') + end + + if ready or has_only_events(rec) then + return status('available', health or 'ok', nil) + end + + return status('degraded', health or 'warning', 'not_ready') +end + +--- Compute public availability for a component snapshot/catalogue entry. +--- Component modules may refine the generic status by exposing: +--- availability(base_status, rec) -> status_table +function M.component_status(rec) + local base = base_component_status(rec) + local mod = type(rec) == 'table' and rec.module or nil + if type(mod) == 'table' and type(mod.availability) == 'function' then + local ok, refined = pcall(mod.availability, base, rec) + if ok and type(refined) == 'table' then + return set_status(base, refined) + elseif not ok then + return status('degraded', 'warning', 'availability_refine_failed', { error = tostring(refined) }) + end + end + return set_status(base) +end + +M.copy_value = copy_value + +return M diff --git a/src/services/device/backpressure.lua b/src/services/device/backpressure.lua new file mode 100644 index 00000000..05cb893f --- /dev/null +++ b/src/services/device/backpressure.lua @@ -0,0 +1,75 @@ +-- services/device/backpressure.lua +-- +-- Explicit Device publication/backpressure policy. +-- +-- This module is deliberately data-only. Coordinators and workers import these +-- constants so queue/full-policy choices are visible in code rather than hidden +-- in ad hoc literals. + +local M = {} + +M.policy = { + -- Completion events are safety/ownership relevant. They must not be silently + -- dropped: reporters use required immediate admission and fail the observing + -- scope if this queue cannot accept a completion while healthy. + completions = { + queue_full = 'fail_observing_scope', + full = 'reject_newest', + default_len = 64, + }, + + -- Observations are semantic input to the Device model. Observer reporters use + -- required immediate admission, so failure to admit is observer/service failure + -- rather than silent telemetry loss. + observations = { + queue_full = 'fail_observing_scope', + full = 'reject_newest', + default_len = 128, + }, + + -- Public action endpoint queues reject new requests when full. The public bus + -- endpoint reports this to the caller; Device does not block the coordinator to + -- wait for endpoint capacity. + action_endpoints = { + queue_full = 'reject_request', + full = 'reject_newest', + default_len = 32, + }, + + -- Raw observer feeds are low-level input queues. Repeated retained/fact/event + -- updates can be coalesced by the observer/model; stale old raw items may be + -- dropped in favour of newer state. + observer_feeds = { + queue_full = 'drop_oldest_raw', + full = 'drop_oldest', + default_len = 8, + }, + + -- Publication is dirty-state coalesced. Repeated model changes mark the same + -- component dirty and are flushed as retained/public state; publication failure + -- fails the Device coordinator rather than being silently dropped. + publication = { + policy = 'coalesce_dirty_state', + failure = 'fail_service', + }, + + -- Source-down/stale observations do not fail the service by themselves. They + -- update the model through availability policy and normally publish degraded or + -- unavailable public state. + availability = { + source_down = 'mark_degraded_or_unavailable', + observer_failure = 'record_and_mark_source_down', + }, +} + +function M.snapshot() + local out = {} + for k, v in pairs(M.policy) do + local copy = {} + for kk, vv in pairs(v) do copy[kk] = vv end + out[k] = copy + end + return out +end + +return M diff --git a/src/services/device/catalogue.lua b/src/services/device/catalogue.lua new file mode 100644 index 00000000..0022c1ac --- /dev/null +++ b/src/services/device/catalogue.lua @@ -0,0 +1,443 @@ +-- services/device/catalogue.lua +-- +-- Pure effective component catalogue construction. +-- The catalogue owns no live resources and performs no Ops. + +local topics = require 'services.device.topics' +local mcu_schema = require 'services.device.schemas.mcu' +local component_host = require 'services.device.component_host' +local component_mcu = require 'services.device.component_mcu' +local tablex = require 'shared.table' + +local M = {} + +local copy_array = tablex.array_copy +local copy_value = tablex.deep_copy +local deep_equal = tablex.deep_equal + +local function is_array(t) + return type(t) == 'table' and #t > 0 and tablex.is_array(t) +end + + +local function component_module_for(id, spec) + if type(spec) ~= 'table' then spec = {} end + if type(spec.module) == 'table' then return spec.module end + if type(spec.module) == 'string' then + local ok, mod = pcall(require, spec.module) + if ok and type(mod) == 'table' then return mod end + local ok2, mod2 = pcall(require, 'services.device.component_' .. spec.module) + if ok2 and type(mod2) == 'table' then return mod2 end + local ok3, mod3 = pcall(require, 'services.device.' .. spec.module) + if ok3 and type(mod3) == 'table' then return mod3 end + return nil, 'unknown component module: ' .. tostring(spec.module) + end + + local kind = spec.kind or spec.subtype or spec.class or id + if kind == 'host' or kind == 'cm5' or spec.class == 'host' then return component_host end + if kind == 'mcu' or spec.member_class == 'mcu' then return component_mcu end + return nil, nil +end + +local function public_method_name(name) + name = tostring(name or '') + return name:gsub('_', '-') +end + +local function assert_topic(topic, where) + if not is_array(topic) then + error(where .. ' must be a non-empty topic array', 0) + end + return copy_array(topic) +end + +local function normalise_fact_routes(facts, where) + local out = {} + if facts == nil then return out end + if type(facts) ~= 'table' then error(where .. ': facts must be a table', 0) end + + for fact_name, spec in pairs(facts) do + if type(fact_name) ~= 'string' or fact_name == '' then + error(where .. ': fact names must be non-empty strings', 0) + end + + local topic = spec + if type(spec) == 'table' and spec.watch_topic ~= nil then + topic = spec.watch_topic + end + + out[fact_name] = { + name = fact_name, + watch_topic = assert_topic(topic, where .. ': fact ' .. fact_name), + } + end + + return out +end + +local function normalise_event_routes(events, where) + local out = {} + if events == nil then return out end + if type(events) ~= 'table' then error(where .. ': events must be a table', 0) end + + for event_name, spec in pairs(events) do + if type(event_name) ~= 'string' or event_name == '' then + error(where .. ': event names must be non-empty strings', 0) + end + + local topic = spec + if type(spec) == 'table' and spec.subscribe_topic ~= nil then + topic = spec.subscribe_topic + end + + out[event_name] = { + name = event_name, + subscribe_topic = assert_topic(topic, where .. ': event ' .. event_name), + } + end + + return out +end + +local function opt_target(v, where) + if v == nil then return nil end + if type(v) ~= 'string' or v == '' then + error(where .. ': fabric_stage target must be a non-empty string', 0) + end + return v +end + +local function opt_pos_int(v, where, default) + if v == nil then return default end + local n = tonumber(v) + if type(n) ~= 'number' or n <= 0 or n % 1 ~= 0 then + error(where .. ': fabric_stage chunk_size must be a positive integer', 0) + end + return n +end + +local function normalise_actions(actions, where) + local out = {} + if actions == nil then return out end + if type(actions) ~= 'table' then error(where .. ': actions must be a table', 0) end + + for action_name, spec in pairs(actions) do + if type(action_name) ~= 'string' or action_name == '' then + error(where .. ': action names must be non-empty strings', 0) + end + + local public_name = public_method_name(action_name) + + if type(spec) == 'table' then + if spec[1] ~= nil and spec.kind == nil and spec.call_topic == nil then + error(where .. ': action ' .. action_name .. ' must be a table with kind and call_topic', 0) + end + local kind = spec.kind or 'rpc' + + if kind == 'rpc' then + if spec.topic ~= nil then + error(where .. ': action ' .. action_name .. ' uses deprecated topic; use call_topic', 0) + end + if spec.timeout ~= nil then + error(where .. ': action ' .. action_name .. ' uses deprecated timeout; use timeout_s', 0) + end + out[public_name] = { + name = public_name, + kind = 'rpc', + call_topic = assert_topic(spec.call_topic, where .. ': action ' .. action_name .. ' call_topic'), + timeout = tonumber(spec.timeout_s) or nil, + dependency = copy_value(spec.dependency), + } + elseif kind == 'fabric_stage' then + if spec.timeout ~= nil then + error(where .. ': action ' .. action_name .. ' uses deprecated timeout; use timeout_s', 0) + end + + if spec.receiver ~= nil then + error(where .. ': action ' .. action_name .. ' uses deprecated receiver; use target', 0) + end + local target = opt_target(spec.target, where .. ': action ' .. action_name .. ' target') + if target == nil then + error(where .. ': action ' .. action_name .. ' requires fabric_stage target', 0) + end + + out[public_name] = { + name = public_name, + kind = 'fabric_stage', + link_id = spec.link_id, + target = target, + chunk_size = opt_pos_int(spec.chunk_size, where .. ': action ' .. action_name .. ' chunk_size', nil), + artifact_store = spec.artifact_store or 'main', + timeout = tonumber(spec.timeout_s) or nil, + dependency = copy_value(spec.dependency), + } + else + error(where .. ': unsupported action kind for ' .. action_name .. ': ' .. tostring(kind), 0) + end + else + error(where .. ': action ' .. action_name .. ' must be a table with kind and call_topic', 0) + end + end + + return out +end + +local function empty_state_for_routes(routes) + local raw, state = {}, {} + for name in pairs(routes or {}) do + raw[name] = nil + state[name] = { seen = false, updated_at = nil } + end + return raw, state +end + +local function normalise_component(id, spec) + spec = type(spec) == 'table' and spec or {} + local where = 'component ' .. tostring(id) + local mod, mod_err = component_module_for(id, spec) + if mod_err then error(where .. ': ' .. mod_err, 0) end + + local facts = normalise_fact_routes(spec.facts, where) + local events = normalise_event_routes(spec.events, where) + if next(facts) == nil and next(events) == nil then + error(where .. ': at least one fact or event is required', 0) + end + + local raw_facts, fact_state = empty_state_for_routes(facts) + local raw_events, event_state = empty_state_for_routes(events) + local observe_opts = spec.observe_opts + + return { + id = id, + name = id, + kind = spec.kind or (mod and mod.kind) or spec.subtype or spec.class or id, + module = mod, + class = spec.class or ((mod == component_host) and 'host' or 'member'), + subtype = spec.subtype or (mod and mod.kind) or id, + role = spec.role or 'member', + member = spec.member or id, + member_class = spec.member_class or spec.subtype or spec.class or id, + link_class = spec.link_class, + display = copy_value(spec.display or {}), + present = spec.present ~= false, + observe_opts = type(observe_opts) == 'table' and copy_value(observe_opts) or {}, + required_facts = copy_array(spec.required_facts), + availability = copy_value(spec.availability or {}), + facts = facts, + events = events, + actions = normalise_actions(spec.actions, where), + raw_facts = raw_facts, + fact_state = fact_state, + raw_events = raw_events, + event_state = event_state, + source_up = false, + source_err = nil, + } +end + +local function default_components() + return { + cm5 = normalise_component('cm5', { + class = 'host', + subtype = 'cm5', + role = 'primary', + member = 'local', + required_facts = { 'software', 'updater' }, + facts = { + software = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'software'), + updater = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'updater'), + health = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'health'), + }, + actions = { + ['prepare-update'] = { kind = 'rpc', call_topic = topics.raw_host_cap_rpc('updater', 'updater', 'cm5', 'prepare') }, + ['stage-update'] = { kind = 'rpc', call_topic = topics.raw_host_cap_rpc('updater', 'updater', 'cm5', 'stage') }, + ['commit-update'] = { kind = 'rpc', call_topic = topics.raw_host_cap_rpc('updater', 'updater', 'cm5', 'commit') }, + }, + }), + + mcu = normalise_component('mcu', { + class = 'member', + subtype = 'mcu', + role = 'controller', + member = 'mcu', + required_facts = { 'software', 'updater' }, + facts = mcu_schema.member_fact_topics('mcu'), + events = mcu_schema.member_event_topics('mcu'), + actions = { + ['restart'] = { kind = 'rpc', call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') }, + ['prepare-update'] = { kind = 'rpc', call_topic = topics.raw_member_cap_rpc('mcu', 'updater', 'main', 'prepare-update') }, + ['stage-update'] = { + kind = 'fabric_stage', + target = 'updater/main', + chunk_size = 2048, + artifact_store = 'main', + }, + ['commit-update'] = { kind = 'rpc', call_topic = topics.raw_member_cap_rpc('mcu', 'updater', 'main', 'commit-update') }, + }, + }), + } +end + +local function non_empty_string(v) + return type(v) == 'string' and v ~= '' +end + +local function reject_assembly_aliases(rec, ctx) + if type(rec) ~= 'table' then return end + if rec.provider ~= nil or rec.provider_id ~= nil or rec.provider_surface_id ~= nil or rec.surface ~= nil then + error(ctx .. ' must use component and observed_surface only', 0) + end +end + +local function validate_assembly_endpoint(endpoint, ctx) + if type(endpoint) ~= 'table' then error(ctx .. ' must be a table', 0) end + reject_assembly_aliases(endpoint, ctx) + if not non_empty_string(endpoint.component) then error(ctx .. '.component must be a non-empty string', 0) end + if not non_empty_string(endpoint.observed_surface) then error(ctx .. '.observed_surface must be a non-empty string', 0) end +end + +local function normalise_assembly(raw) + if raw == nil then return {} end + if type(raw) ~= 'table' then error('device catalogue assembly must be a table', 0) end + local surfaces = raw.surfaces or {} + if type(surfaces) ~= 'table' then error('device catalogue assembly.surfaces must be a table', 0) end + for id, rec in pairs(surfaces) do + if not non_empty_string(id) then error('device catalogue assembly surface id must be a non-empty string', 0) end + if type(rec) ~= 'table' then error('device catalogue assembly surface ' .. tostring(id) .. ' must be a table', 0) end + reject_assembly_aliases(rec, 'device catalogue assembly surface ' .. tostring(id)) + if not non_empty_string(rec.component) then error('device catalogue assembly surface ' .. tostring(id) .. '.component must be a non-empty string', 0) end + if not non_empty_string(rec.observed_surface) then error('device catalogue assembly surface ' .. tostring(id) .. '.observed_surface must be a non-empty string', 0) end + end + local links = raw.links or {} + if type(links) ~= 'table' then error('device catalogue assembly.links must be a table', 0) end + for id, link in pairs(links) do + if not non_empty_string(id) then error('device catalogue assembly link id must be a non-empty string', 0) end + if type(link) ~= 'table' then error('device catalogue assembly link ' .. tostring(id) .. ' must be a table', 0) end + if link.a ~= nil then validate_assembly_endpoint(link.a, 'device catalogue assembly link ' .. tostring(id) .. '.a') end + if link.b ~= nil then validate_assembly_endpoint(link.b, 'device catalogue assembly link ' .. tostring(id) .. '.b') end + end + return copy_value(raw) +end + +local function components_from_config(raw) + if raw == nil then return default_components() end + if type(raw) ~= 'table' then error('device catalogue config must be a table or nil', 0) end + + local list = raw.components + if list == nil then + return default_components() + end + if type(list) ~= 'table' then + error('device catalogue components must be a table', 0) + end + + local out = {} + for key, spec in pairs(list) do + local id = key + if type(spec) == 'table' and spec.id ~= nil then id = spec.id end + if type(id) ~= 'string' or id == '' then + error('device catalogue component id must be a non-empty string', 0) + end + out[id] = normalise_component(id, spec) + end + return out +end + +function M.build(raw, opts) + opts = opts or {} + local components = components_from_config(raw) + return { + kind = 'device_catalogue', + schema = raw and raw.schema or nil, + components = components, + assembly = normalise_assembly(raw and raw.assembly or nil), + meta = copy_value((raw and raw.meta) or {}), + } +end + +function M.copy(v) + return copy_value(v) +end + +function M.equal(a, b) + return deep_equal(a, b) +end + +function M.component(catalogue, component_id) + return catalogue and catalogue.components and catalogue.components[component_id] or nil +end + +local PUBLIC_ONLY_COMPONENT_FIELDS = { + display = true, + present = true, +} + +local PUBLIC_ONLY_CATALOGUE_FIELDS = { + meta = true, +} + +local function material_value(v, seen) + if type(v) ~= 'table' then return v end + seen = seen or {} + if seen[v] then return seen[v] end + local out = {} + seen[v] = out + for k, vv in pairs(v) do + if k == 'components' and type(vv) == 'table' then + local comps = {} + out[k] = comps + for id, comp in pairs(vv) do + local c = {} + comps[id] = c + for ck, cv in pairs(comp) do + if not PUBLIC_ONLY_COMPONENT_FIELDS[ck] then + c[material_value(ck, seen)] = material_value(cv, seen) + end + end + end + elseif not PUBLIC_ONLY_CATALOGUE_FIELDS[k] then + out[material_value(k, seen)] = material_value(vv, seen) + end + end + return out +end + +function M.material_view(catalogue) + return material_value(catalogue) +end + +function M.materially_equal(a, b) + return deep_equal(M.material_view(a), M.material_view(b)) +end + +function M.publicly_equal(a, b) + return deep_equal(a, b) +end + +function M.public_component_ids_changed(a, b) + local out = {} + local seen = {} + local ac = (a and a.components) or {} + local bc = (b and b.components) or {} + + for id in pairs(ac) do seen[id] = true end + for id in pairs(bc) do seen[id] = true end + + for id in pairs(seen) do + if not deep_equal(ac[id], bc[id]) then + out[#out + 1] = id + end + end + + table.sort(out) + return out +end + +function M.public_metadata_changed(a, b) + return not deep_equal(a, b) and M.materially_equal(a, b) +end + +M.public_method_name = public_method_name +M.default_components = default_components + +return M diff --git a/src/services/device/component_host.lua b/src/services/device/component_host.lua new file mode 100644 index 00000000..f74e1f43 --- /dev/null +++ b/src/services/device/component_host.lua @@ -0,0 +1,113 @@ +-- services/device/component_host.lua +-- Pure host component normalisation, action description, and availability overlay. + +local model = require 'services.device.model' +local tablex = require 'shared.table' + +local M = { kind = 'host' } + +local table_or_empty = tablex.table_or_empty + +local function copy(v) + return model.copy_value(v) +end + +local first_non_nil = tablex.first_non_nil + +function M.normalise_software(raw) + raw = table_or_empty(raw) + return { + version = raw.version or raw.fw_version, + build = raw.build or raw.build_id, + image_id = raw.image_id, + boot_id = raw.boot_id, + bootedfw = raw.bootedfw, + targetfw = raw.targetfw, + upgrade_available = raw.upgrade_available, + hw_revision = raw.hw_revision, + serial = raw.serial, + board_revision = raw.board_revision, + } +end + +function M.normalise_updater(raw) + raw = table_or_empty(raw) + return { + state = raw.state or raw.raw_state or raw.status or raw.kind, + raw_state = raw.raw_state, + staged = raw.staged, + artifact_ref = raw.artifact_ref, + artifact_meta = copy(raw.artifact_meta), + expected_image_id = raw.expected_image_id, + last_error = raw.last_error or raw.err, + } +end + +function M.normalise_health(raw) + if type(raw) ~= 'table' then return raw end + return { + health = first_non_nil(raw.health, raw.state, raw.status, raw.kind), + fault = first_non_nil(raw.fault, raw.fault_code, raw.error, raw.err), + details = copy(raw.details), + } +end + +function M.normalise_fact(fact, raw) + if fact == 'software' then return M.normalise_software(raw), nil end + if fact == 'updater' then return M.normalise_updater(raw), nil end + if fact == 'health' then return M.normalise_health(raw), nil end + return copy(raw), nil +end +function M.normalise_event(_, raw) + return copy(raw), nil +end +function M.compose(raw_facts) + raw_facts = table_or_empty(raw_facts) + return { + software = M.normalise_software(raw_facts.software), + updater = M.normalise_updater(raw_facts.updater), + health = M.normalise_health(raw_facts.health), + power = copy(raw_facts.power or {}), + environment = copy(raw_facts.environment or {}), + runtime = copy(raw_facts.runtime or {}), + alerts = copy(raw_facts.alerts or {}), + } +end + +function M.action_spec(_, _, _, base_spec) + return copy(base_spec), nil +end + +function M.availability(base, rec) + local out = copy(base) + local facts = table_or_empty(rec and rec.raw_facts) + local sw_seen = rec and rec.fact_state and rec.fact_state.software and rec.fact_state.software.seen == true + local health = facts.health + + -- Host software is useful but should not make a visible host wholly absent + -- where other host sources are alive. This keeps status reads informative + -- during updater bring-up or partial raw-source availability. + if out.reason == 'missing_required_fact' and out.details and out.details.missing_fact == 'software' then + out.availability = 'degraded' + out.health = 'warning' + out.reason = 'missing_host_software' + end + + if type(health) == 'table' and health.fault ~= nil and health.fault ~= false then + out.availability = 'unavailable' + out.health = 'fault' + out.reason = 'host_fault' + elseif type(health) == 'string' and (health == 'fault' or health == 'failed' or health == 'error') then + out.availability = 'unavailable' + out.health = 'fault' + out.reason = 'host_fault' + elseif sw_seen and out.availability == 'unknown' then + out.availability = 'degraded' + out.health = out.health or 'warning' + out.reason = out.reason or 'partial_host_state' + end + + return out +end + +return M diff --git a/src/services/device/component_mcu.lua b/src/services/device/component_mcu.lua new file mode 100644 index 00000000..3800d8a6 --- /dev/null +++ b/src/services/device/component_mcu.lua @@ -0,0 +1,59 @@ +-- services/device/component_mcu.lua +-- Pure MCU component normalisation, action description, and availability overlay. + +local schema = require 'services.device.schemas.mcu' +local model = require 'services.device.model' + +local M = { kind = 'mcu' } + +local function copy(v) + return model.copy_value(v) +end + +function M.normalise_fact(fact, raw) + if fact == 'software' then return schema.normalise_software(raw), nil end + if fact == 'updater' then return schema.normalise_updater(raw), nil end + if fact == 'health' then return schema.normalise_health(raw), nil end + if fact == 'power_charger' then return schema.normalise_charger(raw), nil end + if fact == 'charger_alert' then return schema.normalise_charger_alert(raw), nil end + return copy(raw), nil +end +function M.normalise_event(event, raw) + if event == 'charger_alert' then return schema.normalise_charger_alert(raw), nil end + return copy(raw), nil +end +function M.compose(raw_facts, raw_events) + return schema.compose(raw_facts, raw_events) +end + +function M.action_spec(_, _, _, base_spec) + return copy(base_spec), nil +end + +function M.availability(base, rec) + local out = copy(base) + local facts = type(rec) == 'table' and rec.raw_facts or {} + local boot = type(facts) == 'table' and facts.boot or nil + local health = type(facts) == 'table' and facts.health or nil + + if type(boot) == 'table' and boot.mode == 'bootloader' and out.availability == 'available' then + out.availability = 'degraded' + out.health = 'warning' + out.reason = 'bootloader_mode' + end + + local health_state = schema.normalise_health(health) + if health_state == 'failed' then + out.availability = 'unavailable' + out.health = 'fault' + out.reason = 'mcu_fault' + elseif health_state == 'degraded' and out.availability == 'available' then + out.availability = 'degraded' + out.health = 'warning' + out.reason = 'mcu_degraded' + end + + return out +end + +return M diff --git a/src/services/device/component_switch.lua b/src/services/device/component_switch.lua new file mode 100644 index 00000000..3d34d1d9 --- /dev/null +++ b/src/services/device/component_switch.lua @@ -0,0 +1,77 @@ +-- services/device/component_switch.lua +-- Component composition for switch-fabric members/providers. +-- +-- This module lets cfg/device compose either a pre-devicecode HTTP-backed +-- switch provider or a later fabric member into the ordinary appliance device +-- view. It deliberately keeps wired topology interpretation in services.wired. + +local model = require 'services.device.model' +local tablex = require 'shared.table' + +local M = { kind = 'switch' } + +local function copy(v) return model.copy_value(v) end +local function table_or_empty(v) return tablex.table_or_empty(v) end +local function first_non_nil(...) + for i = 1, select('#', ...) do + local v = select(i, ...) + if v ~= nil then return v end + end + return nil +end + +function M.normalise_fact(fact, raw) + if fact == 'wired_observation_status' or fact == 'wired_observation_surfaces' or fact == 'wired_observation_topology' then + return copy(raw or {}), nil + end + return copy(raw), nil +end + +function M.normalise_event(_, raw) + return copy(raw), nil +end + +function M.compose(raw_facts) + raw_facts = table_or_empty(raw_facts) + local status = table_or_empty(raw_facts.wired_observation_status) + local surfaces_payload = table_or_empty(raw_facts.wired_observation_surfaces) + local surfaces = surfaces_payload.surfaces or surfaces_payload + local topology = table_or_empty(raw_facts.wired_observation_topology) + return { + health = { + health = (status.available == false) and 'degraded' or 'ok', + fault = status.err or status.reason, + details = copy(status), + }, + observed = { + wired = { + status = copy(status), + surfaces = copy(surfaces or {}), + topology = copy(topology or {}), + }, + }, + runtime = { + provider_mode = status.mode, + driver = status.driver, + }, + } +end + +function M.availability(base, rec) + local out = copy(base) + local facts = table_or_empty(rec and rec.raw_facts) + local status = table_or_empty(facts.wired_observation_status) + local state = first_non_nil(status.state, status.availability) + if status.available == false or state == 'not_configured' or state == 'unavailable' then + out.availability = 'degraded' + out.health = 'warning' + out.reason = status.reason or status.err or state or 'wired_observation_unavailable' + elseif status.available == true or state == 'available' then + out.availability = 'available' + out.health = 'ok' + out.reason = nil + end + return out +end + +return M diff --git a/src/services/device/config.lua b/src/services/device/config.lua new file mode 100644 index 00000000..977133c7 --- /dev/null +++ b/src/services/device/config.lua @@ -0,0 +1,49 @@ +-- services/device/config.lua +-- +-- Raw Device configuration validation and normalisation. + +local catalogue = require 'services.device.catalogue' + +local M = {} + +M.SCHEMA = 'devicecode.config/device/1' + +local function unwrap_config(raw) + if raw == nil then return nil end + if type(raw) ~= 'table' then + return nil, 'device config must be a table' + end + if raw.data ~= nil and type(raw.data) == 'table' then + return raw.data, nil + end + return raw, nil +end + +function M.normalise(raw) + local cfg, err = unwrap_config(raw) + if err then return nil, err end + if cfg ~= nil and cfg.schema ~= nil and cfg.schema ~= M.SCHEMA then + return nil, 'unsupported device config schema: ' .. tostring(cfg.schema) + end + return cfg, nil +end + + +function M.to_catalogue(raw) + local cfg, err = M.normalise(raw) + if err then return nil, err end + + local ok, cat_or_err = pcall(function () + return catalogue.build(cfg) + end) + if not ok then + return nil, tostring(cat_or_err) + end + return cat_or_err, nil +end + +function M.catalogues_equal(a, b) + return catalogue.materially_equal(a, b) +end + +return M diff --git a/src/services/device/dependencies.lua b/src/services/device/dependencies.lua new file mode 100644 index 00000000..83f61aa3 --- /dev/null +++ b/src/services/device/dependencies.lua @@ -0,0 +1,50 @@ +-- services/device/dependencies.lua +-- Pure helpers for explicit component-action capability dependencies. + +local M = {} + +local function non_empty(v) return type(v) == 'string' and v ~= '' end + +function M.action_dependency_key(component_id, action) + return 'action:' .. tostring(component_id) .. ':' .. tostring(action) +end + +local function copy_dependency(dep) + if type(dep) ~= 'table' then return nil end + local out = {} + for k, v in pairs(dep) do out[k] = v end + return out +end + +function M.action_dependency_spec(component_id, action, action_spec) + local dep = copy_dependency(type(action_spec) == 'table' and action_spec.dependency or nil) + if dep == nil then return nil end + if not non_empty(dep.class) then return nil, 'action dependency requires class' end + if (dep.raw_kind == 'host' or dep.raw_kind == 'member') and not non_empty(dep.source) then + return nil, 'raw action dependency requires source' + end + dep.key = dep.key or M.action_dependency_key(component_id, action) + dep.id = dep.id or 'main' + dep.required = dep.required ~= false + dep.component = component_id + dep.action = action + return dep, nil +end + +function M.catalogue_dependencies(catalogue) + local specs, map = {}, {} + for component_id, component in pairs((catalogue and catalogue.components) or {}) do + for action, action_spec in pairs((component and component.actions) or {}) do + local spec, err = M.action_dependency_spec(component_id, action, action_spec) + if err ~= nil then return nil, nil, err end + if spec ~= nil then + specs[#specs + 1] = spec + map[component_id] = map[component_id] or {} + map[component_id][action] = spec.key + end + end + end + return specs, map, nil +end + +return M diff --git a/src/services/device/fabric_stage.lua b/src/services/device/fabric_stage.lua new file mode 100644 index 00000000..1d80a845 --- /dev/null +++ b/src/services/device/fabric_stage.lua @@ -0,0 +1,95 @@ +-- services/device/fabric_stage.lua +-- +-- Fabric-stage client helper with explicit source-owner transfer. Device opens +-- and owns the source; the Fabric client must either take that owner before +-- returning success, or leave it owned by Device on rejection/failure. + +local op_mod = require 'fibers.op' + +local M = {} + +local function is_op(v) + return type(v) == 'table' and getmetatable(v) == op_mod.Op +end + +local function call_transfer_op(client, params, opts) + if type(client) == 'table' and type(client.send_blob_op) == 'function' then + return client:send_blob_op(params, opts) + end + return nil, 'fabric_stage client does not provide send_blob_op' +end + +local function map_result(source_owner, result, perr) + if result == nil then + return nil, perr or 'fabric_stage_failed' + end + if type(result) == 'table' and result.ok == false then + return nil, result.err or result.error or result.reason or 'fabric_stage_failed' + end + if source_owner and type(source_owner.is_owned) == 'function' and source_owner:is_owned() then + return nil, 'fabric_stage client did not take source ownership' + end + return result, nil +end + +--- Stage an owned source through Fabric as an Op. +--- +--- The Fabric client must expose send_blob_op(params, opts). params.source_owner +--- is an owned finaliser-safe resource. On success the client must have taken +--- ownership by detaching or handing off that owner. On failure it must leave +--- ownership with the caller unless it has already accepted responsibility. +--- +--- Return shape when performed: +--- result, nil +--- nil, err +function M.stage_source_op(_, params) + params = params or {} + local source_owner = params.source_owner + if source_owner == nil or type(source_owner.value) ~= 'function' or type(source_owner.is_owned) ~= 'function' then + return op_mod.always(nil, 'source_owner_required') + end + + local payload = type(params.request_payload) == 'table' and params.request_payload or {} + local transfer_meta = type(payload.meta) == 'table' and payload.meta or { + kind = 'firmware', + component = params.component, + job_id = payload.job_id, + image_id = payload.expected_image_id or payload.image_id, + format = payload.format or 'dcmcu-v1', + } + + local ev, err = call_transfer_op(params.client, { + component = params.component, + action = params.action, + job_id = payload.job_id, + request_id = payload.request_id, + xfer_id = payload.xfer_id, + link_id = params.link_id or payload.link_id, + target = params.target or payload.target, + size = payload.size, + digest_alg = payload.digest_alg, + digest = payload.digest, + chunk_size = payload.chunk_size or params.chunk_size, + meta = transfer_meta, + source_owner = source_owner, + artifact_store = params.artifact_store, + request = payload, + }, { + timeout = params.timeout, + deadline = params.deadline, + }) + + if not ev then + return op_mod.always(nil, err or 'fabric_stage_unavailable') + end + + if is_op(ev) then + return ev:wrap(function (result, perr) + return map_result(source_owner, result, perr) + end) + end + + return op_mod.always(map_result(source_owner, ev, nil)) +end + +return M diff --git a/src/services/device/mcu_metrics.lua b/src/services/device/mcu_metrics.lua new file mode 100644 index 00000000..043ee5da --- /dev/null +++ b/src/services/device/mcu_metrics.lua @@ -0,0 +1,168 @@ +-- services/device/mcu_metrics.lua +-- +-- Small bridge from the composed MCU component view to the transitional metrics +-- bus contract. + +local projection = require 'services.device.projection' + +local M = {} + +local function table_or_empty(v) + return type(v) == 'table' and v or {} +end + +local function first_number(...) + for i = 1, select('#', ...) do + local n = tonumber(select(i, ...)) + if n ~= nil then return n end + end + return nil +end + +local function first_boolean(t, names) + if type(t) ~= 'table' then return nil end + for i = 1, #names do + local v = t[names[i]] + if type(v) == 'boolean' then return v end + end + return nil +end + +local function emit(out, name, value, namespace) + if value == nil then return end + out[#out + 1] = { + name = name, + value = value, + namespace = namespace, + } +end + +local function emit_boolean_group(out, group, specs, namespace_prefix) + for i = 1, #specs do + local spec = specs[i] + local value = first_boolean(group, spec.keys) + emit(out, spec.metric, value, { + namespace_prefix[1], + namespace_prefix[2], + namespace_prefix[3], + namespace_prefix[4], + namespace_prefix[5], + spec.metric, + }) + end +end + +local system_flags = { + { metric = 'charger_enabled', keys = { 'charger_enabled' } }, + { metric = 'mppt_en_pin', keys = { 'mppt_en_pin' } }, + { metric = 'equalize_req', keys = { 'equalize_req' } }, + { metric = 'drvcc_good', keys = { 'drvcc_good' } }, + { metric = 'cell_count_error', keys = { 'cell_count_error' } }, + { metric = 'ok_to_charge', keys = { 'ok_to_charge' } }, + { metric = 'no_rt', keys = { 'no_rt' } }, + { metric = 'thermal_shutdown', keys = { 'thermal_shutdown' } }, + { metric = 'vin_ovlo', keys = { 'vin_ovlo' } }, + { metric = 'vin_gt_vbat', keys = { 'vin_gt_vbat' } }, + { metric = 'intvcc_gt_4p3v', keys = { 'intvcc_gt_4p3v' } }, + { metric = 'intvcc_gt_2p8v', keys = { 'intvcc_gt_2p8v' } }, +} + +local status_flags = { + { metric = 'iin_limited', keys = { 'iin_limited', 'iin_limit_active' } }, + { metric = 'uvcl_active', keys = { 'uvcl_active', 'vin_uvcl_active' } }, + { metric = 'cc_phase', keys = { 'cc_phase', 'const_current' } }, + { metric = 'cv_phase', keys = { 'cv_phase', 'const_voltage' } }, +} + +local state_flags = { + { metric = 'bat_short', keys = { 'bat_short', 'bat_short_fault' } }, + { metric = 'bat_missing', keys = { 'bat_missing', 'bat_missing_fault' } }, + { metric = 'max_charge_time_fault', keys = { 'max_charge_time_fault' } }, + { metric = 'c_over_x_term', keys = { 'c_over_x_term' } }, + { metric = 'timer_term', keys = { 'timer_term' } }, + { metric = 'ntc_pause', keys = { 'ntc_pause' } }, + { metric = 'precharge', keys = { 'precharge' } }, + { metric = 'cccv', keys = { 'cccv', 'cccv_charge' } }, + { metric = 'absorb', keys = { 'absorb', 'absorb_charge' } }, + { metric = 'equalize', keys = { 'equalize', 'equalize_charge' } }, + { metric = 'suspended', keys = { 'suspended', 'charger_suspended' } }, +} + +local function has_group(raw_charger, bits_name) + return raw_charger[bits_name] ~= nil +end + +function M.collect_component(rec) + if type(rec) ~= 'table' then return {} end + + local view = projection.component_view('mcu', rec) + local raw_facts = table_or_empty(rec.raw_facts) + local out = {} + + local runtime = table_or_empty(view.runtime) + local memory = table_or_empty(runtime.memory) + emit(out, 'alloc', first_number(memory.alloc_bytes, memory.alloc), { 'mcu', 'sys', 'mem', 'alloc' }) + + local environment = table_or_empty(view.environment) + local temperature = table_or_empty(environment.temperature) + local humidity = table_or_empty(environment.humidity) + local deci_c = first_number(temperature.deci_c) + if deci_c ~= nil then + emit(out, 'core', deci_c / 10, { 'mcu', 'env', 'temperature', 'core' }) + end + local rh_x100 = first_number(humidity.rh_x100) + if rh_x100 ~= nil then + emit(out, 'core', rh_x100 / 100, { 'mcu', 'env', 'humidity', 'core' }) + end + + local power = table_or_empty(view.power) + local battery = table_or_empty(power.battery) + local temp_mc = first_number(battery.temp_mC) + if temp_mc ~= nil then + emit(out, 'internal', temp_mc / 1000, { 'mcu', 'power', 'temperature', 'internal' }) + end + emit(out, 'vbat', + first_number(battery.pack_mV, battery.vbat_mV, battery.vbat), + { 'mcu', 'power', 'battery', 'internal', 'vbat' }) + emit(out, 'ibat', + first_number(battery.ibat_mA, battery.ibat), + { 'mcu', 'power', 'battery', 'internal', 'ibat' }) + + local charger = table_or_empty(power.charger) + emit(out, 'vin', first_number(charger.vin_mV, charger.vin), { 'mcu', 'power', 'charger', 'internal', 'vin' }) + emit(out, 'vsys', first_number(charger.vsys_mV, charger.vsys), { 'mcu', 'power', 'charger', 'internal', 'vsys' }) + emit(out, 'iin', first_number(charger.iin_mA, charger.iin), { 'mcu', 'power', 'charger', 'internal', 'iin' }) + + local raw_charger = table_or_empty(raw_facts.power_charger) + if has_group(raw_charger, 'system_bits') then + emit_boolean_group(out, table_or_empty(charger.system), system_flags, + { 'mcu', 'power', 'charger', 'internal', 'system' }) + end + if has_group(raw_charger, 'status_bits') then + emit_boolean_group(out, table_or_empty(charger.status), status_flags, + { 'mcu', 'power', 'charger', 'internal', 'status' }) + end + if has_group(raw_charger, 'state_bits') then + emit_boolean_group(out, table_or_empty(charger.state), state_flags, + { 'mcu', 'power', 'charger', 'internal', 'state' }) + end + + return out +end + +function M.publish_component(svc, rec) + if type(svc) ~= 'table' or type(svc.obs_metric) ~= 'function' then return true, nil, 0 end + + local metrics = M.collect_component(rec) + for i = 1, #metrics do + local metric = metrics[i] + svc:obs_metric(metric.name, { + value = metric.value, + namespace = metric.namespace, + }) + end + + return true, nil, #metrics +end + +return M diff --git a/src/services/device/model.lua b/src/services/device/model.lua new file mode 100644 index 00000000..461a041b --- /dev/null +++ b/src/services/device/model.lua @@ -0,0 +1,272 @@ +-- services/device/model.lua +-- +-- Pulse-backed public Device state model. +-- The model is non-suspending: it stores snapshots, applies observations, and +-- exposes changed_op(version). It does not publish, call HAL, or spawn work. + +local pulse = require 'fibers.pulse' +local availability = require 'services.device.availability' +local tablex = require 'shared.table' + +local M = {} + +local copy_value = tablex.deep_copy +local deep_equal = tablex.deep_equal + +local function now_or_nil() + local ok, fibers = pcall(require, 'fibers') + if ok and fibers and type(fibers.now) == 'function' then + return fibers.now() + end + return nil +end + +local function empty_snapshot() + return { + generation = 0, + catalogue = nil, + components = {}, + dependencies = {}, + } +end + + +local function recompute_component_status(rec) + if type(rec) ~= 'table' then return rec end + rec.status = availability.component_status(rec) + rec.available = rec.status.available + rec.ready = rec.status.ready + rec.health = rec.status.health + rec.reason = rec.status.reason + return rec +end + +local function copy_component_template(component) + local rec = copy_value(component) + rec.raw_facts = copy_value(rec.raw_facts or {}) + rec.fact_state = copy_value(rec.fact_state or {}) + rec.raw_events = copy_value(rec.raw_events or {}) + rec.event_state = copy_value(rec.event_state or {}) + rec.source_up = rec.source_up == true + rec.source_err = rec.source_err + rec.last_action = copy_value(rec.last_action) + return recompute_component_status(rec) +end + +local function apply_if_changed(self, next_snapshot) + if self._closed then + return nil, self._closed_reason or 'terminated' + end + + if deep_equal(self._snapshot, next_snapshot) then + return false, self:version() + end + + self._snapshot = copy_value(next_snapshot) + local v = self._pulse:signal() + return true, v +end + +local DeviceModel = {} +DeviceModel.__index = DeviceModel + +function DeviceModel:version() + return self._pulse:version() +end + +function DeviceModel:is_terminated() + return self._closed +end + +function DeviceModel:why() + return self._closed_reason +end + +function DeviceModel:snapshot() + return copy_value(self._snapshot) +end + +function DeviceModel:component_snapshot(component_id) + local rec = self._snapshot.components and self._snapshot.components[component_id] or nil + return copy_value(rec) +end + +function DeviceModel:set_snapshot(snapshot) + return apply_if_changed(self, copy_value(snapshot or empty_snapshot())) +end + +function DeviceModel:apply_catalogue(generation, catalogue) + local next_snapshot = empty_snapshot() + next_snapshot.generation = generation or 0 + next_snapshot.catalogue = copy_value(catalogue) + + for id, component in pairs((catalogue and catalogue.components) or {}) do + next_snapshot.components[id] = copy_component_template(component) + end + + return apply_if_changed(self, next_snapshot) +end + + +function DeviceModel:update_catalogue_metadata(generation, catalogue) + local snap = self:snapshot() + if snap.generation ~= generation then + return false, self:version(), 'stale_generation' + end + + catalogue = catalogue or {} + snap.catalogue = copy_value(catalogue) + + for id, component in pairs((catalogue and catalogue.components) or {}) do + local rec = snap.components[id] + if rec then + -- Public metadata refresh only. Material catalogue changes are handled by + -- starting a new generation, so live observation state is intentionally + -- preserved here. + rec.display = copy_value(component.display or {}) + rec.present = component.present ~= false + rec.availability = copy_value(component.availability or rec.availability or {}) + recompute_component_status(rec) + end + end + + return apply_if_changed(self, snap) +end + + +function DeviceModel:update_dependencies(generation, dependencies) + local snap = self:snapshot() + if snap.generation ~= generation then + return false, self:version(), 'stale_generation' + end + snap.dependencies = copy_value(dependencies or {}) + return apply_if_changed(self, snap) +end + +function DeviceModel:apply_observation(generation, observation) + local snap = self:snapshot() + if snap.generation ~= generation then + return false, self:version(), 'stale_generation' + end + + observation = observation or {} + local component_id = observation.component + local rec = component_id and snap.components[component_id] or nil + if not rec then + return false, self:version(), 'unknown_component' + end + + local tag = observation.tag + local ts = observation.at or observation.ts or now_or_nil() + + if tag == 'fact_retained' or tag == 'fact_changed' then + local fact = observation.fact + if type(fact) ~= 'string' or fact == '' then + return false, self:version(), 'missing_fact' + end + rec.raw_facts[fact] = copy_value(observation.payload) + rec.fact_state[fact] = rec.fact_state[fact] or {} + rec.fact_state[fact].seen = observation.payload ~= nil + rec.fact_state[fact].updated_at = ts + rec.source_up = true + rec.source_err = nil + elseif tag == 'fact_unretained' then + local fact = observation.fact + if type(fact) ~= 'string' or fact == '' then + return false, self:version(), 'missing_fact' + end + rec.raw_facts[fact] = nil + rec.fact_state[fact] = rec.fact_state[fact] or {} + rec.fact_state[fact].seen = false + rec.fact_state[fact].updated_at = ts + elseif tag == 'event' or tag == 'event_seen' then + local event = observation.event + if type(event) ~= 'string' or event == '' then + return false, self:version(), 'missing_event' + end + rec.raw_events[event] = copy_value(observation.payload) + rec.event_state[event] = rec.event_state[event] or { count = 0 } + rec.event_state[event].seen = true + rec.event_state[event].updated_at = ts + rec.event_state[event].count = (tonumber(rec.event_state[event].count) or 0) + 1 + rec.source_up = true + rec.source_err = nil + elseif tag == 'source_down' then + rec.source_up = false + rec.source_err = observation.reason or 'source_down' + else + return false, self:version(), 'unknown_observation_tag' + end + + recompute_component_status(rec) + return apply_if_changed(self, snap) +end + +function DeviceModel:apply_source_down(generation, component_id, reason) + return self:apply_observation(generation, { + component = component_id, + tag = 'source_down', + reason = reason or 'source_down', + }) +end + +function DeviceModel:apply_action_result(generation, action_result) + local snap = self:snapshot() + if snap.generation ~= generation then + return false, self:version(), 'stale_generation' + end + + action_result = action_result or {} + local component_id = action_result.component + local rec = component_id and snap.components[component_id] or nil + if not rec then + return false, self:version(), 'unknown_component' + end + + rec.last_action = copy_value(action_result) + recompute_component_status(rec) + return apply_if_changed(self, snap) +end + +function DeviceModel:changed_op(seen) + if type(seen) ~= 'number' or seen < 0 or seen % 1 ~= 0 then + error('device.model.changed_op: seen must be a non-negative integer', 2) + end + + return self._pulse:changed_op(seen):wrap(function (version, reason) + if version == nil then + return nil, nil, reason or self._closed_reason or 'terminated' + end + return version, self:snapshot(), nil + end) +end + +function DeviceModel:terminate(reason) + if self._closed then + if self._closed_reason == nil and reason ~= nil then + self._closed_reason = reason + end + return true + end + + self._closed = true + self._closed_reason = reason or 'terminated' + self._pulse:close(self._closed_reason) + return true +end + +local function new(initial) + return setmetatable({ + _snapshot = copy_value(initial or empty_snapshot()), + _pulse = pulse.new(0), + _closed = false, + _closed_reason = nil, + }, DeviceModel) +end + +M.new = new +M.DeviceModel = DeviceModel +M.copy_value = copy_value +M.deep_equal = deep_equal + +return M diff --git a/src/services/device/observer.lua b/src/services/device/observer.lua new file mode 100644 index 00000000..041cdba5 --- /dev/null +++ b/src/services/device/observer.lua @@ -0,0 +1,250 @@ +-- services/device/observer.lua +-- +-- Component observer worker. Owns raw retained watches and event subscriptions +-- for one component in one generation, and emits semantic observation events. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local queue = require 'devicecode.support.queue' +local model = require 'services.device.model' +local backpressure = require 'services.device.backpressure' + +local M = {} + +local function emit_required(tx, ev) + return queue.assert_admit_required(tx, ev, 'device_observation_event_report_failed', 3) +end + +local function close_all(conn, set, method) + for _, item in pairs(set or {}) do + if method == 'unwatch_retained' then + bus_cleanup.unwatch_retained(conn, item) + elseif method == 'unsubscribe' then + bus_cleanup.unsubscribe(conn, item) + else + bus_cleanup.close_feed(item) + end + end +end + +local function recv_payload(ev) + if ev == nil then return nil end + if type(ev) == 'table' and ev.payload ~= nil then return ev.payload end + return ev +end + + +local function normalise_with_component(rec, kind, name, payload) + local mod = type(rec) == 'table' and rec.module or nil + local fn + if type(mod) == 'table' then + if kind == 'fact' then + fn = mod.normalise_fact + else + fn = mod.normalise_event + end + end + + if type(fn) ~= 'function' then + return model.copy_value(payload), nil + end + + local ok, value, err = pcall(fn, name, payload, rec) + if not ok then return nil, tostring(value) end + return value, err +end + +local function open_fact_watch(conn, route, opts) + return bus_cleanup.watch_retained(conn, route.watch_topic, { + queue_len = opts.queue_len or opts.fact_queue_len or backpressure.policy.observer_feeds.default_len, + full = opts.full or backpressure.policy.observer_feeds.full, + replay = true, + }) +end + +local function open_event_sub(conn, route, opts) + return bus_cleanup.subscribe(conn, route.subscribe_topic, { + queue_len = opts.queue_len or opts.event_queue_len or backpressure.policy.observer_feeds.default_len, + full = opts.full or backpressure.policy.observer_feeds.full, + }) +end + +local function freshness_predicate(rec, fact_watches, event_subs) + local required = {} + local required_count = 0 + for _, name in ipairs(rec.required_facts or {}) do + if type(name) == 'string' and name ~= '' and fact_watches[name] then + required[name] = true + required_count = required_count + 1 + end + end + + local has_facts = next(fact_watches) ~= nil + local has_events = next(event_subs) ~= nil + + return function (item) + if not item then return false end + if item.kind == 'fact' then + if required_count == 0 then return true end + return required[item.name] == true + end + if item.kind == 'event' then + return (not has_facts) and has_events + end + return false + end +end + +local function make_choice(fact_watches, event_subs, stale_deadline) + local arms = {} + + for fact_name, watch in pairs(fact_watches) do + arms['fact:' .. fact_name] = watch:recv_op():wrap(function (ev, err) + return { kind = 'fact', name = fact_name, ev = ev, err = err } + end) + end + + for event_name, sub in pairs(event_subs) do + arms['event:' .. event_name] = sub:recv_op():wrap(function (msg, err) + return { kind = 'event', name = event_name, msg = msg, err = err } + end) + end + + if type(stale_deadline) == 'number' then + arms._stale = sleep.sleep_until_op(stale_deadline):wrap(function () + return { kind = 'stale' } + end) + end + + return fibers.named_choice(arms):wrap(function (_, item) + return item + end) +end + +function M.run(scope, ctx) + ctx = ctx or {} + local rec = ctx.component or {} + local conn = assert(ctx.conn, 'device observer requires conn') + local tx = assert(ctx.tx, 'device observer requires tx') + local generation = assert(ctx.generation, 'device observer requires generation') + local component_id = assert(ctx.component_id or rec.id, 'device observer requires component_id') + local opts = rec.observe_opts or {} + + local function emit(ev) + ev.kind = ev.kind or 'component_observation' + ev.generation = generation + ev.component = ev.component or component_id + emit_required(tx, ev) + end + + local fact_watches = {} + local event_subs = {} + + -- Ownership is established before the first raw handle is opened. If any + -- later source-down report or open step fails, already-opened handles remain + -- owned by this observer scope and are released by its finaliser. + scope:finally(function () + close_all(conn, fact_watches, 'unwatch_retained') + close_all(conn, event_subs, 'unsubscribe') + end) + + for fact_name, route in pairs(rec.facts or {}) do + local watch, err = open_fact_watch(conn, route, opts) + if watch then + fact_watches[fact_name] = watch + else + emit({ tag = 'source_down', reason = tostring(err or ('watch_failed:' .. fact_name)) }) + end + end + + for event_name, route in pairs(rec.events or {}) do + local sub, err = open_event_sub(conn, route, opts) + if sub then + event_subs[event_name] = sub + else + emit({ tag = 'source_down', reason = tostring(err or ('subscribe_failed:' .. event_name)) }) + end + end + + if next(fact_watches) == nil and next(event_subs) == nil then + emit({ tag = 'source_down', reason = 'no_observation_topics' }) + return { reason = 'no_observation_topics' } + end + + local stale_after_s = tonumber(opts.stale_after_s or opts.stale_after) + local stale_deadline = (type(stale_after_s) == 'number' and stale_after_s > 0) and (fibers.now() + stale_after_s) or nil + local stale_latched = false + local refreshes_freshness = freshness_predicate(rec, fact_watches, event_subs) + + while true do + local item = fibers.perform(make_choice(fact_watches, event_subs, stale_deadline)) + if not item then + emit({ tag = 'source_down', reason = 'observer_closed' }) + return { reason = 'observer_closed' } + end + + if item.kind == 'stale' then + if not stale_latched then + stale_latched = true + stale_deadline = nil + emit({ tag = 'source_down', reason = 'stale' }) + end + elseif item.kind == 'fact' then + if not item.ev then + emit({ tag = 'source_down', reason = tostring(item.err or (item.name .. ':closed')) }) + return { reason = 'fact_closed', fact = item.name } + end + + if refreshes_freshness(item) then + stale_latched = false + stale_deadline = (type(stale_after_s) == 'number' and stale_after_s > 0) and (fibers.now() + stale_after_s) or nil + end + + if item.ev.op == 'retain' then + local raw = recv_payload(item.ev) + local payload, nerr = normalise_with_component(rec, 'fact', item.name, raw) + if nerr ~= nil then + emit({ tag = 'source_down', reason = 'bad_fact:' .. item.name .. ':' .. tostring(nerr) }) + else + emit({ tag = 'fact_retained', fact = item.name, payload = payload, raw = model.copy_value(raw) }) + end + elseif item.ev.op == 'unretain' then + emit({ tag = 'fact_unretained', fact = item.name }) + else + local raw = recv_payload(item.ev) + local payload, nerr = normalise_with_component(rec, 'fact', item.name, raw) + if nerr ~= nil then + emit({ tag = 'source_down', reason = 'bad_fact:' .. item.name .. ':' .. tostring(nerr) }) + else + emit({ tag = 'fact_retained', fact = item.name, payload = payload, raw = model.copy_value(raw) }) + end + end + elseif item.kind == 'event' then + if not item.msg then + emit({ tag = 'source_down', reason = tostring(item.err or (item.name .. ':closed')) }) + return { reason = 'event_closed', event = item.name } + end + + if refreshes_freshness(item) then + stale_latched = false + stale_deadline = (type(stale_after_s) == 'number' and stale_after_s > 0) and (fibers.now() + stale_after_s) or nil + end + + local raw = recv_payload(item.msg) + local payload, nerr = normalise_with_component(rec, 'event', item.name, raw) + if nerr ~= nil then + emit({ tag = 'source_down', reason = 'bad_event:' .. item.name .. ':' .. tostring(nerr) }) + else + emit({ + tag = 'event', + event = item.name, + payload = payload, + raw = model.copy_value(item.msg), + }) + end + end + end +end + +return M diff --git a/src/services/device/observer_manager.lua b/src/services/device/observer_manager.lua new file mode 100644 index 00000000..5981cb3e --- /dev/null +++ b/src/services/device/observer_manager.lua @@ -0,0 +1,67 @@ +-- services/device/observer_manager.lua +-- +-- Starts and tracks generation-owned component observer scopes. + +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local service_events = require 'devicecode.support.service_events' +local observer = require 'services.device.observer' + +local M = {} + + +local function completion_reporter(params, identity, label) + local target = params.events_port or params.done_tx + if type(target) == 'table' + and (type(target.emit_required) == 'function' or type(target.send_op) == 'function') + then + return service_events.reporter_for(target, identity, { label = label }) + end + return function (ev) + return queue.try_admit_required(params.done_tx, ev, label) + end +end + +local function observer_identity(generation, component_id) + return { + kind = 'observer_done', + generation = generation, + component = component_id, + } +end + +function M.start_component(active, params, component_id, component) + local handle, err = scoped_work.start { + lifetime_scope = active.observer_root or active.scope, + reaper_scope = active.observer_root or active.scope, + report_scope = params.report_scope or params.service_scope, + identity = observer_identity(active.generation, component_id), + run = function (scope) + return observer.run(scope, { + conn = params.conn, + tx = params.observation_tx, + generation = active.generation, + component_id = component_id, + component = component, + }) + end, + report = completion_reporter(params, observer_identity(active.generation, component_id), 'device_observer_done_report_failed'), + } + + if not handle then return nil, err end + active.observers[component_id] = handle + return handle, nil +end + +function M.start_all(active, params) + active.observers = active.observers or {} + for component_id, component in pairs((active.catalogue and active.catalogue.components) or {}) do + local handle, err = M.start_component(active, params, component_id, component) + if not handle then + return nil, err or ('observer_start_failed:' .. tostring(component_id)) + end + end + return true, nil +end + +return M diff --git a/src/services/device/projection.lua b/src/services/device/projection.lua new file mode 100644 index 00000000..b4488013 --- /dev/null +++ b/src/services/device/projection.lua @@ -0,0 +1,271 @@ +-- services/device/projection.lua +-- +-- Pure projection from Device model state to public payloads. + +local model = require 'services.device.model' +local host = require 'services.device.component_host' +local mcu = require 'services.device.component_mcu' +local topics = require 'services.device.topics' +local availability = require 'services.device.availability' + +local M = {} + +function M.identity_topic() return topics.identity() end +function M.summary_topic() return topics.components() end +function M.assembly_topic() return topics.assembly() end +function M.component_topic(name) return topics.component(name) end +function M.component_software_topic(name) return topics.component_software(name) end +function M.component_update_topic(name) return topics.component_update(name) end +function M.component_cap_topic(name, method) return topics.component_cap_rpc(name, method) end +function M.component_cap_meta_topic(name) return topics.component_cap_meta(name) end +function M.component_cap_status_topic(name) return topics.component_cap_status(name) end +function M.component_cap_event_topic(name, event) return topics.component_cap_event(name, event) end + +local function copy(v) + return model.copy_value(v) +end + +local function public_actions(rec) + local actions = { ['get-status'] = true } + for action_name in pairs(rec.actions or {}) do + actions[action_name] = true + end + return actions +end + +local function compose_component(rec) + local mod = type(rec) == 'table' and rec.module or nil + if type(mod) == 'table' and type(mod.compose) == 'function' then + return mod.compose(rec.raw_facts or {}, rec.raw_events or {}) + end + local subtype = rec and (rec.subtype or rec.member_class or rec.name) or nil + if subtype == 'mcu' or rec.class == 'mcu' then + return mcu.compose(rec.raw_facts or {}, rec.raw_events or {}) + end + return host.compose(rec.raw_facts or {}) +end + + +local function copy_topic(v) + if type(v) ~= 'table' then return nil end + local out = {} + for i = 1, #v do out[i] = v[i] end + return out +end + +local function fact_backing_topic(spec) + if type(spec) == 'table' and spec.watch_topic ~= nil then return spec.watch_topic end + return spec +end + +local function event_backing_topic(spec) + if type(spec) == 'table' and spec.subscribe_topic ~= nil then return spec.subscribe_topic end + return spec +end + +local function collect_backing_refs(rec) + local refs = { facts = {}, events = {}, actions = {} } + for name, spec in pairs(rec.facts or {}) do + refs.facts[name] = copy_topic(fact_backing_topic(spec)) + end + for name, spec in pairs(rec.events or {}) do + refs.events[name] = copy_topic(event_backing_topic(spec)) + end + for name, spec in pairs(rec.actions or {}) do + if type(spec) == 'table' and spec.call_topic then + refs.actions[name] = copy_topic(spec.call_topic) + elseif type(spec) == 'table' and spec.kind == 'fabric_stage' then + refs.actions[name] = { + kind = 'fabric-stage', + target = spec.target, + link_id = spec.link_id, + } + else + refs.actions[name] = copy_topic(spec) + end + end + return refs +end + +local function derive_source(rec) + return { + kind = (rec.class == 'host') and 'host' or 'member', + member = rec.member, + member_class = rec.member_class, + link_class = rec.link_class, + role = rec.role, + reason = rec.source_err, + } +end + +local function derive_health(status, updater_state, explicit_health) + if explicit_health ~= nil then return explicit_health end + if status.health ~= nil and status.health ~= 'unknown' then return status.health end + if not status.available then return 'unknown' end + if updater_state == 'failed' or updater_state == 'unavailable' then return 'degraded' end + return 'ok' +end + +function M.component_view(name, rec, now_ts) + rec = rec or {} + local base = compose_component(rec) + local status = rec.status or availability.component_status(rec) + local updater_state = type(base.updater) == 'table' and base.updater.state or nil + local health = derive_health(status, updater_state, base.health) + + return { + kind = 'device.component', + ts = now_ts, + component = name, + class = rec.class, + subtype = rec.subtype, + role = rec.role, + member = rec.member, + member_class = rec.member_class, + link_class = rec.link_class, + display = copy(rec.display or {}), + present = rec.present ~= false, + availability = status.availability, + available = status.available, + ready = status.ready, + health = health, + reason = status.reason, + actions = public_actions(rec), + software = copy(base.software or {}), + updater = copy(base.updater or {}), + power = copy(base.power or {}), + environment = copy(base.environment or {}), + runtime = copy(base.runtime or {}), + alerts = copy(base.alerts or {}), + observed = copy(base.observed or {}), + source = derive_source(rec), + last_action = copy(rec.last_action), + } +end + +function M.component_payloads(name, rec, now_ts) + local view = M.component_view(name, rec, now_ts) + + local sw = copy(view.software) + sw.kind = 'device.component.software' + sw.ts = now_ts + sw.component = name + sw.role = view.role + sw.member = view.member + sw.member_class = view.member_class + sw.link_class = view.link_class + + local upd = copy(view.updater) + upd.kind = 'device.component.update' + upd.ts = now_ts + upd.component = name + upd.available = view.available + upd.health = view.health + upd.actions = copy(view.actions) + + return { + component = view, + software = sw, + update = upd, + cap_meta = { + owner = 'device', + interface = 'devicecode.cap/component/1', + component = name, + methods = copy(view.actions), + events = { ['state-changed'] = true }, + canonical_state = M.component_topic(name), + backing_source = copy(view.source), + backing = collect_backing_refs(rec), + }, + cap_status = { + state = view.availability or (view.available and 'available' or 'unavailable'), + availability = view.availability, + available = view.available, + health = view.health, + ready = view.ready, + reason = view.reason, + }, + } +end + +function M.summary_payload(snapshot, now_ts) + local items = {} + local counts = { total = 0, available = 0, degraded = 0 } + + for name, rec in pairs((snapshot and snapshot.components) or {}) do + local view = M.component_view(name, rec, now_ts) + counts.total = counts.total + 1 + if view.available then counts.available = counts.available + 1 end + if view.health ~= 'ok' then counts.degraded = counts.degraded + 1 end + items[name] = { + class = view.class, + subtype = view.subtype, + role = view.role, + member = view.member, + member_class = view.member_class, + link_class = view.link_class, + present = view.present, + availability = view.availability, + available = view.available, + ready = view.ready, + health = view.health, + reason = view.reason, + actions = copy(view.actions), + software = copy(view.software), + updater = copy(view.updater), + power = copy(view.power), + environment = copy(view.environment), + runtime = copy(view.runtime), + alerts = copy(view.alerts), + observed = copy(view.observed or {}), + } + end + + return { + kind = 'device.components', + ts = now_ts, + generation = snapshot and snapshot.generation or nil, + components = items, + counts = counts, + } +end + + +function M.assembly_payload(snapshot, now_ts) + local cat = snapshot and snapshot.catalogue or {} + local assembly = copy(cat.assembly or {}) + assembly.kind = assembly.kind or 'device.assembly' + assembly.ts = now_ts + assembly.generation = snapshot and snapshot.generation or nil + return assembly +end + +function M.identity_payload(snapshot, now_ts) + local summary = M.summary_payload(snapshot, now_ts) + return { + kind = 'device.identity', + ts = now_ts, + generation = snapshot and snapshot.generation or nil, + counts = summary.counts, + components = summary.components, + } +end + +function M.state_changed_event(name, rec, now_ts) + local payloads = M.component_payloads(name, rec, now_ts) + return { + kind = 'device.component.event.state-changed', + ts = now_ts, + component = name, + availability = payloads.component.availability, + available = payloads.component.available, + ready = payloads.component.ready, + health = payloads.component.health, + reason = payloads.component.reason, + software = copy(payloads.software), + update = copy(payloads.update), + status = copy(payloads.cap_status), + } +end + +return M diff --git a/src/services/device/publisher.lua b/src/services/device/publisher.lua new file mode 100644 index 00000000..2d198f55 --- /dev/null +++ b/src/services/device/publisher.lua @@ -0,0 +1,111 @@ +-- services/device/publisher.lua +-- +-- Immediate local-bus publication mechanics. Policy remains in the coordinator. + +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local projection = require 'services.device.projection' + +local M = {} + +local function now_value(opts) + if opts and type(opts.now) == 'function' then return opts.now() end + local ok, fibers = pcall(require, 'fibers') + if ok and fibers and type(fibers.now) == 'function' then return fibers.now() end + return os.time() +end + +local function checked(ok, err) + if ok ~= true then return nil, err end + return true, nil +end + +function M.retain_now(conn, topic, payload, opts) + return bus_cleanup.retain(conn, topic, payload, opts) +end + +function M.unretain_now(conn, topic, opts) + return bus_cleanup.unretain(conn, topic, opts) +end + +function M.publish_now(conn, topic, payload, opts) + return bus_cleanup.publish(conn, topic, payload, opts) +end + +function M.publish_component_now(conn, snapshot, component_id, opts) + local rec = snapshot and snapshot.components and snapshot.components[component_id] or nil + if not rec then return true, nil end + + local ts = now_value(opts) + local payloads = projection.component_payloads(component_id, rec, ts) + + local ok, err = checked(bus_cleanup.retain(conn, projection.component_topic(component_id), payloads.component)) + if not ok then return nil, err end + ok, err = checked(bus_cleanup.retain(conn, projection.component_software_topic(component_id), payloads.software)) + if not ok then return nil, err end + ok, err = checked(bus_cleanup.retain(conn, projection.component_update_topic(component_id), payloads.update)) + if not ok then return nil, err end + ok, err = checked(bus_cleanup.retain(conn, projection.component_cap_meta_topic(component_id), payloads.cap_meta)) + if not ok then return nil, err end + ok, err = checked(bus_cleanup.retain(conn, projection.component_cap_status_topic(component_id), payloads.cap_status)) + if not ok then return nil, err end + + + if opts == nil or opts.emit_event ~= false then + ok, err = checked(bus_cleanup.publish( + conn, + projection.component_cap_event_topic(component_id, 'state-changed'), + projection.state_changed_event(component_id, rec, ts) + )) + if not ok then return nil, err end + end + + return true, nil +end + +function M.unpublish_component_now(conn, component_id) + local topics = { + projection.component_topic(component_id), + projection.component_software_topic(component_id), + projection.component_update_topic(component_id), + projection.component_cap_meta_topic(component_id), + projection.component_cap_status_topic(component_id), + } + for i = 1, #topics do + local ok, err = bus_cleanup.unretain(conn, topics[i]) + if ok ~= true then return nil, err end + end + return true, nil +end + +function M.publish_summary_now(conn, snapshot, opts) + local ts = now_value(opts) + local ok, err = bus_cleanup.retain(conn, projection.summary_topic(), projection.summary_payload(snapshot, ts)) + if ok ~= true then return nil, err end + ok, err = bus_cleanup.retain(conn, projection.identity_topic(), projection.identity_payload(snapshot, ts)) + if ok ~= true then return nil, err end + return bus_cleanup.retain(conn, projection.assembly_topic(), projection.assembly_payload(snapshot, ts)) +end + +function M.publish_dirty_now(conn, snapshot, dirty, opts) + dirty = dirty or {} + for component_id in pairs(dirty.components or {}) do + local ok, err = M.publish_component_now(conn, snapshot, component_id, opts) + if ok ~= true then return nil, err end + end + + if dirty.summary then + local ok, err = M.publish_summary_now(conn, snapshot, opts) + if ok ~= true then return nil, err end + end + return true, nil +end + +function M.bind_now(conn, topic, opts) + return bus_cleanup.bind(conn, topic, opts) +end + +function M.unbind_now(conn, ep) + return bus_cleanup.unbind(conn, ep) +end + +return M diff --git a/src/services/device/schemas/mcu.lua b/src/services/device/schemas/mcu.lua new file mode 100644 index 00000000..23ec3fbc --- /dev/null +++ b/src/services/device/schemas/mcu.lua @@ -0,0 +1,196 @@ +-- services/device/schemas/mcu.lua +-- Pure MCU schema helpers. + +local topics = require 'services.device.topics' +local model = require 'services.device.model' +local bit = rawget(_G, 'bit') or require 'bit32' + +local M = {} + +M.alert_kinds = { + vin_lo = true, + vin_hi = true, + bsr_high = true, + bat_missing = true, + bat_short = true, + max_charge_time_fault = true, + absorb = true, + equalize = true, + cccv = true, + precharge = true, + iin_limited = true, + uvcl_active = true, + cc_phase = true, + cv_phase = true, +} + +function M.member_fact_topics(member) + member = member or 'mcu' + return { + software = topics.raw_member_state(member, 'software'), + updater = topics.raw_member_state(member, 'updater'), + health = topics.raw_member_state(member, 'health'), + power_battery = topics.raw_member_state(member, 'power', 'battery'), + power_charger = topics.raw_member_state(member, 'power', 'charger'), + power_charger_config = topics.raw_member_state(member, 'power', 'charger', 'config'), + environment_temperature = topics.raw_member_state(member, 'environment', 'temperature'), + environment_humidity = topics.raw_member_state(member, 'environment', 'humidity'), + runtime_memory = topics.raw_member_state(member, 'runtime', 'memory'), + } +end + +function M.member_event_topics(member) + member = member or 'mcu' + return { + charger_alert = topics.raw_member_cap_event(member, 'telemetry', 'main', 'power', 'charger', 'alert'), + } +end + +local function table_or_empty(v) + return type(v) == 'table' and v or {} +end + +local function copy(v) + return model.copy_value(v) +end + +local function copy_named(raw, names) + local out = {} + raw = table_or_empty(raw) + for i = 1, #names do + local name = names[i] + if raw[name] ~= nil then out[name] = raw[name] end + end + return out +end + +local charger_state_bits = { + { mask = 0x0400, name = 'equalize_charge' }, + { mask = 0x0200, name = 'absorb_charge' }, + { mask = 0x0100, name = 'charger_suspended' }, + { mask = 0x0080, name = 'precharge' }, + { mask = 0x0040, name = 'cccv_charge' }, + { mask = 0x0020, name = 'ntc_pause' }, + { mask = 0x0010, name = 'timer_term' }, + { mask = 0x0008, name = 'c_over_x_term' }, + { mask = 0x0004, name = 'max_charge_time_fault' }, + { mask = 0x0002, name = 'bat_missing_fault' }, + { mask = 0x0001, name = 'bat_short_fault' }, +} + +local charger_status_bits = { + { mask = 0x0008, name = 'vin_uvcl_active' }, + { mask = 0x0004, name = 'iin_limit_active' }, + { mask = 0x0002, name = 'const_current' }, + { mask = 0x0001, name = 'const_voltage' }, +} + +local charger_system_bits = { + { mask = 0x2000, name = 'charger_enabled' }, + { mask = 0x0800, name = 'mppt_en_pin' }, + { mask = 0x0400, name = 'equalize_req' }, + { mask = 0x0200, name = 'drvcc_good' }, + { mask = 0x0100, name = 'cell_count_error' }, + { mask = 0x0040, name = 'ok_to_charge' }, + { mask = 0x0020, name = 'no_rt' }, + { mask = 0x0010, name = 'thermal_shutdown' }, + { mask = 0x0008, name = 'vin_ovlo' }, + { mask = 0x0004, name = 'vin_gt_vbat' }, + { mask = 0x0002, name = 'intvcc_gt_4p3v' }, + { mask = 0x0001, name = 'intvcc_gt_2p8v' }, +} + +local function decode_bits(value, spec) + local out = {} + value = tonumber(value) or 0 + for i = 1, #spec do + local e = spec[i] + out[e.name] = bit.band(value, e.mask) ~= 0 + end + return out +end + +function M.normalise_software(raw) + raw = table_or_empty(raw) + return { + version = raw.version, + build_id = raw.build_id, + image_id = raw.image_id, + boot_id = raw.boot_id, + payload_sha256 = raw.payload_sha256, + } +end + +function M.normalise_updater(raw) + raw = table_or_empty(raw) + return { + state = raw.state, + last_error = raw.last_error, + pending_version = raw.pending_version, + pending_image_id = raw.pending_image_id, + staged_image_id = raw.staged_image_id, + job_id = raw.job_id, + } +end + +function M.normalise_health(raw) + if type(raw) == 'string' then return raw end + raw = table_or_empty(raw) + if raw.state ~= nil then return raw.state end + if next(raw) ~= nil then return 'ok' end + return nil +end + + +function M.normalise_charger(raw) + raw = table_or_empty(raw) + local out = copy_named(raw, { + 'vin_mV', 'vsys_mV', 'iin_mA', + 'state_bits', 'status_bits', 'system_bits', + 'seq', 'uptime_ms', + }) + out.state = decode_bits(raw.state_bits, charger_state_bits) + out.status = decode_bits(raw.status_bits, charger_status_bits) + out.system = decode_bits(raw.system_bits, charger_system_bits) + return out +end + +function M.normalise_charger_alert(raw) + raw = table_or_empty(raw) + local kind = raw.kind + return { + kind = type(kind) == 'string' and kind or nil, + known = type(kind) == 'string' and M.alert_kinds[kind] == true or false, + severity = raw.severity, + source = raw.source, + seq = raw.seq, + uptime_ms = raw.uptime_ms, + } +end + +function M.compose(raw_facts, raw_events) + raw_facts = table_or_empty(raw_facts) + raw_events = table_or_empty(raw_events) + return { + software = M.normalise_software(raw_facts.software), + updater = M.normalise_updater(raw_facts.updater), + health = M.normalise_health(raw_facts.health), + power = { + battery = copy_named(raw_facts.power_battery, { 'pack_mV', 'per_cell_mV', 'ibat_mA', 'temp_mC', 'seq', 'uptime_ms' }), + charger = M.normalise_charger(raw_facts.power_charger), + charger_config = copy(raw_facts.power_charger_config or {}), + }, + environment = { + temperature = copy(raw_facts.environment_temperature or {}), + humidity = copy(raw_facts.environment_humidity or {}), + }, + runtime = { + memory = copy(raw_facts.runtime_memory or {}), + }, + alerts = { + charger = raw_events.charger_alert and M.normalise_charger_alert(raw_events.charger_alert) or nil, + }, + } +end + +return M diff --git a/src/services/device/service.lua b/src/services/device/service.lua new file mode 100644 index 00000000..cc8454de --- /dev/null +++ b/src/services/device/service.lua @@ -0,0 +1,1406 @@ +-- services/device/service.lua +-- +-- Device service coordinator spine. +-- The coordinator has one suspending control point and non-suspending branches. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local pulse = require 'fibers.pulse' +local runtime = require 'fibers.runtime' +local config_mod = require 'services.device.config' +local model_mod = require 'services.device.model' +local topics = require 'services.device.topics' +local publisher = require 'services.device.publisher' +local projection = require 'services.device.projection' +local mcu_metrics = require 'services.device.mcu_metrics' +local observer_manager = require 'services.device.observer_manager' +local action_manager = require 'services.device.action_manager' +local priority_event = require 'devicecode.support.priority_event' +local queue = require 'devicecode.support.queue' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local config_watch = require 'devicecode.support.config_watch' +local service_events = require 'devicecode.support.service_events' +local service_base = require 'devicecode.service_base' +local cap_deps_mod = require 'devicecode.support.capability_dependencies' +local dep_failure = require 'devicecode.support.dependency_failure' +local backpressure = require 'services.device.backpressure' +local dependency_mod = require 'services.device.dependencies' +local fabric_topics = require 'services.fabric.topics' +local tablex = require 'shared.table' + +local M = {} + +local DEFAULT_DONE_QUEUE = backpressure.policy.completions.default_len +local DEFAULT_OBSERVATION_QUEUE = backpressure.policy.observations.default_len + +local shallow_copy = tablex.shallow_copy + +local function new_service_id() + return ('device-%d-%d'):format(os.time(), math.random(1, 1000000)) +end + +local function count_map(t) + local n = 0 + for _ in pairs(t or {}) do n = n + 1 end + return n +end + +local function sorted_keys_map(t) + local keys = {} + for k in pairs(t or {}) do keys[#keys + 1] = tostring(k) end + table.sort(keys) + return keys +end + +local function join_sorted_map_keys(t) + local keys = sorted_keys_map(t) + return table.concat(keys, ',') +end + +local function first_non_empty(...) + for i = 1, select('#', ...) do + local v = select(i, ...) + if v ~= nil and v ~= '' and v ~= false then return v end + end + return nil +end + +local function scalar(v) + local tv = type(v) + if tv == 'string' or tv == 'number' or tv == 'boolean' then return v end + return nil +end + +local function scalar_string(v) + v = scalar(v) + if v == nil or v == '' then return nil end + return tostring(v) +end + +local function status_field(rec, field) + if type(rec) ~= 'table' then return nil end + local st = type(rec.status) == 'table' and rec.status or {} + return scalar_string(st[field]) or scalar_string(rec[field]) +end + +local function is_pending_reason(reason, availability) + return availability == 'unknown' + or reason == 'missing_required_fact' + or reason == 'no_observation' + or reason == 'source_not_confirmed' + or reason == 'missing_host_software' +end + +local function version_from_software(sw) + if type(sw) ~= 'table' then return nil end + return scalar_string(first_non_empty(sw.version, sw.fw_version, sw.firmware_version, sw.fw, sw.build, sw.build_id, sw.image_id)) +end + +local function updater_state(up) + if type(up) ~= 'table' then return nil end + return scalar_string(first_non_empty(up.state, up.raw_state, up.status, up.kind)) +end + +local function pending_update(up) + if type(up) ~= 'table' then return nil end + return scalar_string(first_non_empty(up.pending_version, up.staged_image_id, up.job_id)) +end + +local function component_operator_key(rec) + if type(rec) ~= 'table' then return 'missing' end + return table.concat({ + status_field(rec, 'availability') or '', + status_field(rec, 'health') or '', + status_field(rec, 'reason') or '', + }, '|') +end + +local function component_label(component_id, rec) + rec = rec or {} + local parts = { 'component ' .. tostring(component_id) } + local role = scalar_string(rec.role or rec.member or rec.member_class) + local kind = scalar_string(rec.subtype or rec.kind or rec.class) + if role and role ~= tostring(component_id) then parts[#parts + 1] = 'role=' .. role end + if kind and kind ~= tostring(component_id) then parts[#parts + 1] = 'kind=' .. kind end + return table.concat(parts, ' ') +end + +local function component_summary(component_id, rec, opts) + rec = rec or {} + opts = opts or {} + local parts = { component_label(component_id, rec) } + local availability = status_field(rec, 'availability') + local health = status_field(rec, 'health') + local reason = status_field(rec, 'reason') + if availability then parts[#parts + 1] = 'state=' .. availability end + if health and health ~= 'unknown' then parts[#parts + 1] = 'health=' .. health end + if reason then parts[#parts + 1] = 'reason=' .. reason end + local facts = rec.raw_facts or {} + local swv = version_from_software(facts.software) + if swv then parts[#parts + 1] = 'fw=' .. swv end + local ups = updater_state(facts.updater) + if ups then parts[#parts + 1] = 'updater=' .. ups end + local pend = pending_update(facts.updater) + if pend then parts[#parts + 1] = 'pending=' .. pend end + return table.concat(parts, ' ') +end + + +local function fmt_voltage_mv(v) + v = tonumber(v) + if not v then return nil end + return string.format('%.2fV', v / 1000) +end + +local function fmt_voltage_mv_coarse(v) + v = tonumber(v) + if not v then return nil end + return string.format('%.0fV', v / 1000) +end + +local function fmt_current_ma(v) + v = tonumber(v) + if not v then return nil end + if math.abs(v) >= 1000 then return string.format('%.2fA', v / 1000) end + return string.format('%dmA', math.floor(v + 0.5)) +end + +local function fmt_temp_mc(v) + v = tonumber(v) + if not v then return nil end + return string.format('%.1fC', v / 1000) +end + +local function first_number(t, names) + if type(t) ~= 'table' then return nil end + for i = 1, #names do + local v = tonumber(t[names[i]]) + if v ~= nil then return v end + end + return nil +end + +local function first_scalar(t, names) + if type(t) ~= 'table' then return nil end + for i = 1, #names do + local v = scalar_string(t[names[i]]) + if v ~= nil then return v end + end + return nil +end + +local function bool_flag(t, name) + return type(t) == 'table' and t[name] == true +end + +local function summarise_mcu_power_env_memory(rec) + local facts = rec and rec.raw_facts or {} + local parts = {} + local swv = version_from_software(facts.software) + local ups = updater_state(facts.updater) + if swv then parts[#parts + 1] = 'mcu_fw=' .. swv end + if ups then parts[#parts + 1] = 'updater=' .. ups end + + local battery = facts.power_battery or {} + local pack = fmt_voltage_mv(first_number(battery, { 'pack_mV', 'pack_mv', 'voltage_mV', 'vbat_mV' })) + local ibat = fmt_current_ma(first_number(battery, { 'ibat_mA', 'current_mA', 'battery_current_mA' })) + local btemp = fmt_temp_mc(first_number(battery, { 'temp_mC', 'temperature_mC' })) + local bparts = {} + if pack then bparts[#bparts + 1] = pack end + if ibat then bparts[#bparts + 1] = ibat end + if btemp then bparts[#bparts + 1] = 'temp=' .. btemp end + if #bparts > 0 then parts[#parts + 1] = 'battery=' .. table.concat(bparts, ',') end + + local charger = facts.power_charger or {} + local vin = fmt_voltage_mv_coarse(first_number(charger, { 'vin_mV', 'vin_mv' })) + local vsys = fmt_voltage_mv_coarse(first_number(charger, { 'vsys_mV', 'vsys_mv' })) + local cstate = charger.state or {} + local cstatus = charger.status or {} + local csystem = charger.system or {} + local cparts = {} + if vin then cparts[#cparts + 1] = 'vin=' .. vin end + if vsys then cparts[#cparts + 1] = 'vsys=' .. vsys end + if bool_flag(csystem, 'charger_enabled') then cparts[#cparts + 1] = 'enabled' end + if bool_flag(csystem, 'ok_to_charge') then cparts[#cparts + 1] = 'ok_to_charge' end + if bool_flag(cstatus, 'const_current') then cparts[#cparts + 1] = 'cc' end + if bool_flag(cstatus, 'const_voltage') then cparts[#cparts + 1] = 'cv' end + if bool_flag(cstate, 'charger_suspended') then cparts[#cparts + 1] = 'suspended' end + if bool_flag(cstate, 'bat_missing_fault') then cparts[#cparts + 1] = 'bat_missing' end + if #cparts > 0 then parts[#parts + 1] = 'charger=' .. table.concat(cparts, ',') end + + local temp = facts.environment_temperature or {} + local tempv = fmt_temp_mc(first_number(temp, { 'temp_mC', 'temperature_mC', 'ambient_mC', 'mC' })) + if not tempv then + local c = first_number(temp, { 'temperature_C', 'temperature_c', 'celsius', 'value' }) + if c then tempv = string.format('%.1fC', c) end + end + local hum = facts.environment_humidity or {} + local hpct = first_number(hum, { 'relative_percent', 'rh_percent', 'percent', 'value' }) + if not hpct then + local pptt = first_number(hum, { 'rh_pptt', 'relative_pptt' }) + if pptt then hpct = pptt / 100 end + end + local eparts = {} + if tempv then eparts[#eparts + 1] = 'temp=' .. tempv end + if hpct then eparts[#eparts + 1] = string.format('humidity=%.1f%%', hpct) end + if #eparts > 0 then parts[#parts + 1] = 'env=' .. table.concat(eparts, ',') end + + local mem = facts.runtime_memory or {} + local free = first_number(mem, { 'free', 'free_bytes', 'free_B' }) + local used = first_number(mem, { 'used', 'used_bytes', 'used_B' }) + local total = first_number(mem, { 'total', 'total_bytes', 'total_B' }) + local mparts = {} + if free and total and total > 0 then + mparts[#mparts + 1] = string.format('free=%.0f%%', (free / total) * 100) + elseif used and total and total > 0 then + mparts[#mparts + 1] = string.format('used=%.0f%%', (used / total) * 100) + else + local state = first_scalar(mem, { 'state', 'status' }) + if state then mparts[#mparts + 1] = state end + end + if #mparts > 0 then parts[#parts + 1] = 'memory=' .. table.concat(mparts, ',') end + + return parts +end + +local function log_device_summary(state, reason) + if not state.svc or not state.model then return end + local snap = state.model:snapshot() + local components = snap.components or {} + local mcu = components.mcu + local mcu_parts = summarise_mcu_power_env_memory(mcu) + -- Component inventory and readiness already have their own operator lines. The + -- device summary should only appear once it adds composed product context, + -- particularly MCU firmware/power/environment/runtime facts. + if #mcu_parts < 3 then return end + local parts = {} + parts[#parts + 1] = 'components=' .. tostring(count_map(components)) + local ids = join_sorted_map_keys(components) + if ids ~= '' then parts[#parts + 1] = 'ids=' .. ids end + for i = 1, #mcu_parts do parts[#parts + 1] = mcu_parts[i] end + local summary = 'device summary ' .. table.concat(parts, ' ') + local key = summary + local tnow = runtime.now() + -- MCU measurements can move every poll. Do not let small voltage/current + -- changes turn the operator log into telemetry; after the first meaningful + -- summary, emit at most every five minutes unless the coarse summary is + -- unchanged for ten minutes. + local last_at = state.operator_device_summary_at or 0 + if state.operator_device_summary_key == key and (tnow - last_at) < 600 then return end + if state.operator_device_summary_key ~= nil and (tnow - last_at) < 300 then return end + state.operator_device_summary_key = key + state.operator_device_summary_at = tnow + state.svc:info('device_summary', { + summary = summary, + reason = reason, + components = count_map(components), + component_ids = ids ~= '' and ids or nil, + }) +end + +local function log_component_identity(state, component_id, rec) + if not state.svc or type(rec) ~= 'table' then return end + local facts = rec.raw_facts or {} + local swv = version_from_software(facts.software) + local ups = updater_state(facts.updater) + local pend = pending_update(facts.updater) + if not swv and not ups and not pend then return end + -- The MCU software and updater facts usually arrive as separate retained + -- updates during boot. Avoid half-stories followed immediately by the full + -- story; wait for firmware before emitting the MCU identity line. + if component_id == 'mcu' and (not swv or not ups) then return end + state.operator_component_identity_keys = state.operator_component_identity_keys or {} + local key = table.concat({ tostring(swv or ''), tostring(ups or ''), tostring(pend or '') }, '|') + if state.operator_component_identity_keys[component_id] == key then return end + state.operator_component_identity_keys[component_id] = key + local parts = { component_label(component_id, rec) } + if swv then parts[#parts + 1] = 'fw=' .. swv end + if ups then parts[#parts + 1] = 'updater=' .. ups end + if pend then parts[#parts + 1] = 'pending=' .. pend end + state.svc:info('device_component_identified', { + summary = table.concat(parts, ' '), + component = component_id, + version = swv, + updater_state = ups, + pending_version = pend, + class = rec.class, + subtype = rec.subtype, + role = rec.role, + member = rec.member, + }) +end + +local function log_component_snapshot(state, component_id, rec, opts) + if not state.svc or type(rec) ~= 'table' then return end + state.operator_component_keys = state.operator_component_keys or {} + state.operator_component_ready_seen = state.operator_component_ready_seen or {} + local availability = status_field(rec, 'availability') or (rec.available and 'available') or 'unknown' + local health = status_field(rec, 'health') or 'unknown' + local reason = status_field(rec, 'reason') + local key = component_operator_key(rec) + local status_changed = state.operator_component_keys[component_id] ~= key + local ready = availability == 'available' or availability == 'ok' + + if status_changed then + state.operator_component_keys[component_id] = key + + -- Partial boot-time observation is expected while a component is still collecting + -- its required facts. It is useful in debug logs, but should not read as an + -- operator-facing degradation. + if is_pending_reason(reason, availability) and not state.operator_component_ready_seen[component_id] then + state.svc:debug('device_component_pending', { + summary = component_summary(component_id, rec, opts), + component = component_id, + availability = availability, + health = health, + reason = reason, + }) + return + end + + local level = ready and 'info' or 'warn' + local what = ready and 'device_component_ready' or 'device_component_degraded' + if ready then state.operator_component_ready_seen[component_id] = true end + -- A readiness line is component-level. The source fact that made it true is + -- useful at debug level, but it is not part of the operator story. + local facts = rec.raw_facts or {} + local identity_key = table.concat({ tostring(version_from_software(facts.software) or ''), tostring(updater_state(facts.updater) or ''), tostring(pending_update(facts.updater) or '') }, '|') + if ready and identity_key ~= '||' then + state.operator_component_identity_keys = state.operator_component_identity_keys or {} + state.operator_component_identity_keys[component_id] = identity_key + end + state.svc:log(level, what, { + summary = component_summary(component_id, rec, {}), + component = component_id, + availability = availability, + available = rec.available, + health = health, + reason = reason, + class = rec.class, + subtype = rec.subtype, + role = rec.role, + member = rec.member, + }) + return + end + + -- Status is stable. Only log identity/version changes, not another readiness line. + log_component_identity(state, component_id, rec) +end + +local function log_configured_inventory(state, generation, catalogue) + if not state.svc then return end + local components = catalogue and catalogue.components or {} + local ids = join_sorted_map_keys(components) + state.svc:info('device_inventory_configured', { + summary = string.format('device inventory configured generation=%s components=%d%s', tostring(generation), count_map(components), ids ~= '' and (' ids=' .. ids) or ''), + generation = generation, + components = count_map(components), + component_ids = ids ~= '' and ids or nil, + }) +end + +-- Device action timeout is owned by the action worker scope. The Fabric +-- transfer-manager request may legitimately stay open for that whole action, so +-- do not add lua-bus' default one-second call timeout here. The transfer budget +-- is passed as payload policy, while caller abandonment still aborts this Op. +local FABRIC_SEND_BLOB_CALL_OPTS = { timeout = false } + +local function default_fabric_client(conn) + if type(conn) ~= 'table' or type(conn.call_op) ~= 'function' then return nil end + + return { + send_blob_op = function (_, params, opts) + params = params or {} + opts = opts or {} + local timeout_s = opts.timeout or params.timeout + local ev, err = conn:call_op(fabric_topics.transfer_manager_rpc('send-blob'), { + link_id = params.link_id, + request_id = params.request_id or params.job_id, + xfer_id = params.xfer_id, + target = params.target, + source_owner = params.source_owner, + size = params.size, + digest_alg = params.digest_alg, + digest = params.digest, + chunk_size = params.chunk_size, + meta = params.meta, + timeout_s = timeout_s, + }, FABRIC_SEND_BLOB_CALL_OPTS) + if not ev then return nil, err end + return ev:wrap(function (reply, call_err) + if reply == nil then return nil, call_err end + if type(reply) == 'table' and reply.ok == false then + return nil, reply.err or reply.error or reply.reason or call_err or 'fabric_transfer_failed' + end + return (type(reply) == 'table' and (reply.result or reply.transfer)) or reply, nil + end) + end, + } +end + +local function request_publication(state) + if state.auto_publish == false then return end + if state.publication_requested then return end + state.publication_requested = true + if state.publication_pulse then state.publication_pulse:signal() end +end + +local function mark_component_dirty(state, component_id) + state.dirty.components[component_id] = true + state.dirty.summary = true + request_publication(state) +end + +local function mark_all_dirty(state) + local snap = state.model:snapshot() + for component_id in pairs(snap.components or {}) do + state.dirty.components[component_id] = true + end + state.dirty.summary = true + request_publication(state) +end + +local function unpublish_removed_components(state, snapshot) + local removed = {} + for component_id in pairs(state.published_components) do + if not (snapshot.components and snapshot.components[component_id]) then + local ok, err = publisher.unpublish_component_now(state.conn, component_id) + if ok ~= true then + return nil, err or ('component unpublish failed: ' .. tostring(component_id)) + end + removed[#removed + 1] = component_id + end + end + for i = 1, #removed do + state.published_components[removed[i]] = nil + end + return true, nil +end + +local function flush_publication(state) + if state.auto_publish == false then + state.publication_requested = false + return true, nil + end + if not state.conn then + state.publication_requested = false + return true, nil + end + + local snapshot = state.model:snapshot() + local ok_removed, removed_err = unpublish_removed_components(state, snapshot) + if ok_removed ~= true then return nil, removed_err end + + local ok, err = publisher.publish_dirty_now(state.conn, snapshot, state.dirty, { + now = state.now, + emit_event = state.emit_events ~= false, + }) + if ok ~= true then return nil, err end + + if state.dirty.components.mcu then + local metrics_ok, metrics_err = mcu_metrics.publish_component( + state.svc, + snapshot.components and snapshot.components.mcu + ) + if metrics_ok ~= true then return nil, metrics_err end + end + + for component_id in pairs(state.dirty.components) do + state.published_components[component_id] = true + state.dirty.components[component_id] = nil + end + if state.dirty.summary then + state.published_summary = true + state.published_identity = true + state.published_assembly = true + end + state.dirty.summary = false + state.publication_requested = false + return true, nil +end + +local function cleanup_publication_now(state) + if not state.conn then return true, nil end + local cleaned = {} + for component_id in pairs(state.published_components or {}) do + local ok, err = publisher.unpublish_component_now(state.conn, component_id) + if ok ~= true then + return nil, err or ('component publication cleanup failed: ' .. tostring(component_id)) + end + cleaned[#cleaned + 1] = component_id + end + + if state.published_summary then + local ok, err = bus_cleanup.unretain(state.conn, projection.summary_topic()) + if ok ~= true then return nil, err or 'summary publication cleanup failed' end + end + if state.published_identity then + local ok, err = bus_cleanup.unretain(state.conn, projection.identity_topic()) + if ok ~= true then return nil, err or 'identity publication cleanup failed' end + end + if state.published_assembly then + local ok, err = bus_cleanup.unretain(state.conn, projection.assembly_topic()) + if ok ~= true then return nil, err or 'assembly publication cleanup failed' end + end + + for i = 1, #cleaned do + state.published_components[cleaned[i]] = nil + end + state.published_summary = false + state.published_identity = false + state.published_assembly = false + return true, nil +end + +local function mark_generation_actions_cancelled(state, generation, reason) + for _, rec in pairs(state.pending_actions or {}) do + if rec and rec.generation == generation then + rec.cancelled = true + rec.cancel_reason = reason or 'generation_cancelled' + end + end +end + +local function cancel_generation_actions(state, generation, reason) + -- Cancelling the action handle cancels the action scope. Request resolution is + -- owned by the action-scope finaliser, not by the coordinator. + for _, rec in pairs(state.pending_actions or {}) do + if rec and rec.generation == generation and rec.handle and rec.handle.cancel then + rec.cancelled = true + rec.cancel_reason = reason or 'generation_cancelled' + local ok, err = rec.handle:cancel(reason or 'generation_cancelled') + if ok ~= true then + return nil, err or 'action_cancel_failed' + end + end + end + return true, nil +end + +local terminate_action_deps + +local function cancel_active_generation(state, reason) + local active = state.active + if not active then return true, nil end + + -- Make the generation non-current before cancellation effects can report + -- completions. Any later events from this lifetime are stale for model + -- mutation, but still accounted for in the generation/action outcome ledgers. + state.active = nil + active.cancelled = true + active.cancel_reason = reason or 'generation_replaced' + state.generation_history = state.generation_history or {} + state.generation_history[active.generation] = active + + -- Public admission is a coordinator-visible resource, not merely a child + -- scope finaliser concern. Release generation-owned endpoints immediately so + -- replacement generations can bind the same public routes without waiting for + -- the old generation to be joined by its parent. + local ok_unbind, unbind_err = action_manager.unbind_generation(active, state.conn) + if ok_unbind ~= true then + return nil, unbind_err or 'generation_unbind_failed' + end + + terminate_action_deps(active, reason or 'generation_replaced') + state.action_deps = nil + local ok_actions, action_err = cancel_generation_actions(state, active.generation, reason or 'generation_replaced') + if ok_actions ~= true then + return nil, action_err or 'generation_action_cancel_failed' + end + + mark_generation_actions_cancelled(state, active.generation, reason or 'generation_replaced') + + if active.cancel then + local ok_cancel, cancel_err = active.cancel(reason or 'generation_replaced') + if ok_cancel ~= true then + return nil, cancel_err or 'generation_cancel_failed' + end + elseif active.scope then + active.scope:cancel(reason or 'generation_replaced') + end + + return true, nil +end + + +function terminate_action_deps(active, reason) + if active and active.action_deps and type(active.action_deps.terminate) == 'function' then + active.action_deps:terminate(reason or 'device_action_dependencies_closed') + end + if active then active.action_deps = nil end + return true +end + +local function open_action_deps(state, active, catalogue) + local specs, map, err = dependency_mod.catalogue_dependencies(catalogue) + if err ~= nil then return nil, err end + active.action_dependency_keys = map or {} + if #specs == 0 then return true, nil end + local deps, derr = cap_deps_mod.open(state.conn, specs, { + changed_kind = 'device_dependency_changed', + closed_kind = 'device_dependency_closed', + queue_len = state.dependency_queue_len or 8, + full = 'drop_oldest', + }) + if not deps then return nil, derr or 'device_action_dependencies_open_failed' end + active.action_deps = deps + state.action_deps = deps + return true, nil +end + +local function update_dependency_model(state, active) + if not active then return true, nil end + local changed, _, err = state.model:update_dependencies(active.generation, active.action_deps and active.action_deps:snapshot() or {}) + if err ~= nil then return nil, err end + if changed then state.dirty.summary = true; request_publication(state) end + return true, nil +end + +local function create_generation_lifetime(state, generation, catalogue) + local gen_scope, gen_err = state.scope:child() + if not gen_scope then + return nil, gen_err or 'generation_scope_create_failed' + end + + local observer_root, observer_root_err = gen_scope:child() + if not observer_root then + gen_scope:cancel('observer_root_create_failed') + return nil, observer_root_err or 'observer_root_create_failed' + end + + local action_root, action_root_err = gen_scope:child() + if not action_root then + gen_scope:cancel('action_root_create_failed') + return nil, action_root_err or 'action_root_create_failed' + end + + local active = { + generation = generation, + scope = gen_scope, + observer_root = observer_root, + action_root = action_root, + catalogue = catalogue, + observers = {}, + observer_outcomes = {}, + action_eps = {}, + } + + gen_scope:finally(function () + local ok, err = action_manager.unbind_generation(active, state.conn) + if ok ~= true then error(err or 'generation endpoint cleanup failed', 0) end + end) + + function active.cancel(reason) + gen_scope:cancel(reason or 'generation_cancelled') + return true, nil + end + + return active, nil +end + +local function rollback_generation_start(state, active, reason) + if not active then return true, nil end + local ok_unbind, unbind_err = action_manager.unbind_generation(active, state.conn) + terminate_action_deps(active, reason or 'generation_start_failed') + active.cancel(reason or 'generation_start_failed') + if ok_unbind ~= true then return nil, unbind_err or 'generation_rollback_unbind_failed' end + return true, nil +end + +local function start_generation(state, catalogue) + local ok_cancel, cancel_err = cancel_active_generation(state, 'catalogue_changed') + if ok_cancel ~= true then + return nil, cancel_err or 'generation_cleanup_failed' + end + + state.generation_seq = state.generation_seq + 1 + local generation = state.generation_seq + + local active, err = create_generation_lifetime(state, generation, catalogue) + if not active then + return nil, err or 'generation_start_failed' + end + + local ok_deps, dep_err = open_action_deps(state, active, catalogue) + if ok_deps ~= true then + local ok_rb, rb_err = rollback_generation_start(state, active, 'action_dependency_open_failed') + return nil, dep_err or rb_err or 'action_dependency_open_failed' + end + + if state.enable_actions ~= false then + local ok_bind, bind_err = action_manager.bind_generation(active, { + conn = state.conn, + action_queue_len = state.action_queue_len, + }) + if ok_bind ~= true then + local ok_rb, rb_err = rollback_generation_start(state, active, 'action_bind_failed') + if ok_rb ~= true then return nil, tostring(bind_err or 'action_bind_failed') .. '; rollback failed: ' .. tostring(rb_err) end + return nil, bind_err or 'action_bind_failed' + end + end + + if state.enable_observers ~= false then + local ok_obs, obs_err = observer_manager.start_all(active, { + conn = state.conn, + service_scope = state.scope, + report_scope = state.scope, + observation_tx = state.observation_tx, + done_tx = state.done_tx, + events_port = state.events_port, + }) + if ok_obs ~= true then + local ok_rb, rb_err = rollback_generation_start(state, active, 'observer_start_failed') + if ok_rb ~= true then return nil, tostring(obs_err or 'observer_start_failed') .. '; rollback failed: ' .. tostring(rb_err) end + return nil, obs_err or 'observer_start_failed' + end + end + + local changed, _, merr = state.model:apply_catalogue(generation, catalogue) + if merr ~= nil then + local ok_rb, rb_err = rollback_generation_start(state, active, 'model_catalogue_failed') + if ok_rb ~= true then return nil, tostring(merr) .. '; rollback failed: ' .. tostring(rb_err) end + return nil, merr + end + if changed then mark_all_dirty(state) end + + state.generation_history = state.generation_history or {} + state.generation_history[generation] = active + state.active = active + if state.svc then state.svc:info('device_generation_started', { summary = string.format('device generation %s started (%s component(s))', tostring(generation), tostring(count_map(catalogue and catalogue.components or {}))), generation = generation, components = count_map(catalogue and catalogue.components or {}) }) end + log_configured_inventory(state, generation, catalogue) + return active, nil +end + +local function apply_config_payload(state, payload) + local catalogue, err = config_mod.to_catalogue(payload) + if not catalogue then + return nil, err or 'device_catalogue_failed' + end + + if state.active then + local cat_mod = require 'services.device.catalogue' + if cat_mod.equal(state.active.catalogue, catalogue) then + return true, nil + end + if cat_mod.materially_equal(state.active.catalogue, catalogue) then + local changed_components = cat_mod.public_component_ids_changed(state.active.catalogue, catalogue) + state.active.catalogue = catalogue + local changed, _, merr = state.model:update_catalogue_metadata(state.active.generation, catalogue) + if merr ~= nil then return nil, merr end + if changed then + for i = 1, #changed_components do + mark_component_dirty(state, changed_components[i]) + end + state.dirty.summary = true + end + return true, nil + end + end + + local active, serr = start_generation(state, catalogue) + if not active then return nil, serr end + return true, nil +end + +local function map_config_event(ev, err) + if ev == nil then + return { kind = 'config_closed', err = err or 'closed' } + end + + -- devicecode.support.config_watch normalises retained cfg/ records to + -- config_changed/config_closed events and extracts the public data payload from + -- { rev = n, data = ... } records. Device keeps support for the older retained + -- watch and injected mailbox shapes so existing unit tests and harnesses can + -- still drive the coordinator directly. + if type(ev) == 'table' and ev.kind == 'config_closed' then + return { kind = 'config_closed', err = ev.err or err or 'closed' } + end + if type(ev) == 'table' and ev.kind == 'config_changed' then + if ev.raw ~= nil or ev.record ~= nil or ev.msg ~= nil or ev.rev ~= nil or ev.generation ~= nil then + return { + kind = 'config_changed', + generation = ev.generation, + rev = ev.rev, + payload = ev.raw, + raw = ev.raw, + record = ev.record, + msg = ev.msg, + } + end + return ev + end + + if type(ev) == 'table' and ev.op == 'replay_done' then + return { kind = 'noop' } + end + if type(ev) == 'table' and ev.op == 'retain' then + return { kind = 'config_changed', payload = ev.payload } + end + if type(ev) == 'table' and ev.op == 'unretain' then + return { kind = 'config_changed', payload = nil } + end + + return { kind = 'config_changed', payload = ev } +end + +local function map_queue_event(kind) + return function (ev) + if ev == nil then return { kind = kind .. '_closed' } end + return ev + end +end + +local function try_op_event_now(ev) + local item, err = queue.try_now(ev, nil, 'not_ready') + if item ~= nil then return item end + if err ~= 'not_ready' then return item end + return nil +end + +local function config_event_op(state) + if state.config_feed ~= nil then + return state.config_feed:recv_op():wrap(map_config_event) + elseif state.config_rx ~= nil then + return state.config_rx:recv_op():wrap(function (ev) + return map_config_event(ev) + end) + end + return nil +end + +local function observation_event_op(state) + return state.observation_rx:recv_op():wrap(map_queue_event('observation')) +end + +local function done_event_op(state) + return state.done_rx:recv_op():wrap(map_queue_event('done')) +end + +local function publication_event_op(state) + return fibers.guard(function () + if state.publication_requested then + return fibers.always({ kind = 'publication_flush' }) + end + + return state.publication_pulse:next_op():wrap(function (_, reason) + if reason ~= nil then return { kind = 'publication_closed', err = reason } end + return { kind = 'publication_flush' } + end) + end) +end + +local function action_event_op(rec) + return rec.ep:recv_op():wrap(function (req, err) + if req == nil then + return { + kind = 'action_endpoint_closed', + generation = rec.generation, + component = rec.component, + action = rec.action, + err = err or 'closed', + } + end + return { + kind = 'component_action_request', + generation = rec.generation, + component = rec.component, + action = rec.action, + request = req, + } + end) +end + +local function add_source(sources, name, make_op, try_now) + sources[#sources + 1] = { + name = name, + try_now = try_now, + recv_op = make_op, + } +end + +local function add_config_source(state, sources) + if not config_event_op(state) then return end + add_source(sources, 'config', function () return config_event_op(state) end, function () + return priority_event.take_pending(state.pending_events, 'config') + or try_op_event_now(config_event_op(state)) + end) +end + +local function add_done_source(state, sources) + add_source(sources, 'done', function () return done_event_op(state) end, function () + return priority_event.take_pending(state.pending_events, 'done') + or try_op_event_now(done_event_op(state)) + end) +end + +local function add_observation_source(state, sources) + add_source(sources, 'observation', function () return observation_event_op(state) end, function () + return priority_event.take_pending(state.pending_events, 'observation') + or try_op_event_now(observation_event_op(state)) + end) +end + +local function add_action_sources(state, sources, action_sources) + for _, rec in ipairs(action_sources or {}) do + local rec0 = rec + local name = 'action:' .. rec0.key + add_source(sources, name, function () return action_event_op(rec0) end, function () + return priority_event.take_pending(state.pending_events, name) + or try_op_event_now(action_event_op(rec0)) + end) + end +end + +local function add_publication_source(state, sources) + if state.auto_publish == false then return end + add_source(sources, 'publication', function () return publication_event_op(state) end, function () + return priority_event.take_pending(state.pending_events, 'publication') + or (state.publication_requested and { kind = 'publication_flush' } or nil) + end) +end + + +local function add_dependency_source(state, sources) + if not state.action_deps then return end + local src = state.action_deps:event_source({ name = 'dependencies' }) + if src.enabled ~= nil and src.enabled() ~= true then return end + sources[#sources + 1] = src +end + +local function take_pending_from_sources(state, sources) + for _, source in ipairs(sources) do + local ev = priority_event.take_pending(state.pending_events, source.name) + if ev ~= nil then return ev end + end + return nil +end + +local function prune_unavailable_pending_events(state, action_sources) + local keep = { done = true, observation = true } + if config_event_op(state) ~= nil then keep.config = true end + if state.auto_publish ~= false then keep.publication = true end + if state.action_deps ~= nil then keep.dependencies = true end + + for _, rec in ipairs(action_sources or {}) do + keep['action:' .. rec.key] = true + end + + for name in pairs(state.pending_events or {}) do + if not keep[name] then + state.pending_events[name] = nil + end + end +end + +local function unordered_event_op(state, sources) + return fibers.guard(function () + local pending = take_pending_from_sources(state, sources) + if pending ~= nil then + return fibers.always(pending) + end + + local arms = {} + for _, source in ipairs(sources) do + arms[source.name] = source.recv_op() + end + + return fibers.named_choice(arms):wrap(function (_, ev) + return ev + end) + end) +end + +local function next_event_op(state) + state.pending_events = state.pending_events or {} + + local action_sources = state.active and action_manager.endpoint_sources(state.active) or {} + prune_unavailable_pending_events(state, action_sources) + + local has_actions = #action_sources > 0 + local has_config = config_event_op(state) ~= nil + + -- Default path: Device reducers are stale-safe and idempotent, so ordinary + -- readiness selection is sufficient. Pending events can exist only after a + -- previous priority wait selected a higher-priority wake; drain them before + -- blocking so no stored event is lost when the next pass no longer needs + -- priority. + if not (has_actions and has_config) then + local sources = {} + add_config_source(state, sources) + add_done_source(state, sources) + add_observation_source(state, sources) + add_action_sources(state, sources, action_sources) + add_dependency_source(state, sources) + add_publication_source(state, sources) + return unordered_event_op(state, sources) + end + + -- Admission-sensitive path: a ready configuration change can invalidate the + -- current generation and its action endpoints. Select configuration before + -- admitting an action request. Other events remain ordinary/coalesced and are + -- deliberately after action admission unless a future policy makes their order + -- safety-critical. + local sources = {} + add_config_source(state, sources) + add_action_sources(state, sources, action_sources) + add_dependency_source(state, sources) + add_done_source(state, sources) + add_observation_source(state, sources) + add_publication_source(state, sources) + + return priority_event.sources_op { + label = 'device.next_event.admission_sensitive', + pending = state.pending_events, + sources = sources, + } +end + +local function handle_observation(state, ev) + if not state.active or ev.generation ~= state.active.generation then + return true, nil + end + local changed = state.model:apply_observation(ev.generation, ev) + if changed then + mark_component_dirty(state, ev.component) + log_component_snapshot(state, ev.component, state.model:component_snapshot(ev.component), { source = ev.fact or ev.event or ev.tag }) + if ev.component == 'mcu' then log_device_summary(state, 'mcu_observation') end + end + return true, nil +end + +local function generation_for_event(state, generation) + if state.active and state.active.generation == generation then + return state.active, true + end + local hist = state.generation_history or {} + return hist[generation], false +end + +local function handle_observer_done(state, ev) + local generation_rec, is_current = generation_for_event(state, ev.generation) + + -- Observer completions are outcomes and should be accounted for even when the + -- generation has already been replaced. Stale outcomes do not mutate the + -- service-owned public model, but they remain attached to their generation + -- handle for diagnostics and later inspection. + if generation_rec then + generation_rec.observers = generation_rec.observers or {} + generation_rec.observer_outcomes = generation_rec.observer_outcomes or {} + generation_rec.observers[ev.component] = nil + generation_rec.observer_outcomes[ev.component] = ev + end + + if not is_current then + return true, nil + end + + if ev.status == 'failed' then + state.model:apply_source_down(ev.generation, ev.component, ev.primary or 'observer_failed') + mark_component_dirty(state, ev.component) + end + return true, nil +end + +local function archive_action_outcome(state, ev, rec, is_current) + state.action_outcomes = state.action_outcomes or {} + state.stale_action_outcomes = state.stale_action_outcomes or {} + + local archived = { + event = ev, + record = rec, + current = not not is_current, + } + + state.action_outcomes[ev.request_id] = archived + if not is_current then + state.stale_action_outcomes[ev.request_id] = archived + end +end + +local function handle_action_done(state, ev) + local generation_rec, is_current = generation_for_event(state, ev.generation) + local rec = state.pending_actions[ev.request_id] + + -- Action completions are completion records first and model inputs second. + -- Even stale completions are archived and clear their pending metadata, so + -- generation replacement does not make child outcomes disappear. + archive_action_outcome(state, ev, rec, is_current) + state.pending_actions[ev.request_id] = nil + + if not generation_rec or not is_current then + return true, nil + end + + local result = ev.result or {} + local public_status + local public_ok + local public_error + + if rec and rec.dependency_key and state.action_deps and dep_failure.is_no_route(ev.primary, result, ev.report) then + state.action_deps:classify_call_failure(rec.dependency_key, result, ev.primary) + update_dependency_model(state, generation_rec) + public_status = 'dependency_unavailable' + public_ok = false + public_error = 'dependency_unavailable:' .. tostring(rec.dependency_key) + elseif ev.status == 'ok' then + public_status = result.public_status or (result.ok == true and 'succeeded' or 'remote_failed') + public_ok = result.ok == true or public_status == 'succeeded' + public_error = result.error or result.err + else + public_status = ev.status + public_ok = false + public_error = ev.primary + end + + local changed = state.model:apply_action_result(ev.generation, { + component = ev.component, + action = ev.action, + request_id = ev.request_id, + scope_status = ev.status, + public_status = public_status, + ok = public_ok, + err = public_error, + result = result, + primary = ev.primary, + }) + if changed then mark_component_dirty(state, ev.component) end + return true, nil +end + +local function reduce_event(state, ev) + if ev == nil or ev.kind == 'noop' then + return true, nil + end + + if ev.kind == 'config_closed' then + return nil, 'device config feed closed: ' .. tostring(ev.err or 'closed') + elseif ev.kind == 'config_changed' then + return apply_config_payload(state, ev.payload) + elseif ev.kind == 'component_observation' then + return handle_observation(state, ev) + elseif ev.kind == 'observer_done' then + return handle_observer_done(state, ev) + elseif ev.kind == 'component_action_request' then + if not state.active or ev.generation ~= state.active.generation then + if ev.request and type(ev.request.fail) == 'function' then ev.request:fail('stale_generation') end + return true, nil + end + return action_manager.start_action(state, ev.request, ev) + elseif ev.kind == 'component_action_done' then + return handle_action_done(state, ev) + elseif ev.kind == 'device_dependency_changed' or ev.kind == 'device_dependency_closed' then + if state.active then return update_dependency_model(state, state.active) end + return true, nil + elseif ev.kind == 'publication_flush' then + return flush_publication(state) + elseif ev.kind == 'publication_closed' then + return true, nil + elseif ev.kind == 'observation_closed' or ev.kind == 'done_closed' then + return nil, ev.kind + elseif ev.kind == 'action_endpoint_closed' then + return true, nil + end + + return nil, 'unknown device event kind: ' .. tostring(ev.kind) +end + +local function coordinator_loop(state) + while true do + local ev = fibers.perform(next_event_op(state)) + local ok, err = reduce_event(state, ev) + if ok ~= true then + error(err or 'device coordinator failed', 0) + end + + end +end + +local function build_state(scope, params) + params = params or {} + local service_id = params.service_id or new_service_id() + local done_tx, done_rx = mailbox.new(params.done_queue_len or DEFAULT_DONE_QUEUE, { full = backpressure.policy.completions.full }) + local obs_tx, obs_rx = mailbox.new(params.observation_queue_len or DEFAULT_OBSERVATION_QUEUE, { full = backpressure.policy.observations.full }) + + local state = { + scope = scope, + conn = params.conn, + service_id = service_id, + generation_seq = 0, + active = nil, + model = params.model or model_mod.new(), + done_tx = done_tx, + done_rx = done_rx, + events_port = service_events.port(done_tx, { + service_id = service_id, + source = 'device_service', + source_id = service_id, + }, { label = 'device_service_event_report_failed' }), + observation_tx = obs_tx, + observation_rx = obs_rx, + config_feed = params.config_watch or params.config_feed, + config_rx = params.config_rx, + owns_config_feed = not not params.owns_config_feed, + close_config_feed = params.close_config_feed, + dirty = { components = {}, summary = false }, + publication_pulse = pulse.new(), + publication_requested = false, + published_components = {}, + published_summary = false, + published_identity = false, + published_assembly = false, + pending_actions = {}, + action_outcomes = {}, + stale_action_outcomes = {}, + generation_history = {}, + pending_events = {}, + svc = params.svc, + action_timeout = params.action_timeout or 10.0, + action_queue_len = params.action_queue_len or backpressure.policy.action_endpoints.default_len, + enable_actions = params.enable_actions, + enable_observers = params.enable_observers, + auto_publish = params.auto_publish, + emit_events = params.emit_events, + fabric_client = params.fabric_client or default_fabric_client(params.conn), + open_source = params.open_source, + open_source_op = params.open_source_op, + terminate_source = params.terminate_source, + action_deps = nil, + dependency_queue_len = params.dependency_queue_len, + update_dependency_model = update_dependency_model, + operator_component_keys = {}, + } + + state.now = params.now or function () return fibers.now() end + + scope:finally(function (_, status, primary) + local reason = primary or status or 'device_closed' + + -- Cancel owned generation work before closing local reporting queues. This + -- gives authorised reapers/reporters the best chance to store and submit + -- completion events while the service-owned queues are still open. + local ok_cancel, cancel_err = cancel_active_generation(state, reason) + if ok_cancel ~= true then error(cancel_err or 'generation cleanup failed', 0) end + + done_tx:close(reason) + obs_tx:close(reason) + local ok_pub, pub_err = cleanup_publication_now(state) + if ok_pub ~= true then error(pub_err or 'publication cleanup failed', 0) end + if state.publication_pulse then state.publication_pulse:close(reason) end + state.model:terminate(reason) + if state.owns_config_feed and state.config_feed then + if type(state.close_config_feed) == 'function' then + state.close_config_feed(state.config_feed) + elseif type(state.config_feed.close) == 'function' then + state.config_feed:close() + else + bus_cleanup.unwatch_retained(state.conn, state.config_feed) + end + end + end) + + return state +end + +function M.run(scope, params) + if type(scope) ~= 'table' then error('device.service.run: scope required', 2) end + local state = build_state(scope, params or {}) + + if params and params.initial_config ~= nil then + local ok, err = apply_config_payload(state, params.initial_config) + if ok ~= true then error(err or 'initial device config failed', 0) end + -- Publication is a semantic event. The initial configuration marks the + -- model dirty; the coordinator selects publication_flush next. + end + + if params and params.lifecycle and type(params.lifecycle.ready) == 'function' then + params.lifecycle:ready({ ready = true }) + end + if state.svc then state.svc:info('device_ready', { summary = 'device service ready', generation = state.active and state.active.generation or nil }) end + + coordinator_loop(state) + return { status = 'stopped' } +end + +function M.start(conn, opts) + opts = opts or {} + if conn == nil then + error('device.service.start: conn required', 2) + end + + if not runtime.current_fiber() then + error('device.service.start must be called inside a fiber', 2) + end + + local scope = fibers.current_scope() + + local svc = service_base.new(conn, { + name = opts.name or 'device', + env = opts.env, + meta = opts.meta, + announce = opts.announce, + }) + svc:starting({ ready = false }) + + -- service_base removes retained lifecycle topics from its own finaliser. This + -- finaliser is registered after service_base.new(), so it runs first and leaves + -- a final status visible during shutdown/failure cleanup. + scope:finally(function (_, status, primary) + if status == 'failed' then + svc:failed(primary or 'device_failed') + else + svc:stopped({ reason = primary or status or 'device_stopped' }) + end + end) + + local cfg_watch = opts.config_watch or opts.config_feed + local owns_cfg_watch = false + if cfg_watch == nil and opts.watch_config ~= false then + local watch, err = config_watch.open(conn, 'device', { + topic = opts.config_topic or topics.config(), + queue_len = opts.config_queue_len or backpressure.policy.observer_feeds.default_len, + full = backpressure.policy.observer_feeds.full, + }) + if not watch then + svc:failed(err or 'config_watch_failed') + error(err or 'config_watch_failed', 2) + end + cfg_watch = watch + owns_cfg_watch = true + end + + svc:running({ ready = false }) + return M.run(scope, { + conn = conn, + config_watch = cfg_watch, + owns_config_feed = owns_cfg_watch, + initial_config = opts.initial_config, + svc = svc, + done_queue_len = opts.done_queue_len, + observation_queue_len = opts.observation_queue_len, + action_queue_len = opts.action_queue_len, + action_timeout = opts.action_timeout, + enable_actions = opts.enable_actions, + enable_observers = opts.enable_observers, + auto_publish = opts.auto_publish, + emit_events = opts.emit_events, + fabric_client = opts.fabric_client, + open_source = opts.open_source, + open_source_op = opts.open_source_op, + terminate_source = opts.terminate_source, + now = opts.now, + lifecycle = svc, + }) +end + +M.next_event_op = next_event_op +M.build_state = build_state +M.apply_config_payload = apply_config_payload +M.reduce_event = reduce_event +M.start_generation = start_generation +M.cancel_active_generation = cancel_active_generation +M.flush_publication = flush_publication +M.cleanup_publication_now = cleanup_publication_now +M.default_fabric_client = default_fabric_client + +return M diff --git a/src/services/device/topics.lua b/src/services/device/topics.lua new file mode 100644 index 00000000..fa2961da --- /dev/null +++ b/src/services/device/topics.lua @@ -0,0 +1,77 @@ +-- services/device/topics.lua +-- +-- Pure topic construction helpers for the Device service. + +local topic = require 'shared.topic' + +local M = {} + +function M.copy(t) + return topic.copy(t) +end + +function M.config() + return { 'cfg', 'device' } +end + +function M.identity() + return { 'state', 'device', 'identity' } +end + +function M.components() + return { 'state', 'device', 'components' } +end + +function M.assembly() + return { 'state', 'device', 'assembly' } +end + +function M.component(name) + return { 'state', 'device', 'component', name } +end + +function M.component_software(name) + return topic.append(M.component(name), 'software') +end + +function M.component_update(name) + return topic.append(M.component(name), 'update') +end + +function M.component_cap_meta(name) + return { 'cap', 'component', name, 'meta' } +end + +function M.component_cap_status(name) + return { 'cap', 'component', name, 'status' } +end + +function M.component_cap_event(name, event) + return { 'cap', 'component', name, 'event', event } +end + +function M.component_cap_rpc(name, method) + return { 'cap', 'component', name, 'rpc', method } +end + +function M.raw_member_state(member, ...) + return topic.append({ 'raw', 'member', member, 'state' }, ...) +end + +function M.raw_member_cap_event(member, class, id, ...) + return topic.append({ 'raw', 'member', member, 'cap', class, id, 'event' }, ...) +end + +function M.raw_member_cap_rpc(member, class, id, method) + return { 'raw', 'member', member, 'cap', class, id, 'rpc', method } +end + +function M.raw_host_cap_state(source, class, id, ...) + return topic.append({ 'raw', 'host', source, 'cap', class, id, 'state' }, ...) +end + +function M.raw_host_cap_rpc(source, class, id, method) + return { 'raw', 'host', source, 'cap', class, id, 'rpc', method } +end + +return M diff --git a/src/services/fabric.lua b/src/services/fabric.lua new file mode 100644 index 00000000..4fc8b6e1 --- /dev/null +++ b/src/services/fabric.lua @@ -0,0 +1,195 @@ +-- services/fabric.lua +-- +-- Public Fabric assembly entry point. +-- +-- This module deliberately stays thin. The semantic owners live under +-- services.fabric.*; this file wires the default service runner to the standard +-- composed link runner. + +local fibers = require 'fibers' +local service = require 'services.fabric.service' +local link = require 'services.fabric.link' +local io = require 'services.fabric.io' +local bridge = require 'services.fabric.bridge' +local bus_adapter = require 'services.fabric.bus_adapter' +local session = require 'services.fabric.session' +local transfer = require 'services.fabric.transfer' +local transfer_client = require 'services.fabric.transfer_client' +local transfer_sender = require 'services.fabric.transfer_sender' +local transfer_receive = require 'services.fabric.transfer_receive' +local hal_transport = require 'services.fabric.hal_transport' +local model = require 'services.fabric.model' +local protocol = require 'services.fabric.protocol' +local topics = require 'services.fabric.topics' +local config = require 'services.fabric.config' +local state = require 'services.fabric.state' +local dep_failure = require 'devicecode.support.dependency_failure' +local tablex = require 'shared.table' + +local M = { + service = service, + link = link, + io = io, + bridge = bridge, + bus_adapter = bus_adapter, + session = session, + transfer = transfer, + transfer_client = transfer_client, + transfer_sender = transfer_sender, + transfer_receive = transfer_receive, + model = model, + protocol = protocol, + topics = topics, + config = config, + state = state, +} + +local shallow_copy = tablex.shallow_copy + +local function has_terminate_contract(x) + return type(x) == 'table' and type(x.terminate) == 'function' +end + +local function is_frame_transport(x) + return type(x) == 'table' + and type(x.read_frame_op) == 'function' + and type(x.write_frame_op) == 'function' + and has_terminate_contract(x) +end + +local function link_conn(link_spec, service_caps) + return link_spec.conn or (service_caps and service_caps.conn) +end + +local function link_dependency_key(link_spec) + if type(link_spec) ~= 'table' then return nil end + if type(link_spec.dependency_key) == 'string' and link_spec.dependency_key ~= '' then + return link_spec.dependency_key + end + local t = link_spec.transport + if type(t) == 'table' and type(t.dependency_key) == 'string' and t.dependency_key ~= '' then + return t.dependency_key + end + return nil +end + +local function normalise_transport_open_error(link_spec, out) + local dep_key = out.dependency_key or link_dependency_key(link_spec) + if out.link_id == nil and type(link_spec) == 'table' then out.link_id = link_spec.link_id end + if dep_key ~= nil then out.dependency_key = dep_key end + local failure = dep_failure.from_no_route(dep_key, out, { + link_id = out.link_id, + source = out.source, + class = out.class, + id = out.id, + }) + if failure ~= nil then return failure end + return out +end +local function transport_open_error(link_spec, err, fallback) + local out + if type(err) == 'table' then + out = normalise_transport_open_error(link_spec, shallow_copy(err)) + else + out = normalise_transport_open_error(link_spec, { + err = err or fallback or 'transport_open_failed', + detail = err, + reason = err or fallback or 'transport_open_failed', + }) + end + if type(out) == 'table' and out.kind == 'dependency_failure' then + return out + end + if type(err) == 'table' then + return out + end + return out.err or out.reason or fallback or 'transport_open_failed' +end + +local function open_transport_for_link(scope, link_spec, service_caps) + if is_frame_transport(link_spec.transport) then + return link_spec.transport, nil + end + + if type(link_spec.open_transport_op) == 'function' then + local transport, err = fibers.perform(link_spec.open_transport_op(scope, service_caps)) + if transport == nil then + return nil, transport_open_error(link_spec, err, 'transport_open_failed') + end + if not is_frame_transport(transport) then + return nil, transport_open_error(link_spec, 'transport_open_returned_non_frame_transport') + end + return transport, nil + end + + if link_spec.transport ~= nil then + local transport, err = hal_transport.open_transport( + link_conn(link_spec, service_caps), + link_spec.transport + ) + if transport == nil then + return nil, transport_open_error(link_spec, err, 'transport_open_failed') + end + return transport, nil + end + + return nil, transport_open_error(link_spec, 'fabric link transport required') +end + +--- Default link runner used by the public Fabric service entry point. +function M.run_link(scope, link_spec, service_caps) + if type(scope) ~= 'table' then + error('fabric.run_link: scope required', 2) + end + if type(link_spec) ~= 'table' then + error('fabric.run_link: link_spec table required', 2) + end + + local p = shallow_copy(link_spec) + local transport, err = open_transport_for_link(scope, p, service_caps) + if transport == nil then + error(err or 'fabric link transport open failed', 0) + end + + p.transport = transport + p.open_transport_op = nil + + return link.run_composed(scope, p, service_caps) +end + +--- Start the long-lived public Fabric service shell. +--- +--- opts is the same shape accepted by services.fabric.service.start, except that +--- link_runner defaults to services.fabric.link.run_composed. +function M.start(conn, opts) + opts = opts or {} + local p = shallow_copy(opts) + p.link_runner = p.link_runner or M.run_link + if p.link_runner == M.run_link then + p._private_link_runtime = true + end + return service.start(conn, p) +end + +--- Run one Fabric generation using the standard composed-link implementation. +--- +--- params is the same shape accepted by services.fabric.service.run, except that +--- link_runner defaults to services.fabric.link.run_composed. +function M.run(scope, params) + if type(scope) ~= 'table' then + error('fabric.run: scope required', 2) + end + if type(params) ~= 'table' then + error('fabric.run: params table required', 2) + end + + local p = shallow_copy(params) + p.link_runner = p.link_runner or M.run_link + if p.link_runner == M.run_link then + p._private_link_runtime = true + end + + return service.run(scope, p) +end + +return M diff --git a/src/services/fabric/bridge.lua b/src/services/fabric/bridge.lua new file mode 100644 index 00000000..ceb06402 --- /dev/null +++ b/src/services/fabric/bridge.lua @@ -0,0 +1,1356 @@ +-- services/fabric/bridge.lua +-- +-- RPC bridge coordinator. +-- +-- The bridge owns: +-- * local/remote RPC and state routing +-- * imported retained state for the current peer session +-- * replay of local retained exports on new peer sessions +-- * scoped outbound calls +-- * scoped inbound local-bus calls +-- * reply routing by call id and session identity +-- * bridge diagnostics and state projection +-- +-- It does not own transport, session establishment, local-bus feed adaptation, +-- device policy, update policy, or transfer policy. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local sleep = require 'fibers.sleep' + +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local priority_event = require 'devicecode.support.priority_event' +local request_owner = require 'devicecode.support.request_owner' +local contracts = require 'devicecode.support.contracts' +local validate = require 'shared.validate' + +local model_mod = require 'services.fabric.model' +local protocol = require 'services.fabric.protocol' +local topics = require 'services.fabric.topics' +local session_mod = require 'services.fabric.session' +local state_mod = require 'services.fabric.state' + +local M = {} + +local Bridge = {} +Bridge.__index = Bridge + +local DEFAULT_DONE_QUEUE = 64 +local DEFAULT_CALL_TIMEOUT = 1.0 +local DEFAULT_MAX_PENDING_CALLS = 64 +local DEFAULT_MAX_INBOUND_CALLS = 64 + +local copy_context = session_mod.copy_context +local same_session = session_mod.same_session +local context_from_event = session_mod.context_from_event + +-------------------------------------------------------------------------------- +-- Small helpers +-------------------------------------------------------------------------------- + +local function count_keys(t) + local n = 0 + for _ in pairs(t or {}) do n = n + 1 end + return n +end + +local function require_params(params) + if type(params) ~= 'table' then + error('fabric.bridge.run: params table required', 3) + end + return params +end + +local function require_rx(v, name, level) + return contracts.require_rx(v, name, (level or 1) + 1) +end + +local function require_outbound(v, name, level) + if type(v) ~= 'table' or type(v.send_rpc_frame_now) ~= 'function' then + error(name .. ' must be a fabric.session outbound gate', (level or 1) + 1) + end + return v +end + +local function nonneg_int(v, name, level) + return validate.non_negative_integer(v, name, (level or 1) + 1) +end + +local function resolve_nonneg_int(v, default, name, level) + if v == nil then return default end + return nonneg_int(v, name, (level or 1) + 1) +end + +local function positive_number(v, default, name, level) + if v == nil then return default end + if type(v) ~= 'number' + or v <= 0 + or v ~= v + or v == math.huge + or v == -math.huge + then + error(name .. ' must be a positive finite number', (level or 1) + 1) + end + return v +end + +local function fail_request(req, reason) + if type(req) == 'table' and type(req.fail) == 'function' then + return req:fail(reason) + end + return false +end + +local function initial_snapshot(link_id, link_generation) + return { + link_id = link_id, + link_generation = link_generation, + state = 'starting', + imported_topics = 0, + pending_calls = 0, + inbound_calls = 0, + frames_sent = 0, + frames_received = 0, + last_err = nil, + session = nil, + session_drop_reason = nil, + } +end + +local function public_snapshot(self) + return self._model:snapshot() +end + +local function publish_state(self) + if self._state_tx == nil then + return true, nil + end + + return state_mod.admit_component_snapshot_now( + self._state_tx, + self._link_id, + self._link_generation, + self._component_name, + public_snapshot(self), + 'fabric_bridge_state_admit_failed' + ) +end + +local function update_model(self, patch) + local changed = self._model:update(function (s) + for k, v in pairs(patch or {}) do + s[k] = v + end + + s.imported_topics = count_keys(self._imported_retained) + s.pending_calls = count_keys(self._pending_calls) + s.inbound_calls = count_keys(self._inbound_calls) + end) + + if changed then publish_state(self) end + return changed +end + +local function next_call_id(self) + self._next_call_seq = self._next_call_seq + 1 + return ('%s-call-%d'):format(self._link_id, self._next_call_seq) +end + +local function current_session(self) + return copy_context(self._session) +end + +local function has_current_session(self) + return type(self._session) == 'table' + and type(self._session.session_generation) == 'number' + and type(self._session.peer_sid) == 'string' + and self._session.peer_sid ~= '' +end + +-------------------------------------------------------------------------------- +-- Frame output +-------------------------------------------------------------------------------- + +local function admit_frame_now(outbound, session, frame, label) + local checked, err = protocol.validate(frame) + if not checked then + return nil, (label or 'bridge_frame_invalid') .. ': ' .. tostring(err) + end + + return outbound:send_rpc_frame_now(session, checked, label or 'bridge_frame_send_failed') +end + +local function record_frame_sent(self) + self._frames_sent = self._frames_sent + 1 + update_model(self, { frames_sent = self._frames_sent }) +end + +local function send_frame_now(self, frame, label, session) + local ok, err = admit_frame_now( + self._outbound, + session or current_session(self), + frame, + label + ) + + if ok == true then + record_frame_sent(self) + end + + return ok, err +end + +local function must_send_frame_now(self, frame, label, session) + local ok, err = send_frame_now(self, frame, label, session) + if ok ~= true then + error(err or label or 'bridge_frame_send_failed', 0) + end +end + +local function make_bridge_caps(self, session) + local ctx = copy_context(session) or current_session(self) + + return { + link_id = self._link_id, + link_generation = self._link_generation, + session = copy_context(ctx), + + send_rpc_frame_now = function (frame, label) + return admit_frame_now(self._outbound, ctx, frame, label) + end, + } +end + +-------------------------------------------------------------------------------- +-- Topic mapping +-------------------------------------------------------------------------------- + +local function map_or_passthrough(rules, topic, mapper) + if #rules == 0 then + return topics.copy(topic), nil + end + + return mapper(rules, topic) +end + +local function map_local_publish(self, topic, retain) + if retain and #self._export_retained_rules > 0 then + return map_or_passthrough(self._export_retained_rules, topic, topics.map_local_to_remote) + end + + return map_or_passthrough(self._export_publish_rules, topic, topics.map_local_to_remote) +end + +local function map_local_unretain(self, topic) + return map_or_passthrough(self._export_retained_rules, topic, topics.map_local_to_remote) +end + +local function map_local_call(self, topic) + return map_or_passthrough(self._outbound_call_rules, topic, topics.map_local_call_to_remote) +end + +local function map_remote_import(self, topic) + return map_or_passthrough(self._import_rules, topic, topics.map_remote_to_local) +end + +local function map_remote_call(self, topic) + return map_or_passthrough(self._inbound_call_rules, topic, topics.map_remote_call_to_local) +end + +-------------------------------------------------------------------------------- +-- Local outbound events +-------------------------------------------------------------------------------- + +local function remember_local_retained_export(self, topic, payload) + self._local_retained_exports[topics.key(topic)] = { + topic = topics.copy(topic), + payload = payload, + } +end + +local function forget_local_retained_export(self, topic) + self._local_retained_exports[topics.key(topic)] = nil +end + +local function send_retained_export_now(self, rec) + local remote_topic = map_local_publish(self, rec.topic, true) + if not remote_topic then return true end + + must_send_frame_now( + self, + protocol.pub(remote_topic, rec.payload, true), + 'bridge_retained_replay_send_failed' + ) + + return true +end + +local function replay_local_retained_exports(self) + if not has_current_session(self) then + return true + end + + for _, rec in pairs(self._local_retained_exports) do + send_retained_export_now(self, rec) + end + + return true +end + +local function handle_local_publish(self, ev) + local checked_topic, err = protocol.validate_topic(ev.topic) + if not checked_topic then + error('bridge publish invalid topic: ' .. tostring(err), 0) + end + + local remote_topic = map_local_publish(self, checked_topic, not not ev.retain) + if not remote_topic then + return + end + + if ev.retain then + remember_local_retained_export(self, checked_topic, ev.payload) + end + + if not has_current_session(self) then + return + end + + must_send_frame_now( + self, + protocol.pub(remote_topic, ev.payload, not not ev.retain), + 'bridge_publish_send_failed' + ) +end + +local function handle_local_unretain(self, ev) + local checked_topic, err = protocol.validate_topic(ev.topic) + if not checked_topic then + error('bridge unretain invalid topic: ' .. tostring(err), 0) + end + + local remote_topic = map_local_unretain(self, checked_topic) + if not remote_topic then + return + end + + forget_local_retained_export(self, checked_topic) + + if not has_current_session(self) then + return + end + + must_send_frame_now( + self, + protocol.unretain(remote_topic), + 'bridge_unretain_send_failed' + ) +end + +-------------------------------------------------------------------------------- +-- Scoped work +-------------------------------------------------------------------------------- + +local function report_done_to(self, label) + return function (ev) + return queue.try_admit_required(self._done_tx, ev, label) + end +end + +local function start_bridge_work(self, identity, run, report_label, cancel_op) + return scoped_work.start { + lifetime_scope = self._scope, + reaper_scope = self._scope, + report_scope = self._scope, + identity = identity, + run = run, + report = report_done_to(self, report_label), + cancel_op = cancel_op, + } +end + +-------------------------------------------------------------------------------- +-- Outbound calls +-------------------------------------------------------------------------------- + +local function run_outbound_call(call) + return function (call_scope) + call_scope:finally(function (_, status, primary) + call.owner:finalise_unresolved(primary or status or 'outbound_call_closed') + call.reply_tx:close(primary or status or 'outbound_call_closed') + end) + + local ok, send_err = call.caps.send_rpc_frame_now( + protocol.call(call.id, call.topic, call.payload), + 'bridge_outbound_call_send_failed' + ) + + if ok ~= true then + call.owner:fail_once(send_err or 'send_failed') + return { + call_id = call.id, + ok = false, + err = send_err or 'send_failed', + frame_sent = false, + } + end + + call.mark_frame_admitted() + + if call.reply_policy == 'sent-is-accepted' then + local payload = { + accepted = true, + frame_sent = true, + call_id = call.id, + } + + call.owner:reply_once(payload) + + return { + call_id = call.id, + ok = true, + frame_sent = true, + sent_is_accepted = true, + } + end + + local which, reply, recv_err = fibers.perform(fibers.named_choice { + reply = call.reply_rx:recv_op(), + timeout = sleep.sleep_op(call.timeout), + }) + + if which == 'timeout' then + call.owner:fail_once('timeout') + call_scope:cancel('timeout') + return { + call_id = call.id, + timed_out = true, + frame_sent = true, + } + end + + if reply == nil then + local why = recv_err + or (call.reply_rx.why and call.reply_rx:why()) + or 'closed' + + call.owner:fail_once(why) + return { + call_id = call.id, + closed = true, + err = tostring(why), + frame_sent = true, + } + end + + if reply.ok then + call.owner:reply_once(reply.payload) + + return { + call_id = call.id, + ok = true, + payload = reply.payload, + frame_sent = true, + } + end + + local err = reply.err or 'remote_call_failed' + call.owner:fail_once(err) + + return { + call_id = call.id, + ok = false, + err = err, + frame_sent = true, + } + end +end + +local function start_outbound_call(self, ev) + local checked_topic, err = protocol.validate_topic(ev.topic) + if not checked_topic then + error('bridge call invalid topic: ' .. tostring(err), 0) + end + + if not has_current_session(self) then + fail_request(ev.request or ev, 'no_session') + return + end + + local remote_topic, rule = map_local_call(self, checked_topic) + if not remote_topic then + fail_request(ev.request or ev, 'no_route') + return + end + + local id = ev.id or next_call_id(self) + if type(id) ~= 'string' or id == '' then + error('bridge call id must be a non-empty string', 0) + end + + if self._pending_calls[id] ~= nil then + fail_request(ev.request or ev, 'duplicate_call_id') + error('bridge duplicate outbound call id: ' .. id, 0) + end + + if count_keys(self._pending_calls) >= self._max_pending_calls then + fail_request(ev.request or ev, 'too_many_pending_calls') + return + end + + local reply_tx, reply_rx = mailbox.new(1, { full = 'reject_newest' }) + local owner = request_owner.new(ev.request or ev) + + local rec = { + id = id, + reply_tx = reply_tx, + reply_rx = reply_rx, + reply_routed = false, + frame_admitted = false, + session = current_session(self), + } + + local call = { + id = id, + topic = remote_topic, + payload = ev.payload, + timeout = ev.timeout or (rule and rule.timeout) or self._default_call_timeout, + owner = owner, + reply_policy = ev.reply_policy or (rule and rule.reply_policy) or 'reply-required', + caps = make_bridge_caps(self, rec.session), + reply_tx = reply_tx, + reply_rx = reply_rx, + + mark_frame_admitted = function () + rec.frame_admitted = true + return true + end, + } + + self._pending_calls[id] = rec + update_model(self) + + local handle, start_err = start_bridge_work( + self, + { + kind = 'outbound_call_done', + link_id = self._link_id, + link_generation = self._link_generation, + call_id = id, + }, + run_outbound_call(call), + 'bridge_outbound_call_completion_report_failed', + owner:caller_cancel_op() + ) + + if not handle then + self._pending_calls[id] = nil + reply_tx:close('outbound_call_start_failed') + update_model(self) + + owner:fail_once(start_err or 'outbound_call_start_failed') + error(start_err or 'outbound_call_start_failed', 0) + end + + rec.handle = handle +end + +-------------------------------------------------------------------------------- +-- Local bus command surface +-------------------------------------------------------------------------------- + +local function admit_bus_command(self, cmd, label) + if self._bus_tx == nil then + return nil, tostring(label or 'bridge_bus_command_failed') .. ': no_local_bus' + end + + return queue.try_admit_required( + self._bus_tx, + cmd, + label or 'bridge_bus_command_failed' + ) +end + +local function bus_publish_import(self, topic, payload, frame, session) + return admit_bus_command(self, { + kind = frame and frame.retain and 'retain' or 'publish', + topic = topic, + payload = payload, + frame = frame, + session = copy_context(session), + origin_kind = frame and frame.retain and 'remote_retain' or 'remote_publish', + }, 'bridge_bus_publish_import_failed') +end + +local function bus_unretain_import(self, topic, frame, session) + return admit_bus_command(self, { + kind = 'unretain', + topic = topic, + frame = frame, + session = copy_context(session), + origin_kind = 'remote_unretain', + }, 'bridge_bus_unretain_import_failed') +end + +local function perform_bus_call(call_scope, self, frame, session, timeout) + if self._bus_tx == nil then + return { ok = false, err = 'no_local_bus' } + end + + local reply_tx, reply_rx = mailbox.new(1, { full = 'reject_newest' }) + + call_scope:finally(function (_, status, primary) + reply_tx:close(primary or status or 'bus_call_closed') + end) + + local ok, err = admit_bus_command(self, { + kind = 'call', + topic = frame.topic, + payload = frame.payload, + frame = frame, + session = copy_context(session), + timeout = timeout, + reply_tx = reply_tx, + origin_kind = 'remote_call', + }, 'bridge_bus_call_admit_failed') + + if ok ~= true then + reply_tx:close(err or 'bus_call_admit_failed') + return { ok = false, err = err or 'bus_call_admit_failed' } + end + + local which, reply, recv_err = fibers.perform(fibers.named_choice { + reply = reply_rx:recv_op(), + timeout = sleep.sleep_op(timeout), + }) + + if which == 'timeout' then + return { ok = false, err = 'timeout' } + end + + if reply == nil then + return { + ok = false, + err = recv_err + or (reply_rx.why and reply_rx:why()) + or 'bus_call_closed', + } + end + + return reply +end + +-------------------------------------------------------------------------------- +-- Session and remote frames +-------------------------------------------------------------------------------- + +local cancel_pending_calls +local cancel_inbound_calls + +local function clear_imported_retained(self) + for key, rec in pairs(self._imported_retained) do + self._imported_retained[key] = nil + + if rec and rec.topic then + local frame = protocol.unretain(rec.topic) + local ok, err = bus_unretain_import(self, rec.topic, frame, rec.session) + if ok ~= true then + error(err or 'bridge_clear_imported_unretain_failed', 0) + end + end + end +end + +local function clear_peer_session(self, reason, session, opts) + opts = opts or {} + + if session ~= nil + and self._session ~= nil + and not same_session(self._session, session) + then + return + end + + local had_session_state = self._session ~= nil + or count_keys(self._imported_retained) > 0 + + if not had_session_state and self._session_drop_reason ~= nil then + return + end + + clear_imported_retained(self) + + if opts.cancel_calls ~= false then + cancel_pending_calls(self, reason or 'session_dropped') + cancel_inbound_calls(self, reason or 'session_dropped', false) + end + + self._session = nil + self._session_drop_reason = reason + + update_model(self, { + session = nil, + session_drop_reason = self._session_drop_reason, + }) +end + +local function require_session_context(ev, kind) + local ctx = context_from_event(ev) + + if type(ctx) ~= 'table' or type(ctx.session_generation) ~= 'number' then + error('bridge ' .. kind .. ' missing session context generation', 0) + end + + if type(ctx.peer_sid) ~= 'string' or ctx.peer_sid == '' then + error('bridge ' .. kind .. ' missing peer_sid', 0) + end + + return ctx +end + +local function handle_peer_session(self, ev) + local ctx = require_session_context(ev, 'peer_session') + + if self._session ~= nil and not same_session(self._session, ctx) then + clear_peer_session(self, 'new_peer_session') + end + + self._session = copy_context(ctx) + self._session_drop_reason = nil + + update_model(self, { + session = copy_context(self._session), + session_drop_reason = nil, + }) + + replay_local_retained_exports(self) +end + +local function handle_peer_session_dropped(self, ev) + local ctx = require_session_context(ev, 'peer_session_dropped') + + if not same_session(self._session, ctx) then + return + end + + clear_peer_session(self, ev.reason or 'session_dropped', ctx) +end + +local function apply_remote_publish(self, frame, session) + local local_topic = map_remote_import(self, frame.topic) + if not local_topic then + return + end + + local local_frame = protocol.pub(local_topic, frame.payload, not not frame.retain) + + if frame.retain then + self._imported_retained[topics.key(local_topic)] = { + topic = topics.copy(local_topic), + payload = frame.payload, + session = copy_context(session), + } + end + + local ok, err = bus_publish_import(self, local_topic, frame.payload, local_frame, session) + if ok ~= true then + error(err or 'bridge_bus_publish_import_failed', 0) + end + + update_model(self) +end + +local function apply_remote_unretain(self, frame, session) + local local_topic = map_remote_import(self, frame.topic) + if not local_topic then + return + end + + local local_frame = protocol.unretain(local_topic) + self._imported_retained[topics.key(local_topic)] = nil + + local ok, err = bus_unretain_import(self, local_topic, local_frame, session) + if ok ~= true then + error(err or 'bridge_bus_unretain_import_failed', 0) + end + + update_model(self) +end + +local function route_remote_reply(self, frame, session) + local rec = self._pending_calls[frame.id] + + if not rec or rec.reply_routed or not same_session(rec.session, session) then + return + end + + local routed = {} + for k, v in pairs(frame) do routed[k] = v end + routed.session = copy_context(session) + + local ok, err = queue.try_admit_required( + rec.reply_tx, + routed, + 'bridge_reply_route_failed' + ) + + if ok == true then + rec.reply_routed = true + return + end + + if type(err) == 'string' and err:match('closed') then + return + end + + error(err or 'bridge_reply_route_failed', 0) +end + +local function start_inbound_call(self, frame, session) + local id = frame.id + local local_topic, rule = map_remote_call(self, frame.topic) + + if not local_topic then + must_send_frame_now( + self, + protocol.reply(id, false, nil, 'no_route'), + 'bridge_no_route_inbound_reply_failed', + session + ) + return + end + + if self._inbound_calls[id] ~= nil then + must_send_frame_now( + self, + protocol.reply(id, false, nil, 'duplicate_call_id'), + 'bridge_duplicate_inbound_reply_failed', + session + ) + return + end + + if count_keys(self._inbound_calls) >= self._max_inbound_calls then + must_send_frame_now( + self, + protocol.reply(id, false, nil, 'too_many_inbound_calls'), + 'bridge_inbound_busy_reply_failed', + session + ) + return + end + + local local_frame = protocol.call(id, local_topic, frame.payload) + local timeout = rule and rule.timeout or self._default_call_timeout + + self._inbound_calls[id] = { + id = id, + session = copy_context(session), + } + + update_model(self) + + local handle, err = start_bridge_work( + self, + { + kind = 'inbound_call_done', + link_id = self._link_id, + link_generation = self._link_generation, + remote_call_id = id, + }, + function (call_scope) + local reply = perform_bus_call(call_scope, self, local_frame, session, timeout) + + if type(reply) ~= 'table' then + return { ok = false, err = 'local_call_failed' } + end + + if reply.ok == true then + return { ok = true, payload = reply.payload } + end + + return { ok = false, err = reply.err or 'local_call_failed' } + end, + 'bridge_inbound_call_completion_report_failed' + ) + + if not handle then + self._inbound_calls[id] = nil + update_model(self) + + must_send_frame_now( + self, + protocol.reply(id, false, nil, tostring(err or 'inbound_call_start_failed')), + 'bridge_inbound_start_failure_reply_failed', + session + ) + + error(err or 'inbound_call_start_failed', 0) + end + + self._inbound_calls[id].handle = handle +end + +local function handle_frame(self, ev) + local frame = ev.frame or ev + local session = context_from_event(ev) + + local checked, err = protocol.validate(frame) + if not checked then + error('bridge invalid frame: ' .. tostring(err), 0) + end + + if protocol.dispatch_lane(checked) ~= 'rpc' then + return + end + + self._frames_received = self._frames_received + 1 + update_model(self, { frames_received = self._frames_received }) + + if checked.type == 'pub' then + apply_remote_publish(self, checked, session) + + elseif checked.type == 'unretain' then + apply_remote_unretain(self, checked, session) + + elseif checked.type == 'reply' then + route_remote_reply(self, checked, session) + + elseif checked.type == 'call' then + start_inbound_call(self, checked, session) + + else + error('bridge unknown rpc frame type: ' .. tostring(checked.type), 0) + end +end + +local function session_frame_matches(self, ev) + if type(ev) ~= 'table' then + error('bridge session frame event must be a table', 0) + end + + if ev.kind ~= 'session_frame' then + error('bridge expected session_frame event', 0) + end + + if ev.lane ~= 'rpc' then + error('bridge received non-rpc session frame: ' .. tostring(ev.lane), 0) + end + + local ctx = require_session_context(ev, 'session_frame') + + return has_current_session(self) and same_session(self._session, ctx) +end + +local function handle_frame_event(self, ev) + if session_frame_matches(self, ev) then + handle_frame(self, ev) + end +end + +-------------------------------------------------------------------------------- +-- Completion handling +-------------------------------------------------------------------------------- + +local function handle_outbound_call_done(self, ev) + local rec = self._pending_calls[ev.call_id] + if not rec then return end + + self._pending_calls[ev.call_id] = nil + rec.reply_tx:close('outbound_call_done') + + if rec.frame_admitted then + record_frame_sent(self) + end + + update_model(self) +end + +local function inbound_reply_frame(ev) + local remote_id = ev.remote_call_id + + if ev.status ~= 'ok' then + return protocol.reply( + remote_id, + false, + nil, + tostring(ev.primary or ev.status or 'local_call_failed') + ) + end + + local result = ev.result or {} + + if result.ok == false then + return protocol.reply( + remote_id, + false, + nil, + tostring(result.err or 'local_call_failed') + ) + end + + local payload = result.payload + if payload == nil then payload = result.result end + + return protocol.reply(remote_id, true, payload, nil) +end + +local function handle_inbound_call_done(self, ev) + local rec = self._inbound_calls[ev.remote_call_id] + if rec == nil then return end + + must_send_frame_now( + self, + inbound_reply_frame(ev), + 'bridge_inbound_reply_send_failed', + rec.session + ) + + self._inbound_calls[ev.remote_call_id] = nil + update_model(self) +end + +local function handle_done(self, ev) + if ev.kind == 'outbound_call_done' then + handle_outbound_call_done(self, ev) + + elseif ev.kind == 'inbound_call_done' then + handle_inbound_call_done(self, ev) + + else + error('fabric.bridge: unknown completion kind: ' .. tostring(ev.kind), 0) + end +end + +-------------------------------------------------------------------------------- +-- Event selection +-------------------------------------------------------------------------------- + +local function normalise_local_item(item) + return item or { kind = 'local_closed' } +end + +local function normalise_session_item(item) + if item == nil then + return { kind = 'session_closed' } + end + + if type(item) ~= 'table' then + error('bridge expected session event', 0) + end + + return item +end + +local function done_event_from_recv(ev) + return ev or { kind = 'done_queue_closed' } +end + +local function try_done_now(self) + local ev, err = queue.try_recv_now(self._done_rx) + + if ev ~= nil then return done_event_from_recv(ev) end + if err ~= 'not_ready' then return done_event_from_recv(nil) end + + return nil +end + +local function try_local_now(self) + if self._stopping or self._local_closed then + return nil + end + + local item, err = queue.try_recv_now(self._local_rx) + + if item ~= nil then return normalise_local_item(item) end + if err ~= 'not_ready' then return normalise_local_item(nil) end + + return nil +end + +local function try_session_now(self) + if self._stopping or self._session_closed then + return nil + end + + local item, err = queue.try_recv_now(self._session_rx) + + if item ~= nil then return normalise_session_item(item) end + if err ~= 'not_ready' then return normalise_session_item(nil) end + + return nil +end + +local function next_event_op(self) + return priority_event.sources_op { + label = 'fabric.bridge', + pending = self._event_pending, + sources = { + { + name = 'session', + enabled = function () + return not self._stopping and not self._session_closed + end, + try_now = function () + return try_session_now(self) + end, + recv_op = function () + return self._session_rx:recv_op():wrap(normalise_session_item) + end, + }, + { + name = 'done', + try_now = function () + return try_done_now(self) + end, + recv_op = function () + return self._done_rx:recv_op():wrap(done_event_from_recv) + end, + }, + { + name = 'local_event', + enabled = function () + return not self._stopping and not self._local_closed + end, + try_now = function () + return try_local_now(self) + end, + recv_op = function () + return self._local_rx:recv_op():wrap(normalise_local_item) + end, + }, + }, + } +end + +-------------------------------------------------------------------------------- +-- Cancellation and shutdown +-------------------------------------------------------------------------------- + +cancel_pending_calls = function (self, reason, prune) + for id, rec in pairs(self._pending_calls) do + if rec.handle and rec.handle.cancel then + rec.handle:cancel(reason) + end + + if rec.reply_tx and type(rec.reply_tx.close) == 'function' then + rec.reply_tx:close(reason) + end + + if prune then + self._pending_calls[id] = nil + end + end + + update_model(self) +end + +cancel_inbound_calls = function (self, reason, reply_remote) + for id, rec in pairs(self._inbound_calls) do + if reply_remote and has_current_session(self) then + must_send_frame_now( + self, + protocol.reply(id, false, nil, tostring(reason or 'cancelled')), + 'bridge_inbound_cancel_reply_send_failed', + self._session + ) + end + + if rec.handle and rec.handle.cancel then + rec.handle:cancel(reason) + end + + self._inbound_calls[id] = nil + end + + update_model(self) +end + +local function should_finish(self) + if self._stopping then + return count_keys(self._pending_calls) == 0 + and count_keys(self._inbound_calls) == 0 + end + + if not (self._local_closed and self._session_closed) then + return false + end + + return count_keys(self._pending_calls) == 0 + and count_keys(self._inbound_calls) == 0 +end + +-------------------------------------------------------------------------------- +-- Coordinator +-------------------------------------------------------------------------------- + +local function handle_event(self, ev) + if ev.kind == 'done_queue_closed' then + error('fabric.bridge completion queue closed', 0) + + elseif ev.kind == 'local_closed' then + self._local_closed = true + update_model(self) + + elseif ev.kind == 'session_closed' then + self._session_closed = true + self._local_closed = true + clear_peer_session(self, 'session_closed') + update_model(self) + + elseif ev.kind == 'peer_session' then + handle_peer_session(self, ev) + + elseif ev.kind == 'peer_session_dropped' then + handle_peer_session_dropped(self, ev) + + elseif ev.kind == 'stop' then + local reason = ev.reason or 'stopped' + + self._stopping = true + self._local_closed = true + + cancel_pending_calls(self, reason) + cancel_inbound_calls(self, reason, true) + clear_peer_session(self, reason, nil, { cancel_calls = false }) + + update_model(self, { + state = 'stopping', + reason = reason, + }) + + elseif ev.kind == 'publish' then + handle_local_publish(self, ev) + + elseif ev.kind == 'unretain' then + handle_local_unretain(self, ev) + + elseif ev.kind == 'call' then + start_outbound_call(self, ev) + + elseif ev.kind == 'session_frame' then + handle_frame_event(self, ev) + + elseif ev.kind == 'outbound_call_done' + or ev.kind == 'inbound_call_done' + then + handle_done(self, ev) + + else + error('fabric.bridge: unknown event kind: ' .. tostring(ev.kind), 0) + end +end + +local function coordinator_loop(self) + update_model(self, { state = 'running' }) + + while not should_finish(self) do + handle_event(self, fibers.perform(next_event_op(self))) + end + + update_model(self, { state = 'completed' }) + + return { + role = 'rpc_bridge', + link_id = self._link_id, + link_generation = self._link_generation, + snapshot = public_snapshot(self), + } +end + +-------------------------------------------------------------------------------- +-- Public API +-------------------------------------------------------------------------------- + +function M.run(scope, params) + if type(scope) ~= 'table' then + error('fabric.bridge.run: scope required', 2) + end + + params = require_params(params) + + local link_id = params.link_id or 'link' + local link_generation = params.link_generation or 1 + + if type(link_id) ~= 'string' or link_id == '' then + error('fabric.bridge.run: link_id must be a non-empty string', 2) + end + + local model = model_mod.new(initial_snapshot(link_id, link_generation), { + label = 'fabric.bridge', + }) + + scope:finally(function (_, status, primary) + model:terminate(primary or status or 'fabric bridge closed') + end) + + local done_tx, done_rx = mailbox.new( + params.done_queue_len or DEFAULT_DONE_QUEUE, + { full = 'reject_newest' } + ) + + scope:finally(function () + done_tx:close('fabric bridge closed') + end) + + local self = setmetatable({ + _scope = scope, + _link_id = link_id, + _link_generation = link_generation, + _component_name = params.component_name or 'rpc_bridge', + _model = model, + + _local_rx = require_rx(params.local_rx, 'bridge: local_rx', 2), + _session_rx = require_rx(params.session_rx, 'bridge: session_rx', 2), + _outbound = require_outbound(params.outbound, 'bridge: outbound', 2), + _bus_tx = params.bus_tx, + _state_tx = params.state_tx, + _done_tx = done_tx, + _done_rx = done_rx, + + _default_call_timeout = positive_number(params.call_timeout_s, DEFAULT_CALL_TIMEOUT, 'bridge.call_timeout_s', 2), + _max_pending_calls = resolve_nonneg_int(params.max_pending_calls, DEFAULT_MAX_PENDING_CALLS, 'bridge.max_pending_calls', 2), + _max_inbound_calls = resolve_nonneg_int(params.max_inbound_calls, DEFAULT_MAX_INBOUND_CALLS, 'bridge.max_inbound_calls', 2), + + _import_rules = params.import_rules or {}, + _export_publish_rules = params.export_publish_rules or {}, + _export_retained_rules = params.export_retained_rules or {}, + _outbound_call_rules = params.outbound_call_rules or {}, + _inbound_call_rules = params.inbound_call_rules or {}, + + _imported_retained = {}, + _local_retained_exports = {}, + _pending_calls = {}, + _inbound_calls = {}, + + _local_closed = false, + _session_closed = false, + _stopping = false, + _session = nil, + _session_drop_reason = nil, + + _frames_sent = 0, + _frames_received = 0, + _event_pending = {}, + _next_call_seq = 0, + }, Bridge) + + publish_state(self) + + scope:finally(function (_, status, primary) + local reason = primary or status or 'fabric bridge closed' + + clear_peer_session(self, reason, nil, { cancel_calls = false }) + cancel_pending_calls(self, reason, true) + cancel_inbound_calls(self, reason, false) + end) + + return coordinator_loop(self) +end + +M.make_bridge_caps = make_bridge_caps +M.RequestOwner = request_owner.RequestOwner +M.Bridge = Bridge + +return M diff --git a/src/services/fabric/bus_adapter.lua b/src/services/fabric/bus_adapter.lua new file mode 100644 index 00000000..08d50baf --- /dev/null +++ b/src/services/fabric/bus_adapter.lua @@ -0,0 +1,554 @@ +-- services/fabric/bus_adapter.lua +-- +-- Local bus adapter for Fabric bridge semantics. +-- +-- Owns local-bus subscriptions, retained watches, endpoint bindings, imported +-- publication/unretain, and local RPC calls on behalf of remote Fabric peers. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' + +local queue = require 'devicecode.support.queue' +local service_events = require 'devicecode.support.service_events' +local scoped_work = require 'devicecode.support.scoped_work' +local priority_event = require 'devicecode.support.priority_event' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local topics = require 'services.fabric.topics' +local session_mod = require 'services.fabric.session' + +local M = {} + +local DEFAULT_CALL_TIMEOUT = 1.0 +local DEFAULT_LOCAL_EVENT_QUEUE = 64 +local DEFAULT_COMMAND_QUEUE = 64 + +local function close_list(conn, list, close_fn, label) + local first_err + + for i = #list, 1, -1 do + local item = list[i] + list[i] = nil + + local ok, err = close_fn(conn, item) + if ok ~= true and first_err == nil then + first_err = err or label + end + end + + if first_err then + return nil, first_err + end + + return true, nil +end + +local function checked_topic(v, label) + if type(v) ~= 'table' then + return nil, label + end + return topics.copy(v), nil +end + +local function rule_watch_topic(rule) + return checked_topic( + type(rule) == 'table' and rule.local_watch_topic or nil, + 'fabric bus adapter rule has no local_watch_topic' + ) +end + +local function rule_call_topic(rule) + return checked_topic( + type(rule) == 'table' and rule.local_topic or nil, + 'fabric bus adapter outbound call rule has no local_topic' + ) +end + +local function origin_kind(cmd) + if type(cmd.origin_kind) ~= 'string' or cmd.origin_kind == '' then + error('fabric bus adapter command missing origin_kind', 0) + end + + return cmd.origin_kind +end + +local function origin_extra(params, kind, item) + local fabric = { + kind = kind, + link_id = params.link_id, + link_generation = params.link_generation or params.generation, + } + + if type(item) == 'table' and item.session ~= nil then + fabric.session = session_mod.copy_context(item.session) + end + + local frame = item and item.frame or item + if type(frame) == 'table' then + fabric.call_id = frame.id + fabric.frame_type = frame.type + end + + return { fabric = fabric } +end + +local function reply_once(cmd) + local done = false + + return function (value, label) + if done then + return false, 'already_replied' + end + + done = true + + if cmd.reply_tx == nil then + return true, nil + end + + return queue.try_admit_required( + cmd.reply_tx, + value, + label or 'fabric_bus_adapter_call_reply_failed' + ) + end, + function () + return done + end +end + +function M.local_runtime(scope, conn, params) + if type(scope) ~= 'table' then + error('fabric.bus_adapter.local_runtime: scope required', 2) + end + if type(conn) ~= 'table' then + error('fabric.bus_adapter.local_runtime: conn required', 2) + end + + params = params or {} + + local local_tx, local_rx = mailbox.new( + params.local_queue_len or DEFAULT_LOCAL_EVENT_QUEUE, + { full = 'reject_newest' } + ) + + local command_tx, command_rx = mailbox.new( + params.bus_command_queue_len or params.command_queue_len or DEFAULT_COMMAND_QUEUE, + { full = 'reject_newest' } + ) + + local done_tx, done_rx = mailbox.new( + params.bus_call_done_queue_len or params.call_done_queue_len or DEFAULT_COMMAND_QUEUE, + { full = 'reject_newest' } + ) + + local runtime = { + local_rx = local_rx, + command_tx = command_tx, + + _subs = {}, + _watches = {}, + _eps = {}, + _calls = {}, + _call_count = 0, + _closed = false, + _command_loop_started = false, + } + + local function cancel_calls(reason) + for _, rec in pairs(runtime._calls) do + if rec.handle and rec.handle.cancel then + rec.handle:cancel(reason or 'fabric bus adapter closing') + end + end + end + + local function terminate(_, reason) + if runtime._closed then + return true, nil + end + + runtime._closed = true + cancel_calls(reason or 'fabric bus adapter runtime closed') + + local first_err + local ok, err + + ok, err = close_list(conn, runtime._eps, bus_cleanup.unbind, 'fabric bus adapter endpoint cleanup failed') + if ok ~= true and first_err == nil then first_err = err end + + ok, err = close_list(conn, runtime._watches, bus_cleanup.unwatch_retained, 'fabric bus adapter retained-watch cleanup failed') + if ok ~= true and first_err == nil then first_err = err end + + ok, err = close_list(conn, runtime._subs, bus_cleanup.unsubscribe, 'fabric bus adapter subscription cleanup failed') + if ok ~= true and first_err == nil then first_err = err end + + local why = reason or first_err or 'fabric bus adapter runtime closed' + local_tx:close(why) + command_tx:close(why) + + -- Once the command loop has started it owns done_tx until it has + -- drained active call completions. Closing it here can race the loop + -- into seeing call_done_closed before command_closed, turning orderly + -- runtime termination into a coordinator failure. During setup failure, + -- before the command loop exists, terminate still closes done_tx. + if not runtime._command_loop_started then + done_tx:close(why) + end + + if first_err then + return nil, first_err + end + + return true, nil + end + + runtime.terminate = terminate + + local function abort_setup(err) + runtime:terminate('fabric bus adapter runtime setup failed') + error(err, 3) + end + + local function admit_local(ev, label) + queue.assert_admit_required(local_tx, ev, label, 3) + end + + local function subscribe(rule, retained) + local topic, err = rule_watch_topic(rule) + if not topic then abort_setup(err) end + + local handle, herr + if retained then + handle, herr = bus_cleanup.watch_retained(conn, topic, { + replay = true, + queue_len = params.local_feed_queue_len, + full = params.local_feed_full or 'reject_newest', + }) + else + handle, herr = bus_cleanup.subscribe(conn, topic, { + queue_len = params.local_feed_queue_len, + full = params.local_feed_full or 'reject_newest', + }) + end + + if not handle then + abort_setup(herr or 'fabric bus adapter local feed setup failed') + end + + if retained then + runtime._watches[#runtime._watches + 1] = handle + + local ok_spawn, spawn_err = fibers.spawn(function () + while true do + local ev = fibers.perform(handle:recv_op()) + if ev == nil then return end + + if ev.op == 'retain' then + admit_local({ + kind = 'publish', + topic = ev.topic, + payload = ev.payload, + retain = true, + }, 'fabric_bus_adapter_retained_event_admit_failed') + + elseif ev.op == 'unretain' then + admit_local({ + kind = 'unretain', + topic = ev.topic, + }, 'fabric_bus_adapter_unretain_event_admit_failed') + + elseif ev.op ~= 'replay_done' then + error('fabric bus adapter retained watch unknown event: ' .. tostring(ev.op), 0) + end + end + end) + if not ok_spawn then + abort_setup('fabric bus adapter retained source spawn failed: ' .. tostring(spawn_err)) + end + + else + runtime._subs[#runtime._subs + 1] = handle + + local ok_spawn, spawn_err = fibers.spawn(function () + while true do + local msg = fibers.perform(handle:recv_op()) + if msg == nil then return end + + admit_local({ + kind = 'publish', + topic = msg.topic, + payload = msg.payload, + retain = false, + }, 'fabric_bus_adapter_publish_event_admit_failed') + end + end) + if not ok_spawn then + abort_setup('fabric bus adapter publish source spawn failed: ' .. tostring(spawn_err)) + end + end + end + + local function bind(rule) + local topic, err = rule_call_topic(rule) + if not topic then abort_setup(err) end + + local ep, eerr = bus_cleanup.bind(conn, topic, { + queue_len = params.local_endpoint_queue_len or 1, + }) + if not ep then abort_setup(eerr or 'fabric bus adapter bind failed') end + + runtime._eps[#runtime._eps + 1] = ep + + local ok_spawn, spawn_err = fibers.spawn(function () + while true do + local req = fibers.perform(ep:recv_op()) + if req == nil then return end + + local ok, admit_err = queue.try_admit_required(local_tx, { + kind = 'call', + topic = req.topic, + payload = req.payload, + timeout = rule.timeout or params.call_timeout_s, + reply_policy = rule.reply_policy, + request = req, + }, 'fabric_bus_adapter_call_event_admit_failed') + + if ok ~= true then + bus_cleanup.fail(req, 'fabric_bus_adapter_queue_full') + error(admit_err or 'fabric_bus_adapter_call_event_admit_failed', 0) + end + end + end) + if not ok_spawn then + abort_setup('fabric bus adapter endpoint source spawn failed: ' .. tostring(spawn_err)) + end + end + + local function publish_to_bus(cmd) + local opts = { + extra = origin_extra(params, origin_kind(cmd), cmd), + } + + if cmd.kind == 'retain' then + return bus_cleanup.retain(conn, cmd.topic, cmd.payload, opts) + end + + return bus_cleanup.publish(conn, cmd.topic, cmd.payload, opts) + end + + local function unretain_from_bus(cmd) + return bus_cleanup.unretain(conn, cmd.topic, { + extra = origin_extra(params, origin_kind(cmd), cmd), + }) + end + + local function start_call(cmd, seq) + local key = 'bus-call-' .. tostring(seq) + local reply, replied = reply_once(cmd) + + local handle, err = scoped_work.start { + lifetime_scope = scope, + reaper_scope = scope, + report_scope = scope, + + identity = { + kind = 'call_done', + call_key = key, + }, + + run = function (call_scope) + call_scope:finally(function (_, status, primary) + reply({ + ok = false, + err = tostring(primary or status or 'local_call_closed'), + }, 'fabric_bus_adapter_call_final_reply_failed') + end) + + local value, call_err = fibers.perform(conn:call_op(cmd.topic, cmd.payload, { + timeout = cmd.timeout or params.call_timeout_s or DEFAULT_CALL_TIMEOUT, + extra = origin_extra(params, origin_kind(cmd), cmd), + })) + + if value == nil then + local reason = call_err or 'local_call_failed' + reply({ ok = false, err = reason }, 'fabric_bus_adapter_call_reply_failed') + return { ok = false, err = reason } + end + + reply({ ok = true, payload = value }, 'fabric_bus_adapter_call_reply_failed') + return { ok = true } + end, + + report = service_events.reporter_for( + done_tx, + { + kind = 'call_done', + call_key = key, + source = 'fabric_bus_adapter_call', + source_id = key, + }, + { label = 'fabric_bus_adapter_call_completion_report_failed' } + ), + } + + if not handle then + reply({ + ok = false, + err = err or 'local_call_start_failed', + }, 'fabric_bus_adapter_call_start_reply_failed') + + return nil, err or 'local_call_start_failed' + end + + runtime._calls[key] = { + handle = handle, + reply = reply, + replied = replied, + } + runtime._call_count = runtime._call_count + 1 + + return true, nil, seq + 1 + end + + local function command_event(cmd) + return cmd and { kind = 'command', cmd = cmd } or { kind = 'command_closed' } + end + + local function done_event(ev) + return ev or { kind = 'call_done_closed' } + end + + local function try_recv(rx, map) + local item, err = queue.try_recv_now(rx) + + if item ~= nil then + return map(item) + end + + if err ~= 'not_ready' then + return map(nil) + end + + return nil + end + + local function command_loop() + local command_open = true + local pending = {} + local seq = 1 + + scope:finally(function (_, status, primary) + local why = primary or status or 'fabric bus adapter command loop closed' + cancel_calls(why) + done_tx:close(why) + end) + + local function next_event_op() + return priority_event.sources_op { + label = 'fabric.bus_adapter.commands', + pending = pending, + sources = { + { + name = 'done', + try_now = function () return try_recv(done_rx, done_event) end, + recv_op = function () return done_rx:recv_op():wrap(done_event) end, + }, + { + name = 'command', + enabled = function () return command_open end, + try_now = function () return try_recv(command_rx, command_event) end, + recv_op = function () return command_rx:recv_op():wrap(command_event) end, + }, + }, + } + end + + while command_open or runtime._call_count > 0 do + local ev = fibers.perform(next_event_op()) + + if ev.kind == 'command_closed' then + command_open = false + cancel_calls('fabric bus adapter command queue closed') + + elseif ev.kind == 'call_done_closed' then + if command_open or runtime._call_count > 0 then + error('fabric bus adapter call completion queue closed', 0) + end + + elseif ev.kind == 'call_done' then + local rec = runtime._calls[ev.call_key] + if rec then + runtime._calls[ev.call_key] = nil + runtime._call_count = runtime._call_count - 1 + + if ev.status ~= 'ok' and not rec.replied() then + rec.reply({ + ok = false, + err = tostring(ev.primary or ev.status or 'local_call_failed'), + }, 'fabric_bus_adapter_call_done_reply_failed') + end + end + + elseif ev.kind == 'command' then + local cmd = ev.cmd + local ok, err + + if cmd.kind == 'publish' or cmd.kind == 'retain' then + ok, err = publish_to_bus(cmd) + + elseif cmd.kind == 'unretain' then + ok, err = unretain_from_bus(cmd) + + elseif cmd.kind == 'call' then + ok, err, seq = start_call(cmd, seq) + + elseif cmd.kind == 'stop' then + command_open = false + cancel_calls('fabric bus adapter stopped') + ok = true + + else + error('fabric bus adapter unknown command: ' .. tostring(cmd.kind), 0) + end + + if ok ~= true then + error(err or ('fabric bus adapter command failed: ' .. tostring(cmd.kind)), 0) + end + + else + error('fabric bus adapter unknown event: ' .. tostring(ev.kind), 0) + end + end + + done_tx:close('fabric bus adapter command loop completed') + end + + for _, rule in ipairs(params.export_publish_rules or {}) do + subscribe(rule, false) + end + + for _, rule in ipairs(params.export_retained_rules or {}) do + subscribe(rule, true) + end + + for _, rule in ipairs(params.outbound_call_rules or {}) do + bind(rule) + end + + local ok_spawn, spawn_err = fibers.spawn(command_loop) + if not ok_spawn then + abort_setup('fabric bus adapter command loop spawn failed: ' .. tostring(spawn_err)) + end + runtime._command_loop_started = true + + scope:finally(function () + local ok, err = runtime:terminate('fabric bus adapter scope closed') + if ok ~= true then + error(err or 'fabric bus adapter cleanup failed', 0) + end + end) + + return runtime +end + +return M diff --git a/src/services/fabric/config.lua b/src/services/fabric/config.lua new file mode 100644 index 00000000..3af43ce0 --- /dev/null +++ b/src/services/fabric/config.lua @@ -0,0 +1,638 @@ +-- services/fabric/config.lua +-- +-- Pure Fabric configuration compiler. +-- +-- This module validates untrusted raw configuration and returns a normalised +-- runtime plan for the new Fabric semantic modules. It intentionally performs +-- no I/O, starts no fibres, and calls no bus/HAL APIs. +-- +---@module 'services.fabric.config' +-- +-- Fabric configuration compiler. +-- +-- Contract: +-- * raw config is untrusted and fully validated here; +-- * the returned compiled config is normalised and may be trusted by services; +-- * compiled config does not alias raw config tables; +-- * compiled tables are sealed against new fields, but not proxy-immutable +-- (Lua 5.1 cannot trap writes to existing keys without much heavier wrappers). + +local M = {} + +local SCHEMA = 'devicecode.config/fabric/1' + +local DEFAULTS = { + reader = { + bad_frame_limit = 5, + bad_frame_window_s = 10.0, + }, + session = { + hello_interval_s = 2.0, + ping_interval_s = 10.0, + liveness_timeout_s = 30.0, + }, + writer = { + rpc_quota = 4, + bulk_quota = 1, + }, + bridge = { + max_pending_calls = 64, + max_inbound_calls = 64, + call_timeout_s = 5.0, + }, + transfer = { + chunk_size = 2048, + timeout_s = 30.0, + }, + queues = { + rpc_in = 64, + xfer_in = 64, + tx_control = 32, + tx_rpc = 128, + tx_bulk = 64, + }, +} + +local ROOT_KEYS = { + schema = true, local_node = true, links = true, +} + +local LINK_KEYS = { + id = true, peer_id = true, + transport = true, session = true, reader = true, writer = true, + bridge = true, transfer = true, queues = true, +} + +local TRANSPORT_KEYS = { + kind = true, source = true, class = true, id = true, + open_verb = true, open_opts = true, terminator = true, +} + +local SESSION_KEYS = { + local_node = true, identity_claim = true, auth_claim = true, + hello_interval_s = true, ping_interval_s = true, liveness_timeout_s = true, +} + +local READER_KEYS = { + bad_frame_limit = true, bad_frame_window_s = true, +} + +local WRITER_KEYS = { rpc_quota = true, bulk_quota = true } + +local BRIDGE_KEYS = { + imports = true, exports = true, rpc = true, + max_pending_calls = true, max_inbound_calls = true, call_timeout_s = true, +} + +local BRIDGE_RPC_KEYS = { inbound = true, outbound = true } + +local RULE_KEYS = { + id = true, ['local'] = true, remote = true, topic = true, timeout_s = true, +} + +local RPC_RULE_KEYS = { + id = true, ['local'] = true, remote = true, topic = true, + timeout_s = true, reply_policy = true, +} + +local EXPORT_RULE_KEYS = { + id = true, ['local'] = true, remote = true, topic = true, + timeout_s = true, publish = true, retain = true, +} + +local TRANSFER_KEYS = { + chunk_size = true, timeout_s = true, +} + +local QUEUE_KEYS = { + rpc_in = true, xfer_in = true, + tx_control = true, tx_rpc = true, tx_bulk = true, +} + +------------------------------------------------------------------------------- +-- Tiny validator / compiler helpers +------------------------------------------------------------------------------- + +local SEALED_MT = { + __newindex = function(_, k) + error('attempt to add field to compiled fabric config: ' .. tostring(k), 2) + end, + __metatable = false, +} + +local function fail(msg) return nil, msg end + +local function is_finite_number(v) + return type(v) == 'number' and v == v and v ~= math.huge and v ~= -math.huge +end + +local function copy_plain(v) + if type(v) ~= 'table' then return v end + local out = {} + for k, subv in pairs(v) do out[k] = copy_plain(subv) end + return out +end + +local function seal(v, seen) + if type(v) ~= 'table' then return v end + seen = seen or {} + if seen[v] then return v end + seen[v] = true + for _, subv in pairs(v) do seal(subv, seen) end + return setmetatable(v, SEALED_MT) +end + +local function allowed(t, keys, path) + for k in pairs(t) do + if not keys[k] then + return nil, path .. ' has unknown field: ' .. tostring(k) + end + end + return true, nil +end + +local function table_or_empty(v, path) + if v == nil then return {}, nil end + if type(v) ~= 'table' then return nil, path .. ' must be a table' end + return v, nil +end + +local function list_or_empty(v, path) + if v == nil then return {}, nil end + if type(v) ~= 'table' then return nil, path .. ' must be a dense list' end + local n = #v + for k in pairs(v) do + if type(k) ~= 'number' or k < 1 or k % 1 ~= 0 or k > n then + return nil, path .. ' must be a dense list' + end + end + return v, nil +end + +local function str(v, path) + if type(v) ~= 'string' or v == '' then + return nil, path .. ' must be a non-empty string' + end + return v, nil +end + +local function opt_str(v, path, default) + if v == nil then return default, nil end + return str(v, path) +end + +local function bool(v, path, default) + if v == nil then return default, nil end + if type(v) ~= 'boolean' then return nil, path .. ' must be boolean' end + return v, nil +end + +local function opt_number(v, path, default) + if v == nil then return default, nil end + if not is_finite_number(v) then return nil, path .. ' must be a finite number' end + return v, nil +end + +local function opt_pos_number(v, path) + if v == nil then return nil, nil end + if not is_finite_number(v) or v <= 0 then + return nil, path .. ' must be a positive finite number' + end + return v, nil +end + +local function pos_number(v, path, default) + if v == nil then v = default end + if not is_finite_number(v) or v <= 0 then + return nil, path .. ' must be a positive finite number' + end + return v, nil +end + +local function nonneg_int(v, path, default) + local n, err = opt_number(v, path, default) + if err then return nil, err end + if type(n) ~= 'number' or n < 0 or n % 1 ~= 0 then + return nil, path .. ' must be a non-negative integer' + end + return n, nil +end + +local function pos_int(v, path, default) + local n, err = opt_number(v, path, default) + if err then return nil, err end + if type(n) ~= 'number' or n <= 0 or n % 1 ~= 0 then + return nil, path .. ' must be a positive integer' + end + return n, nil +end + + +local function rpc_reply_policy(v, path) + if v == nil then return 'reply-required', nil end + if v == 'reply-required' or v == 'sent-is-accepted' then + return v, nil + end + return nil, path .. ' must be reply-required or sent-is-accepted' +end + +local function topic_token_ok(v) + if type(v) == 'string' then return v ~= '' end + return is_finite_number(v) +end + +local function topic(v, path) + if type(v) ~= 'table' then return nil, path .. ' must be a dense topic array' end + local n = #v + for i = 1, n do + if not topic_token_ok(v[i]) then + return nil, path .. '[' .. tostring(i) .. '] must be string or finite number' + end + end + for k in pairs(v) do + if type(k) ~= 'number' or k < 1 or k % 1 ~= 0 or k > n then + return nil, path .. ' must be a dense topic array' + end + end + local out = {} + for i = 1, n do out[i] = v[i] end + return out, nil +end + +local function append_index(index, key, value) + local bucket = index[key] + if bucket == nil then + bucket = {} + index[key] = bucket + end + bucket[#bucket + 1] = value +end + + +------------------------------------------------------------------------------- +-- Section compilers +------------------------------------------------------------------------------- + +local function compile_transport(raw, link_id) + raw = raw or {} + if type(raw) ~= 'table' then return nil, 'transport must be a table' end + local ok, err = allowed(raw, TRANSPORT_KEYS, 'transport') + if not ok then return nil, err end + + local kind, e1 = opt_str(raw.kind, 'transport.kind', 'uart') + if e1 then return nil, e1 end + local source, e2 = opt_str(raw.source, 'transport.source', kind) + if e2 then return nil, e2 end + local class, e3 = opt_str(raw.class, 'transport.class', kind) + if e3 then return nil, e3 end + local id, e4 = opt_str(raw.id, 'transport.id', link_id) + if e4 then return nil, e4 end + local open_verb, e5 = opt_str(raw.open_verb, 'transport.open_verb', 'open') + if e5 then return nil, e5 end + + local terminator = raw.terminator + if terminator == nil then terminator = '\n' end + if type(terminator) ~= 'string' then + return nil, 'transport.terminator must be a string' + end + + return { + kind = kind, + source = source, + class = class, + id = id, + open_verb = open_verb, + open_opts = copy_plain(raw.open_opts), + terminator = terminator, + }, nil +end + +local function compile_session(raw, service_local_node) + raw = raw or {} + if type(raw) ~= 'table' then return nil, 'session must be a table' end + local ok, err = allowed(raw, SESSION_KEYS, 'session') + if not ok then return nil, err end + + local local_node, e1 = opt_str(raw.local_node, 'session.local_node', service_local_node) + if e1 then return nil, e1 end + local hello, e2 = pos_number(raw.hello_interval_s, 'session.hello_interval_s', DEFAULTS.session.hello_interval_s) + if e2 then return nil, e2 end + local ping, e3 = pos_number(raw.ping_interval_s, 'session.ping_interval_s', DEFAULTS.session.ping_interval_s) + if e3 then return nil, e3 end + local live, e4 = pos_number(raw.liveness_timeout_s, 'session.liveness_timeout_s', DEFAULTS.session.liveness_timeout_s) + if e4 then return nil, e4 end + + return { + local_node = local_node, + identity_claim = copy_plain(raw.identity_claim), + auth_claim = copy_plain(raw.auth_claim), + hello_interval_s = hello, + ping_interval_s = ping, + liveness_timeout_s = live, + }, nil +end + +local function compile_reader(raw) + raw = raw or {} + if type(raw) ~= 'table' then return nil, 'reader must be a table' end + local ok, err = allowed(raw, READER_KEYS, 'reader') + if not ok then return nil, err end + + local bad_frame_limit, e1 = pos_int(raw.bad_frame_limit, 'reader.bad_frame_limit', DEFAULTS.reader.bad_frame_limit) + if e1 then return nil, e1 end + local bad_frame_window_s, e2 = pos_number(raw.bad_frame_window_s, 'reader.bad_frame_window_s', DEFAULTS.reader.bad_frame_window_s) + if e2 then return nil, e2 end + + return { + bad_frame_limit = bad_frame_limit, + bad_frame_window_s = bad_frame_window_s, + }, nil +end + +local function compile_writer(raw) + raw = raw or {} + if type(raw) ~= 'table' then return nil, 'writer must be a table' end + local ok, err = allowed(raw, WRITER_KEYS, 'writer') + if not ok then return nil, err end + + local rpc_quota, e1 = pos_int(raw.rpc_quota, 'writer.rpc_quota', DEFAULTS.writer.rpc_quota) + if e1 then return nil, e1 end + local bulk_quota, e2 = pos_int(raw.bulk_quota, 'writer.bulk_quota', DEFAULTS.writer.bulk_quota) + if e2 then return nil, e2 end + + return { rpc_quota = rpc_quota, bulk_quota = bulk_quota }, nil +end + +local function compile_rule_item(raw, direction, path, keys) + if type(raw) ~= 'table' then return nil, path .. ' must be a table' end + local ok, err = allowed(raw, keys, path) + if not ok then return nil, err end + if raw.timeout ~= nil then return nil, path .. '.timeout is not supported; use timeout_s' end + + local local_prefix, e1 = topic(raw['local'], path .. '.local') + if e1 then return nil, e1 end + local remote_prefix, e2 = topic(raw.remote, path .. '.remote') + if e2 then return nil, e2 end + + local rule_topic = nil + if raw.topic ~= nil then + local e3 + rule_topic, e3 = topic(raw.topic, path .. '.topic') + if e3 then return nil, e3 end + end + + local timeout_s, e4 = opt_pos_number(raw.timeout_s, path .. '.timeout_s') + if e4 then return nil, e4 end + + local local_watch_topic = rule_topic + if local_watch_topic == nil then + local_watch_topic = {} + for i = 1, #local_prefix do local_watch_topic[i] = local_prefix[i] end + local_watch_topic[#local_watch_topic + 1] = '#' + end + + return { + id = raw.id, + local_prefix = local_prefix, + remote_prefix = remote_prefix, + topic = rule_topic, + local_watch_topic = local_watch_topic, + timeout = timeout_s, + direction = direction, + }, nil +end + +local function compile_rule_list(raw, direction, path, keys) + local list, err = list_or_empty(raw, path) + if not list then return nil, err end + local out = {} + for i = 1, #list do + local rule, rerr = compile_rule_item(list[i], direction, path .. '[' .. tostring(i) .. ']', keys) + if not rule then return nil, rerr end + out[#out + 1] = rule + end + return out, nil +end + +local function compile_rpc_rule_item(raw, direction, path) + if type(raw) ~= 'table' then return nil, path .. ' must be a table' end + local ok, err = allowed(raw, RPC_RULE_KEYS, path) + if not ok then return nil, err end + if raw.timeout ~= nil then return nil, path .. '.timeout is not supported; use timeout_s' end + if raw.topic ~= nil then return nil, path .. '.topic is not supported for rpc rules; use exact local/remote topics' end + + local local_topic, e1 = topic(raw['local'], path .. '.local') + if e1 then return nil, e1 end + local remote_topic, e2 = topic(raw.remote, path .. '.remote') + if e2 then return nil, e2 end + local timeout_s, e3 = opt_pos_number(raw.timeout_s, path .. '.timeout_s') + if e3 then return nil, e3 end + local reply_policy, e4 = rpc_reply_policy(raw.reply_policy, path .. '.reply_policy') + if e4 then return nil, e4 end + + return { + id = raw.id, + local_topic = local_topic, + remote_topic = remote_topic, + timeout = timeout_s, + reply_policy = reply_policy, + direction = direction, + }, nil +end + +local function compile_rpc_rule_list(raw, direction, path) + local list, err = list_or_empty(raw, path) + if not list then return nil, err end + local out = {} + for i = 1, #list do + local rule, rerr = compile_rpc_rule_item(list[i], direction, path .. '[' .. tostring(i) .. ']') + if not rule then return nil, rerr end + out[#out + 1] = rule + end + return out, nil +end + +local function compile_bridge(raw) + raw = raw or {} + if type(raw) ~= 'table' then return nil, 'bridge must be a table' end + local ok, err = allowed(raw, BRIDGE_KEYS, 'bridge') + if not ok then return nil, err end + + local import_rules, e1 = compile_rule_list(raw.imports, 'import', 'bridge.imports', RULE_KEYS) + if e1 then return nil, e1 end + + local exports, e2 = list_or_empty(raw.exports, 'bridge.exports') + if not exports then return nil, e2 end + + local export_publish_raw, export_retained_raw = {}, {} + for i = 1, #exports do + local item = exports[i] + local path = 'bridge.exports[' .. tostring(i) .. ']' + if type(item) ~= 'table' then return nil, path .. ' must be a table' end + local eok, eerr = allowed(item, EXPORT_RULE_KEYS, path) + if not eok then return nil, eerr end + if item.timeout ~= nil then return nil, path .. '.timeout is not supported; use timeout_s' end + + local publish, perr = bool(item.publish, path .. '.publish', true) + if perr then return nil, perr end + local retain, rerr = bool(item.retain, path .. '.retain', false) + if rerr then return nil, rerr end + if not publish and not retain then return nil, path .. ' must enable publish or retain' end + + if publish then export_publish_raw[#export_publish_raw + 1] = item end + if retain then export_retained_raw[#export_retained_raw + 1] = item end + end + + local export_publish_rules, e3 = compile_rule_list(export_publish_raw, 'export_publish', 'bridge.exports', EXPORT_RULE_KEYS) + if e3 then return nil, e3 end + local export_retained_rules, e4 = compile_rule_list(export_retained_raw, 'export_retained', 'bridge.exports', EXPORT_RULE_KEYS) + if e4 then return nil, e4 end + + local rpc, e5 = table_or_empty(raw.rpc, 'bridge.rpc') + if not rpc then return nil, e5 end + local rok, rerr = allowed(rpc, BRIDGE_RPC_KEYS, 'bridge.rpc') + if not rok then return nil, rerr end + + local outbound_call_rules, e6 = compile_rpc_rule_list(rpc.outbound, 'outbound_call', 'bridge.rpc.outbound') + if e6 then return nil, e6 end + local inbound_call_rules, e7 = compile_rpc_rule_list(rpc.inbound, 'inbound_call', 'bridge.rpc.inbound') + if e7 then return nil, e7 end + + local max_pending_calls, e8 = nonneg_int(raw.max_pending_calls, 'bridge.max_pending_calls', DEFAULTS.bridge.max_pending_calls) + if e8 then return nil, e8 end + local max_inbound_calls, e9 = nonneg_int(raw.max_inbound_calls, 'bridge.max_inbound_calls', DEFAULTS.bridge.max_inbound_calls) + if e9 then return nil, e9 end + local call_timeout_s, e10 = pos_number(raw.call_timeout_s, 'bridge.call_timeout_s', DEFAULTS.bridge.call_timeout_s) + if e10 then return nil, e10 end + + return { + import_rules = import_rules, + export_publish_rules = export_publish_rules, + export_retained_rules = export_retained_rules, + outbound_call_rules = outbound_call_rules, + inbound_call_rules = inbound_call_rules, + max_pending_calls = max_pending_calls, + max_inbound_calls = max_inbound_calls, + call_timeout_s = call_timeout_s, + }, nil +end + +local function compile_transfer(raw) + raw = raw or {} + if type(raw) ~= 'table' then return nil, 'transfer must be a table' end + local ok, err = allowed(raw, TRANSFER_KEYS, 'transfer') + if not ok then return nil, err end + + local chunk_size, e2 = pos_int(raw.chunk_size, 'transfer.chunk_size', DEFAULTS.transfer.chunk_size) + if e2 then return nil, e2 end + local timeout_s, e3 = pos_number(raw.timeout_s, 'transfer.timeout_s', DEFAULTS.transfer.timeout_s) + if e3 then return nil, e3 end + + return { + chunk_size = chunk_size, + timeout_s = timeout_s, + }, nil +end + +local function compile_queues(raw) + raw = raw or {} + if type(raw) ~= 'table' then return nil, 'queues must be a table' end + local ok, err = allowed(raw, QUEUE_KEYS, 'queues') + if not ok then return nil, err end + + local out = {} + for k, default in pairs(DEFAULTS.queues) do + local v, verr = pos_int(raw[k], 'queues.' .. k, default) + if verr then return nil, verr end + out[k] = v + end + return out, nil +end + +local function compile_link(raw, service_local_node) + if type(raw) ~= 'table' then return nil, 'link must be a table' end + local ok, err = allowed(raw, LINK_KEYS, 'link') + if not ok then return nil, err end + + local link_id, e1 = str(raw.id, 'link.id') + if e1 then return nil, e1 end + local peer_id, e2 = str(raw.peer_id, 'link.peer_id') + if e2 then return nil, e2 end + + local transport, e4 = compile_transport(raw.transport, link_id) + if e4 then return nil, e4 end + local session, e5 = compile_session(raw.session, service_local_node) + if e5 then return nil, e5 end + local reader, e6 = compile_reader(raw.reader) + if e6 then return nil, e6 end + local writer, e7 = compile_writer(raw.writer) + if e7 then return nil, e7 end + local bridge, e8 = compile_bridge(raw.bridge) + if e8 then return nil, e8 end + local transfer, e9 = compile_transfer(raw.transfer) + if e9 then return nil, e9 end + local queues, e10 = compile_queues(raw.queues) + if e10 then return nil, e10 end + + return { + link_id = link_id, + peer_id = peer_id, + transport = transport, + session = session, + reader = reader, + writer = writer, + bridge = bridge, + transfer = transfer, + queues = queues, + }, nil +end + +------------------------------------------------------------------------------- +-- Public compiler +------------------------------------------------------------------------------- + +function M.compile(raw) + if type(raw) ~= 'table' then return fail('fabric config must be a table') end + local ok, err = allowed(raw, ROOT_KEYS, 'fabric config') + if not ok then return fail(err) end + if raw.schema ~= SCHEMA then + return fail('fabric config schema must be ' .. SCHEMA) + end + + local local_node, nerr = opt_str(raw.local_node, 'fabric.local_node') + if nerr then return fail(nerr) end + + local links_in, lerr = list_or_empty(raw.links, 'fabric.links') + if not links_in then return fail(lerr) end + + local compiled = { + service = { + schema = raw.schema, + local_node = local_node, + }, + links = {}, + routing = { + by_link_id = {}, + by_peer_id = {}, + }, + } + + for i = 1, #links_in do + local link, cerr = compile_link(links_in[i], local_node) + if not link then return fail('links[' .. tostring(i) .. ']: ' .. tostring(cerr)) end + + if compiled.routing.by_link_id[link.link_id] ~= nil then + return fail('duplicate link id: ' .. link.link_id) + end + compiled.links[#compiled.links + 1] = link + compiled.routing.by_link_id[link.link_id] = link + append_index(compiled.routing.by_peer_id, link.peer_id, link) + end + + return seal(compiled), nil +end + +M.DEFAULTS = seal(copy_plain(DEFAULTS)) +M.SCHEMA = SCHEMA + +return M diff --git a/src/services/fabric/dependencies.lua b/src/services/fabric/dependencies.lua new file mode 100644 index 00000000..e4dc79dd --- /dev/null +++ b/src/services/fabric/dependencies.lua @@ -0,0 +1,64 @@ +-- services/fabric/dependencies.lua +-- Pure helpers for deriving Fabric capability dependencies from compiled config. + +local M = {} + +local function non_empty(v) + return type(v) == 'string' and v ~= '' +end + +function M.transport_dependency_key(link_id) + return 'transport:' .. tostring(link_id) +end + +local function bypassed_by_override(override) + return type(override) == 'table' and ( + override.open_transport_op ~= nil + or override.transport_session ~= nil + or override.local_rx ~= nil + ) +end + +function M.transport_dependency_for_link(link, override) + if type(link) ~= 'table' then return nil end + if bypassed_by_override(override) then return nil end + if link.open_transport_op ~= nil or link.transport_session ~= nil or link.local_rx ~= nil then return nil end + local t = link.transport + if type(t) ~= 'table' then return nil end + if not (non_empty(t.source) and non_empty(t.class) and non_empty(t.id)) then return nil end + return { + key = link.dependency_key or M.transport_dependency_key(link.link_id or link.id), + raw_kind = 'host', + source = t.source, + class = t.class, + id = t.id, + required = true, + link_id = link.link_id or link.id, + } +end + +function M.assign_transport_dependency(link, override) + local dep = M.transport_dependency_for_link(link, override) + if dep == nil then return link, nil end + link.dependency_key = dep.key + if type(link.transport) == 'table' then + local t = {} + for k, v in pairs(link.transport) do t[k] = v end + t.dependency_key = dep.key + link.transport = t + end + return link, dep +end + +function M.transport_dependencies(compiled, overrides) + local out = {} + for i, link in ipairs((compiled and compiled.links) or {}) do + local link_id = link.link_id or link.id + local override = type(overrides) == 'table' and (overrides[link_id] or overrides[i]) or nil + local spec = M.transport_dependency_for_link(link, override) + if spec ~= nil then out[#out + 1] = spec end + end + return out +end + +return M diff --git a/src/services/fabric/hal_transport.lua b/src/services/fabric/hal_transport.lua new file mode 100644 index 00000000..e4148965 --- /dev/null +++ b/src/services/fabric/hal_transport.lua @@ -0,0 +1,381 @@ +-- services/fabric/hal_transport.lua +-- +-- HAL-backed Fabric JSONL transport adapter. +-- +-- Core Fabric consumes frame transports. This module adapts line/stream HAL +-- sessions to the fabric-jsonl/1 frame transport contract, and can open such a +-- session through a raw host HAL capability. + +local fibers = require 'fibers' +local op = require 'fibers.op' + +local protocol = require 'services.fabric.protocol' +local resource = require 'devicecode.support.resource' +local cap_sdk = require 'services.hal.sdk.cap' +local cap_args = require 'services.hal.types.capability_args' +local dep_failure = require 'devicecode.support.dependency_failure' + +local M = {} + +-------------------------------------------------------------------------------- +-- JSONL transport wrapper +-------------------------------------------------------------------------------- + +local Transport = {} +Transport.__index = Transport + +local function has_terminate_contract(x) + return type(x) == 'table' and type(x.terminate) == 'function' +end + +local function has_line_contract(x) + return type(x) == 'table' + and type(x.read_line_op) == 'function' + and type(x.write_line_op) == 'function' + and has_terminate_contract(x) +end + +local function has_stream_contract(x) + return type(x) == 'table' + and type(x.read_line_op) == 'function' + and type(x.write_op) == 'function' + and has_terminate_contract(x) +end + +function M.is_transport_session(x) + return has_line_contract(x) or has_stream_contract(x) +end + +local function normalise_bool(ok, err, fallback) + if ok == true then return true, nil end + return nil, err or fallback +end + +local function write_bytes_result(n, err) + if n == nil then + return nil, err or 'write_failed' + end + return true, nil +end + +--- Wrap a raw HAL line/stream session as a fabric-jsonl/1 frame transport. +--- +--- Accepted raw shapes: +--- * read_line_op / write_line_op +--- * read_line_op / write_op +function M.wrap_transport(session, opts) + opts = opts or {} + + if type(session) ~= 'table' then + return nil, 'transport_session_is_not_table' + end + + local mode + if has_line_contract(session) then + mode = 'line' + elseif has_stream_contract(session) then + mode = 'stream' + else + return nil, 'transport_session_is_not_jsonl_line_like' + end + + local terminator = opts.terminator + if terminator == nil then terminator = '\n' end + if type(terminator) ~= 'string' then + return nil, 'invalid_transport_terminator' + end + + return setmetatable({ + _session = session, + _mode = mode, + _terminator = terminator, + _closed = false, + }, Transport), nil +end + +M.wrap = M.wrap_transport + +function Transport:_take_session() + local session = self._session + self._session = nil + return session +end + +function Transport:_active_session_op() + if self._closed or self._session == nil then + return nil, op.always(nil, 'closed') + end + return self._session, nil +end + +function Transport:read_line_op() + return op.guard(function () + local session, closed = self:_active_session_op() + if session == nil then return closed end + + return session:read_line_op({ + terminator = self._terminator, + keep_terminator = false, + }):wrap(function (line, err) + if line ~= nil then return line, nil end + return nil, err or 'closed' + end) + end) +end + +function Transport:write_line_op(line) + return op.guard(function () + local session, closed = self:_active_session_op() + if session == nil then return closed end + + if type(line) ~= 'string' then + return op.always(nil, 'line_must_be_string') + end + + if self._mode == 'line' then + return session:write_line_op(line):wrap(function (ok, err) + return normalise_bool(ok, err, 'write_failed') + end) + end + + if self._terminator ~= '' then + line = line .. self._terminator + end + + return session:write_op(line):wrap(write_bytes_result) + end) +end + +function Transport:read_frame_op() + return self:read_line_op():wrap(function (line, err) + if line == nil then + return nil, err + end + return protocol.decode_line(line) + end) +end + +function Transport:write_frame_op(frame) + return op.guard(function () + local session, closed = self:_active_session_op() + if session == nil then return closed end + + local checked, verr = protocol.validate(frame) + if checked == nil then + return op.always(nil, verr) + end + + local line, enc_err = protocol.encode_line(checked) + if line == nil then + return op.always(nil, enc_err) + end + + return self:write_line_op(line) + end) +end + +function Transport:flush_op() + return op.guard(function () + local session, closed = self:_active_session_op() + if session == nil then + return op.always(true, nil) + end + + if type(session.flush_op) ~= 'function' then + return op.always(true, nil) + end + + return session:flush_op():wrap(function (ok, err) + return normalise_bool(ok, err, 'flush_failed') + end) + end) +end + +function Transport:terminate(reason) + if self._closed then + return true, nil + end + + self._closed = true + return resource.terminate(self:_take_session(), reason) +end + +function Transport:close_op(reason) + return op.guard(function () + if self._closed then + return op.always(true, nil) + end + + self._closed = true + local session = self:_take_session() + + if session ~= nil and type(session.close_op) == 'function' then + return session:close_op(reason):wrap(function (ok, err) + return normalise_bool(ok, err, 'close_failed') + end) + end + + return op.always(resource.terminate(session, reason)) + end) +end + +M.Transport = Transport + +-------------------------------------------------------------------------------- +-- HAL open adapter +-------------------------------------------------------------------------------- + +local function require_transport_cfg(cfg, level) + if type(cfg) ~= 'table' then + error('fabric.hal_transport.open_transport_op: transport config table required', (level or 1) + 1) + end + + for _, field in ipairs({ 'source', 'class', 'id' }) do + if type(cfg[field]) ~= 'string' or cfg[field] == '' then + error('fabric.hal_transport.open_transport_op: transport.' .. field .. ' must be a non-empty string', (level or 1) + 1) + end + end + + return cfg +end + +local function transport_open_error(cfg, err, detail) + local e = { + err = err or 'transport_open_failed', + detail = detail or err, + reason = detail or err or 'transport_open_failed', + } + if type(cfg) == 'table' then + e.dependency_key = cfg.dependency_key + e.source = cfg.source + e.class = cfg.class + e.id = cfg.id + end + local failure = dep_failure.from_no_route(e.dependency_key, e, { + source = e.source, + class = e.class, + id = e.id, + }) + return failure or e +end +local function reason_text(reason, fallback) + if type(reason) == 'table' then + return reason.err or reason.detail or reason.reason or fallback or 'transport_open_failed' + end + return reason or fallback or 'transport_open_failed' +end + +local function open_opts_for_transport(transport_cfg) + local opts = transport_cfg.open_opts + if transport_cfg.class == 'uart' then + if opts == nil or getmetatable(opts) ~= cap_args.UARTOpenOpts then + local open_opts, err = cap_args.new.UARTOpenOpts(opts) + if not open_opts then + return nil, transport_open_error(transport_cfg, err or 'invalid_uart_open_opts', err) + end + return open_opts, nil + end + end + + return opts or {}, nil +end + +local function unwrap_open_transport_reply(transport_cfg, reply, err) + -- Backwards-compatible public helper: old callers passed (reply, err). + -- New internal callers pass (transport_cfg, reply, err) so structured failures + -- can carry dependency_key/source/class/id. + if type(transport_cfg) == 'table' + and reply ~= nil + and (transport_cfg.ok ~= nil or transport_cfg.reason ~= nil or transport_cfg.err ~= nil) + and transport_cfg.source == nil + and transport_cfg.class == nil + and transport_cfg.dependency_key == nil + then + err = reply + reply = transport_cfg + transport_cfg = nil + elseif transport_cfg == nil and reply ~= nil and err == nil then + -- Old nil-reply call shape: unwrap_open_transport_reply(nil, err). + err = reply + reply = nil + end + + if reply == nil then + return nil, transport_open_error(transport_cfg, err or 'transport_open_failed', err) + end + + if type(reply) ~= 'table' then + return nil, transport_open_error(transport_cfg, 'transport_open_reply_must_be_table', reply) + end + + if reply.ok ~= true then + local reason = reply.reason or reply.err or err or 'transport_open_failed' + return nil, transport_open_error(transport_cfg, reason_text(reason), reason) + end + + if type(reply.reason) ~= 'table' then + return nil, transport_open_error(transport_cfg, 'transport_open_reply_reason_must_be_table', reply.reason) + end + + local session = reply.reason.session + if session == nil then + return nil, transport_open_error(transport_cfg, 'transport_open_reply_missing_session') + end + + return session, nil +end + +function M.open_transport_op(conn, transport_cfg, transport_session) + transport_cfg = require_transport_cfg(transport_cfg, 2) + + return op.guard(function () + if transport_session ~= nil then + local transport, terr = M.wrap_transport(transport_session, transport_cfg) + if not transport then + return op.always(nil, terr or 'transport_wrap_failed') + end + return op.always(transport, nil) + end + + if conn == nil then + return op.always(nil, 'transport_open_requires_bus_connection') + end + + local cap = cap_sdk.new_raw_host_cap_ref( + conn, + transport_cfg.source, + transport_cfg.class, + transport_cfg.id + ) + + local open_opts, opts_err = open_opts_for_transport(transport_cfg) + if not open_opts then + return op.always(nil, opts_err) + end + + return cap:call_control_op( + transport_cfg.open_verb or 'open', + open_opts + ):wrap(function (reply, err) + local session, uerr = unwrap_open_transport_reply(transport_cfg, reply, err) + if not session then + return nil, uerr + end + + local transport, terr = M.wrap_transport(session, transport_cfg) + if not transport then + return nil, terr or 'transport_wrap_failed' + end + + return transport, nil + end) + end) +end + +function M.open_transport(conn, transport_cfg, transport_session) + return fibers.perform(M.open_transport_op(conn, transport_cfg, transport_session)) +end + +M.unwrap_open_transport_reply = unwrap_open_transport_reply + +return M diff --git a/src/services/fabric/io.lua b/src/services/fabric/io.lua new file mode 100644 index 00000000..2e3a343b --- /dev/null +++ b/src/services/fabric/io.lua @@ -0,0 +1,387 @@ +-- services/fabric/io.lua +-- +-- Fabric frame reader and lane writer owners. +-- +-- I/O owns only: +-- * reading already-decoded Fabric frames from a transport/HAL adapter +-- * writing already-validated Fabric frames to a transport/HAL adapter +-- * writer-lane selection for control/rpc/bulk outbound queues +-- +-- Transport wrapping, JSONL line handling and HAL opening belong outside +-- this module. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local protocol = require 'services.fabric.protocol' +local queue = require 'devicecode.support.queue' +local priority_event = require 'devicecode.support.priority_event' +local contracts = require 'devicecode.support.contracts' +local validate = require 'shared.validate' +local trace = require 'services.fabric.trace' + +local M = {} + +-------------------------------------------------------------------------------- +-- Checks and small helpers +-------------------------------------------------------------------------------- + +local function require_table(v, name, level) + return validate.table(v, name, (level or 1) + 1) +end + +local function require_function(v, name, level) + return validate.function_(v, name, (level or 1) + 1) +end + +local function require_rx(v, name, level) + return contracts.require_rx(v, name, (level or 1) + 1) +end + +local function require_tx(v, name, level) + return contracts.require_tx(v, name, (level or 1) + 1) +end + +local function reason_of(rx_or_tx, fallback) + if type(rx_or_tx) == 'table' and type(rx_or_tx.why) == 'function' then + return rx_or_tx:why() or fallback + end + return fallback +end + +local function perform_send(tx, value) + local ok, err = fibers.perform(tx:send_op(value)) + + if ok == true then return true, nil end + if ok == false then return false, err or 'full' end + + return nil, reason_of(tx, err or 'closed') +end + +local function perform_write(write_frame_op, frame) + local ok, err = fibers.perform(write_frame_op(frame)) + if ok == true then return true, nil end + return nil, err or 'write_failed' +end + +local function perform_flush(flush_op) + if flush_op == nil then return true, nil end + + local ok, err = fibers.perform(flush_op()) + if ok == true then return true, nil end + return nil, err or 'flush_failed' +end + +local function positive_int(v, fallback, name) + if v == nil then return fallback end + if type(v) ~= 'number' or v <= 0 or v % 1 ~= 0 then + error(name .. ' must be a positive integer', 3) + end + return v +end + +local function clean_read_end(frame, err) + return frame == nil and (err == nil or err == 'eof' or err == 'closed') +end + +local function reader_result(frames_read, wire_errors, reason) + return { + role = 'reader', + frames_read = frames_read, + wire_errors = wire_errors, + reason = reason, + } +end + +local function frame_event(frame) + return { + kind = 'frame_received', + frame = frame, + } +end + +local function wire_error_event(err) + return { + kind = 'wire_error', + err = err, + at = fibers.now(), + } +end + +local function send_item_frame(item) + if type(item) ~= 'table' or item.kind ~= 'send_frame' then + error('writer expected send_frame item', 0) + end + + local frame = item.frame + if frame == nil then + error('writer send_frame item has nil frame', 0) + end + + return frame +end + +-------------------------------------------------------------------------------- +-- Reader owner +-------------------------------------------------------------------------------- + +function M.run_reader(scope, params) + require_table(scope, 'fabric.io.run_reader: scope', 2) + params = require_table(params, 'fabric.io.run_reader: params table', 2) + + local read_frame_op = require_function(params.read_frame_op, 'run_reader: read_frame_op', 2) + local downstream_tx = require_tx(params.downstream_tx, 'run_reader: downstream_tx', 2) + local trace_base = { + component = params.component_name or 'reader', + link_id = params.link_id, + link_generation = params.link_generation, + } + local state_tx = params.state_tx + + local frames_read = 0 + local wire_errors = 0 + + while true do + local frame, read_err = fibers.perform(read_frame_op()) + + if clean_read_end(frame, read_err) then + return reader_result(frames_read, wire_errors, read_err or 'eof') + end + + local ev + local label + + if frame ~= nil then + trace.frame(state_tx, trace_base, 'rx', frame) + ev = frame_event(frame) + label = 'frame' + + elseif protocol.is_wire_protocol_error + and protocol.is_wire_protocol_error(read_err) + then + trace.error(state_tx, trace_base, 'rx', read_err) + ev = wire_error_event(read_err) + label = 'wire error' + + else + error('reader read failed: ' .. tostring(read_err or 'unknown'), 0) + end + + local ok, send_err = perform_send(downstream_tx, ev) + + if ok == true then + if frame ~= nil then + frames_read = frames_read + 1 + else + wire_errors = wire_errors + 1 + end + + elseif ok == nil then + return reader_result(frames_read, wire_errors, send_err or 'downstream_closed') + + else + error('reader downstream rejected ' .. label .. ': ' .. tostring(send_err or 'full'), 0) + end + end +end + +-------------------------------------------------------------------------------- +-- Lane-aware writer owner +-------------------------------------------------------------------------------- + +local ORDER_RPC = { 'control', 'rpc', 'bulk' } +local ORDER_BULK = { 'control', 'bulk', 'rpc' } + +local function lane_order(turn) + if turn == 'bulk' then return ORDER_BULK end + return ORDER_RPC +end + +local function lane_counts(counts) + return { + control = counts.control or 0, + rpc = counts.rpc or 0, + bulk = counts.bulk or 0, + } +end + +local function mark_closed(state, lane, reason) + if state.closed[lane] then return end + state.closed[lane] = reason or 'closed' + state.open = state.open - 1 +end + +local function receive_now(state) + for _, lane in ipairs(lane_order(state.turn)) do + if not state.closed[lane] then + local pending = state.pending[lane] + if pending ~= nil then + state.pending[lane] = nil + return lane, pending + end + + local rx = state.rxs[lane] + local item, err = queue.try_recv_now(rx) + + if item ~= nil then + return lane, item + end + + if err ~= 'not_ready' then + mark_closed(state, lane, reason_of(rx, err or 'closed')) + end + end + end + + return nil, nil +end + +local function receive_blocking_op(state) + return op.guard(function () + local arms = {} + + for lane, rx in pairs(state.rxs) do + if not state.closed[lane] then + arms[lane] = rx:recv_op():wrap(function (item) + return lane, item + end) + end + end + + if next(arms) == nil then + return op.always(nil, nil) + end + + return fibers.named_choice(arms):wrap(function (_, lane, item) + return lane, item + end) + end) +end + +local function commit_turn(state, lane) + if lane == 'control' then return end + + if state.turn ~= lane then + state.turn = lane + state.quota_left = state.quotas[lane] + end + + state.quota_left = state.quota_left - 1 + + if state.quota_left <= 0 then + state.turn = (lane == 'rpc') and 'bulk' or 'rpc' + state.quota_left = state.quotas[state.turn] + end +end + +local function next_writer_item_op(state) + return priority_event.next_op { + label = 'fabric.io.lane_writer', + allow_no_event = true, + + select_now = function () + local lane, item = receive_now(state) + if lane == nil then return nil end + return { lane = lane, item = item } + end, + + wait_op = function () + return receive_blocking_op(state):wrap(function (lane, item) + return { lane = lane, item = item } + end) + end, + + store_wake = function (wake) + local lane = wake and wake.lane + if lane == nil then return end + + if wake.item == nil then + mark_closed(state, lane, reason_of(state.rxs[lane], 'closed')) + else + state.pending[lane] = wake.item + end + end, + } +end + +function M.run_lane_writer(scope, params) + require_table(scope, 'fabric.io.run_lane_writer: scope', 2) + params = require_table(params, 'fabric.io.run_lane_writer: params table', 2) + + local write_frame_op = require_function(params.write_frame_op, 'run_lane_writer: write_frame_op', 2) + local trace_base = { + component = params.component_name or 'writer', + link_id = params.link_id, + link_generation = params.link_generation, + } + local state_tx = params.state_tx + + local flush_op = params.flush_op + if flush_op ~= nil then + require_function(flush_op, 'run_lane_writer: flush_op', 2) + end + + local state = { + rxs = { + control = require_rx(params.control_rx, 'run_lane_writer: control_rx', 2), + rpc = require_rx(params.rpc_rx, 'run_lane_writer: rpc_rx', 2), + bulk = require_rx(params.bulk_rx, 'run_lane_writer: bulk_rx', 2), + }, + + closed = {}, + open = 3, + turn = params.initial_turn or 'rpc', + pending = {}, + quotas = { + rpc = positive_int(params.rpc_quota, 4, 'run_lane_writer: rpc_quota'), + bulk = positive_int(params.bulk_quota, 1, 'run_lane_writer: bulk_quota'), + }, + quota_left = nil, + } + + state.quota_left = state.quotas[state.turn] or state.quotas.rpc + + local flush_each = not not params.flush_each + local written = 0 + local by_lane = { control = 0, rpc = 0, bulk = 0 } + + while state.open > 0 do + local selected = fibers.perform(next_writer_item_op(state)) + + if selected ~= nil and selected.item ~= nil then + local lane = selected.lane + local frame = send_item_frame(selected.item) + trace.frame(state_tx, trace_base, 'tx', frame, { lane = lane }) + + local ok, err = perform_write(write_frame_op, frame) + if ok ~= true then + trace.error(state_tx, trace_base, 'tx', err, { event = 'write_failed', lane = lane, frame_type = type(frame) == 'table' and frame.type or nil }) + error('writer write failed: ' .. tostring(err), 0) + end + + written = written + 1 + by_lane[lane] = (by_lane[lane] or 0) + 1 + commit_turn(state, lane) + + if flush_each then + local flushed, flush_err = perform_flush(flush_op) + if flushed ~= true then + error('writer flush failed: ' .. tostring(flush_err), 0) + end + end + end + end + + local flushed, flush_err = perform_flush(flush_op) + if flushed ~= true then + error('writer flush failed: ' .. tostring(flush_err), 0) + end + + return { + role = 'writer', + frames_written = written, + reason = 'closed', + lanes = lane_counts(by_lane), + } +end + +return M diff --git a/src/services/fabric/link.lua b/src/services/fabric/link.lua new file mode 100644 index 00000000..cbb9750a --- /dev/null +++ b/src/services/fabric/link.lua @@ -0,0 +1,937 @@ +-- services/fabric/link.lua +-- +-- Link-scope supervisor and standard composition. +-- +-- A Fabric link owns component lifetimes and link-local policy: +-- +-- * a link scope owns component lifetimes +-- * components are started as scoped work +-- * component completion is reported as data +-- * the link coordinator decides policy +-- * the coordinator has one suspending control point +-- * model updates are non-yielding +-- +-- Protocol, transport, bridge, and transfer behaviour remain in their named +-- semantic owners; this module composes and supervises them. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local service_events = require 'devicecode.support.service_events' +local model_mod = require 'services.fabric.model' +local io_mod = require 'services.fabric.io' +local bridge_mod = require 'services.fabric.bridge' +local bus_adapter = require 'services.fabric.bus_adapter' +local session_mod = require 'services.fabric.session' +local transfer_mod = require 'services.fabric.transfer' +local state_mod = require 'services.fabric.state' +local resource = require 'devicecode.support.resource' +local tablex = require 'shared.table' + +local M = {} + +local Link = {} +Link.__index = Link + +local DEFAULT_DONE_QUEUE = 32 + +local shallow_copy = tablex.shallow_copy + +local function copy_component_entry(c) + local out = shallow_copy(c) + + if type(out.result) == 'table' then + out.result = shallow_copy(out.result) + end + + return out +end + +local function copy_components(components) + local out = {} + + for name, c in pairs(components or {}) do + out[name] = copy_component_entry(c) + end + + return out +end + +local function copy_snapshot(s) + local out = shallow_copy(s or {}) + out.components = copy_components(out.components) + return out +end + +local function shallow_value_equal(a, b) + if a == b then return true end + if type(a) ~= 'table' or type(b) ~= 'table' then return false end + + for k, v in pairs(a) do + if b[k] ~= v then + return false + end + end + + for k in pairs(b) do + if a[k] == nil then + return false + end + end + + return true +end + +local function snapshots_equal(a, b) + if a == b then return true end + if type(a) ~= 'table' or type(b) ~= 'table' then return false end + + if a.link_id ~= b.link_id then return false end + if a.link_generation ~= b.link_generation then return false end + if a.state ~= b.state then return false end + if a.completed ~= b.completed then return false end + if a.total ~= b.total then return false end + if a.reason ~= b.reason then return false end + + local ac = a.components or {} + local bc = b.components or {} + + for name, av in pairs(ac) do + local bv = bc[name] + if type(bv) ~= 'table' then return false end + + if av.status ~= bv.status then return false end + if av.primary ~= bv.primary then return false end + if not shallow_value_equal(av.result, bv.result) then return false end + end + + for name in pairs(bc) do + if ac[name] == nil then return false end + end + + return true +end + +local function component_list(params) + local list = params.components + if type(list) ~= 'table' then + error('fabric.link: components array required', 3) + end + + local out = {} + + for _, c in ipairs(list) do + if type(c) ~= 'table' then + error('fabric.link: component entry must be a table', 3) + end + + if type(c.name) ~= 'string' or c.name == '' then + error('fabric.link: component.name must be a non-empty string', 3) + end + + if type(c.run) ~= 'function' then + error('fabric.link: component.run must be a function', 3) + end + + if out[c.name] ~= nil then + error('fabric.link: duplicate component name: ' .. c.name, 3) + end + + out[#out + 1] = c + out[c.name] = c + end + + if #out == 0 then + error('fabric.link: at least one component required', 3) + end + + return out +end + +local function initial_snapshot(link_id, link_generation, components) + local cs = {} + + for _, c in ipairs(components) do + cs[c.name] = { + status = 'starting', + } + end + + return { + link_id = link_id, + link_generation = link_generation, + state = 'starting', + total = #components, + completed = 0, + components = cs, + } +end + +local function public_snapshot(self) + return self._model:snapshot() +end + +local function publish_state(self) + return state_mod.admit_link_snapshot_now( + self._state_tx, + self._link_id, + self._link_generation, + public_snapshot(self), + 'fabric_link_state_admit_failed' + ) +end + +local function publish_state_checked(self) + local ok, err = publish_state(self) + if ok ~= true then + error(err or 'fabric_link_state_admit_failed', 0) + end + return true +end + +local function set_state(self, state, reason) + local changed = self._model:update(function (s) + s.state = state + s.reason = reason + end) + + if changed then + publish_state_checked(self) + end +end + +local function advance_completion_state(s) + if s.state == 'failed' or s.state == 'cancelling' then + return + end + + if s.completed >= s.total then + s.state = 'completed' + else + s.state = 'running' + end +end + +local function record_component_done(self, ev) + local accepted = false + + local changed = self._model:update(function (s) + local c = s.components[ev.component] + + if not c then + return + end + + if c.status == 'ok' or c.status == 'failed' or c.status == 'cancelled' then + return + end + + c.status = ev.status + + if ev.status == 'ok' then + if type(ev.result) == 'table' then + c.result = shallow_copy(ev.result) + else + c.result = ev.result + end + else + c.primary = ev.primary + end + + s.completed = (s.completed or 0) + 1 + advance_completion_state(s) + accepted = true + end) + + if changed then + publish_state_checked(self) + end + + return accepted +end + +local function default_policy(_, ev) + if ev.kind ~= 'component_done' or ev.status == 'ok' then + return { action = 'continue' } + end + + if ev.status == 'failed' then + return { + action = 'fail', + reason = ('component %s failed: %s'):format( + tostring(ev.component), + tostring(ev.primary or 'failed') + ), + } + end + + if ev.status == 'cancelled' then + return { + action = 'fail', + reason = ('component %s cancelled unexpectedly: %s'):format( + tostring(ev.component), + tostring(ev.primary or 'cancelled') + ), + } + end + + return { + action = 'fail', + reason = ('component %s ended with invalid status: %s'):format( + tostring(ev.component), + tostring(ev.status) + ), + } +end + +local function apply_policy(self, ev) + local decision = (self._policy or default_policy)(self, ev) or { action = 'continue' } + local action = decision.action or 'continue' + + if action == 'continue' then + return + end + + if action == 'fail' then + local reason = decision.reason or 'link policy failure' + set_state(self, 'failed', reason) + error(reason, 0) + end + + if action == 'cancel' then + local reason = decision.reason or 'link policy cancellation' + set_state(self, 'cancelling', reason) + + for _, h in pairs(self._components) do + if h and h.cancel then + h:cancel(reason) + end + end + + return + end + + error('fabric.link: unknown policy action: ' .. tostring(action), 0) +end + +local function make_component_identity(self, component) + return { + kind = 'component_done', + link_id = self._link_id, + link_generation = self._link_generation, + component = component.name, + } +end + +local function make_link_caps(self, component) + return { + link_id = self._link_id, + link_generation = self._link_generation, + component = component and component.name or nil, + + snapshot = public_snapshot(self), + } +end + +local function start_component(self, component) + local handle, err = scoped_work.start { + lifetime_scope = self._scope, + reaper_scope = self._scope, + report_scope = self._scope, + + identity = make_component_identity(self, component), + + run = function (component_scope) + return component.run(component_scope, make_link_caps(self, component)) + end, + + report = service_events.reporter_for( + self._done_tx, + make_component_identity(self, component), + { label = 'link_component_completion_report_failed' } + ), + } + + if not handle then + return nil, err + end + + self._components[component.name] = handle + return handle, nil +end + +local function start_all_components(self, components) + for _, component in ipairs(components) do + local h, err = start_component(self, component) + + if not h then + set_state(self, 'failed', err or 'component_start_failed') + error(err or 'component_start_failed', 0) + end + end + + set_state(self, 'running') +end + +local function close_state_tx(self, reason) + if not self._state_tx_owned or self._state_tx_closed then + return true, nil + end + + self._state_tx_closed = true + if self._state_tx and type(self._state_tx.close) == 'function' then + self._state_tx:close(reason or 'fabric link state projector closed') + end + return true, nil +end + +local function start_state_projector(self, params) + local rx = params.state_projector_rx + if rx == nil then + self._state_projector_done = true + return true, nil + end + + local handle, err = scoped_work.start { + lifetime_scope = self._scope, + reaper_scope = self._scope, + report_scope = self._scope, + + identity = { + kind = 'state_projector_done', + link_id = self._link_id, + link_generation = self._link_generation, + }, + + run = function (projector_scope) + return state_mod.run_projector(projector_scope, { + conn = params.state_projector_conn, + state_rx = rx, + svc = params.state_projector_svc, + }) + end, + + report = service_events.reporter_for( + self._done_tx, + { + kind = 'state_projector_done', + link_id = self._link_id, + link_generation = self._link_generation, + source = 'fabric_link_state_projector', + source_id = self._link_id, + }, + { label = 'link_state_projector_completion_report_failed' } + ), + } + + if not handle then + return nil, err or 'state_projector_start_failed' + end + + self._state_projector_handle = handle + self._state_projector_done = false + return true, nil +end + +local function components_complete(self) + local snap = public_snapshot(self) + return (snap.completed or 0) >= (snap.total or 0) +end + +local function maybe_close_state_projector(self) + if components_complete(self) then + close_state_tx(self, 'fabric link components completed') + end +end + +local function handle_state_projector_done(self, ev) + if ev.link_id ~= self._link_id or ev.link_generation ~= self._link_generation then + return + end + + self._state_projector_done = true + self._state_projector_handle = nil + + if ev.status ~= 'ok' then + local reason = ev.primary or ev.status or 'state_projector_failed' + set_state(self, 'failed', reason) + error(reason, 0) + end +end + +local function next_event_op(self) + return self._done_rx:recv_op():wrap(function (ev) + if ev == nil then + return { + kind = 'done_queue_closed', + } + end + + return ev + end) +end + +local function coordinator_loop(self) + while not self._complete do + local ev = fibers.perform(next_event_op(self)) + + -- Own-scope cancellation is not a service event. Scope-aware perform + -- exits through the scope machinery before this branch is reached. + + if ev.kind == 'done_queue_closed' then + error('link completion queue closed', 0) + end + + if ev.kind == 'component_done' then + if ev.link_id ~= self._link_id or ev.link_generation ~= self._link_generation then + -- Stale or foreign completion. Ignore it. + else + local accepted = record_component_done(self, ev) + + if accepted then + apply_policy(self, ev) + end + end + elseif ev.kind == 'state_projector_done' then + handle_state_projector_done(self, ev) + else + error('fabric.link: unknown event kind: ' .. tostring(ev.kind), 0) + end + + maybe_close_state_projector(self) + if components_complete(self) and self._state_projector_done then + self._complete = true + end + end + + return { + link_id = self._link_id, + link_generation = self._link_generation, + snapshot = public_snapshot(self), + } +end + +-------------------------------------------------------------------------------- +-- Composed link wiring +-------------------------------------------------------------------------------- + +local DEFAULT_FRAME_QUEUE = 32 +local DEFAULT_OUTBOUND_QUEUE = 32 + +local function qlen(params, name, default) + local queues = params.queues or {} + if type(queues) ~= 'table' then + error('fabric.link.run_composed: queues must be a table', 3) + end + + local n = queues[name] + if n == nil then + return default + end + + if type(n) ~= 'number' or n <= 0 or n % 1 ~= 0 then + error('fabric.link.run_composed: queues.' .. tostring(name) .. ' must be a positive integer', 3) + end + + return n +end + +local function closed_rx(reason) + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + tx:close(reason or 'closed') + return rx +end + +local function has_terminate_contract(x) + return type(x) == 'table' and type(x.terminate) == 'function' +end + +local function is_frame_transport(x) + return type(x) == 'table' + and type(x.read_frame_op) == 'function' + and type(x.write_frame_op) == 'function' + and has_terminate_contract(x) +end + +local function require_transport(scope, transport) + if not is_frame_transport(transport) then + error('fabric.link.run_composed: transport must be a Fabric frame transport', 3) + end + + scope:finally(function (_, status, primary) + resource.terminate_checked( + transport, + primary or status or 'fabric link closed', + 'fabric transport cleanup failed' + ) + end) + + return transport +end + +local function bus_conn_from(params, service_caps) + return params.conn or (service_caps and service_caps.conn) +end + +local function bridge_params_from(params, local_rx, session_rx, outbound, bus_tx, state_tx, service_caps) + local b = shallow_copy(params.bridge or {}) + + b.link_id = params.link_id + b.link_generation = params.link_generation + b.local_rx = local_rx + b.session_rx = session_rx + b.outbound = outbound + b.conn = bus_conn_from(params, service_caps) + b.bus_tx = bus_tx + b.state_tx = state_tx + b.component_name = 'rpc_bridge' + + return b +end + +local function transfer_params_from(params, admission_rx, session_rx, outbound, state_tx) + local t = shallow_copy(params.transfer or {}) + + t.manager_id = t.manager_id or (params.link_id .. ':transfer') + t.link_id = params.link_id + t.link_generation = params.link_generation + t.admission_rx = admission_rx + t.session_rx = session_rx + t.outbound = outbound + t.state_tx = state_tx + t.component_name = 'transfer_manager' + t.receive_targets = t.receive_targets or params.receive_targets + + return t +end + +-- Build the standard Fabric link component set around an already-open frame +-- transport. +-- +-- The composed link has these components: +-- reader transport -> raw inbound frame queue +-- session hello/ack/ping/pong, liveness and semantic frame tagging +-- writer control/rpc/bulk outbound lanes -> transport +-- rpc_bridge local/RPC-frame semantic events -> session outbound gate +-- transfer_manager transfer slot admission and transfer-frame lane -> session outbound gate +-- +-- The reader owns closing the raw inbound frame queue. Session owns the raw +-- outbound writer lanes and exposes only generation-checked admission to bridge +-- and transfer. This lets dependent components finish naturally without the link +-- coordinator joining inline or performing cross-component waits. +function M.composed_components(scope, params, service_caps) + params = params or {} + + params.link_id = params.link_id or 'link' + params.link_generation = params.link_generation or 1 + + local transport = require_transport(scope, params.transport) + + local state_conn = bus_conn_from(params, service_caps) + local state_tx + if state_conn ~= nil then + local projector_tx, projector_rx = state_mod.new_queue(qlen(params, 'state', DEFAULT_OUTBOUND_QUEUE)) + state_tx = projector_tx + params.state_tx = state_tx + params.state_projector_conn = state_conn + params.state_projector_rx = projector_rx + params.state_tx_owned = true + else + state_tx = params.state_tx + end + local read_frame_op = function () + return transport:read_frame_op() + end + local write_frame_op = function (frame) + return transport:write_frame_op(frame) + end + local flush_op + if type(transport.flush_op) == 'function' then + flush_op = function () + return transport:flush_op() + end + end + + local inbound_frame_tx, inbound_frame_rx = mailbox.new( + qlen(params, 'frame_in', DEFAULT_FRAME_QUEUE), + { full = 'reject_newest' } + ) + + local bridge_session_tx, bridge_session_rx = mailbox.new( + qlen(params, 'rpc_in', DEFAULT_FRAME_QUEUE), + { full = 'reject_newest' } + ) + + local transfer_session_tx, transfer_session_rx = mailbox.new( + qlen(params, 'xfer_in', DEFAULT_FRAME_QUEUE), + { full = 'reject_newest' } + ) + + local outbound_control_tx, outbound_control_rx = mailbox.new( + qlen(params, 'tx_control', DEFAULT_OUTBOUND_QUEUE), + { full = 'reject_newest' } + ) + + local outbound_rpc_tx, outbound_rpc_rx = mailbox.new( + qlen(params, 'tx_rpc', DEFAULT_OUTBOUND_QUEUE), + { full = 'reject_newest' } + ) + + local outbound_bulk_tx, outbound_bulk_rx = mailbox.new( + qlen(params, 'tx_bulk', DEFAULT_OUTBOUND_QUEUE), + { full = 'reject_newest' } + ) + + local outbound_gate = session_mod.new_outbound_gate { + tx_control = outbound_control_tx, + tx_rpc = outbound_rpc_tx, + tx_bulk = outbound_bulk_tx, + } + + local local_rx = params.local_rx + if local_rx == nil and bus_conn_from(params, service_caps) == nil then + local_rx = closed_rx('no local fabric events') + end + + local admission_rx = params.transfer_admission_rx or closed_rx('no transfer slot admissions') + + local session_cfg = params.session or {} + local reader_cfg = params.reader or {} + local writer_cfg = params.writer or {} + if type(session_cfg) ~= 'table' then + error('fabric.link.run_composed: session must be a table', 2) + end + if type(reader_cfg) ~= 'table' then + error('fabric.link.run_composed: reader must be a table', 2) + end + if type(writer_cfg) ~= 'table' then + error('fabric.link.run_composed: writer must be a table', 2) + end + + local components = {} + + components[#components + 1] = { + name = 'reader', + run = function (component_scope) + component_scope:finally(function () + inbound_frame_tx:close('reader closed') + end) + + return io_mod.run_reader(component_scope, { + read_frame_op = read_frame_op, + downstream_tx = inbound_frame_tx, + state_tx = state_tx, + link_id = params.link_id, + link_generation = params.link_generation, + component_name = 'reader', + }) + end, + } + + components[#components + 1] = { + name = 'session', + run = function (component_scope) + component_scope:finally(function () + bridge_session_tx:close('session closed') + transfer_session_tx:close('session closed') + end) + + return session_mod.run(component_scope, { + link_id = params.link_id, + link_generation = params.link_generation, + peer_id = params.peer_id, + local_node = session_cfg.local_node, + local_sid = session_cfg.local_sid, + identity_claim = session_cfg.identity_claim, + auth_claim = session_cfg.auth_claim, + frame_rx = inbound_frame_rx, + tx_control = outbound_control_tx, + outbound = outbound_gate, + rpc_tx = bridge_session_tx, + transfer_tx = transfer_session_tx, + hello_interval_s = session_cfg.hello_interval_s, + ping_interval_s = session_cfg.ping_interval_s, + liveness_timeout_s = session_cfg.liveness_timeout_s, + bad_frame_limit = reader_cfg.bad_frame_limit, + bad_frame_window_s = reader_cfg.bad_frame_window_s, + state_tx = state_tx, + component_name = 'session', + }) + end, + } + + components[#components + 1] = { + name = 'writer', + run = function (component_scope) + return io_mod.run_lane_writer(component_scope, { + control_rx = outbound_control_rx, + rpc_rx = outbound_rpc_rx, + bulk_rx = outbound_bulk_rx, + write_frame_op = write_frame_op, + flush_op = flush_op, + flush_each = params.flush_each, + rpc_quota = writer_cfg.rpc_quota, + bulk_quota = writer_cfg.bulk_quota, + state_tx = state_tx, + link_id = params.link_id, + link_generation = params.link_generation, + component_name = 'writer', + }) + end, + } + + components[#components + 1] = { + name = 'rpc_bridge', + run = function (component_scope) + local runtime + local bus_tx + local bridge_local_rx = local_rx + local bridge_conn = bus_conn_from(params, service_caps) + + if bridge_local_rx == nil and bridge_conn ~= nil then + runtime = bus_adapter.local_runtime(component_scope, bridge_conn, params.bridge or params) + bridge_local_rx = runtime.local_rx + bus_tx = runtime.command_tx + end + + local bp = bridge_params_from( + params, + bridge_local_rx, + bridge_session_rx, + outbound_gate, + bus_tx, + state_tx, + service_caps + ) + + local result = bridge_mod.run(component_scope, bp) + if runtime ~= nil then + local ok, err = runtime:terminate('bridge completed') + if ok ~= true then + error(err or 'fabric bus adapter cleanup failed', 0) + end + end + return result + end, + } + + components[#components + 1] = { + name = 'transfer_manager', + run = function (component_scope) + local tp = transfer_params_from( + params, + admission_rx, + transfer_session_rx, + outbound_gate, + state_tx + ) + return transfer_mod.run(component_scope, tp) + end, + } + + return components +end + +--- Run a standard Fabric link composition. +function M.run_composed(scope, params, service_caps) + if type(scope) ~= 'table' then + error('fabric.link.run_composed: scope required', 2) + end + if type(params) ~= 'table' then + error('fabric.link.run_composed: params table required', 2) + end + + local run_params = shallow_copy(params) + run_params.link_id = run_params.link_id or 'link' + run_params.link_generation = run_params.link_generation or 1 + run_params.components = M.composed_components(scope, run_params, service_caps) + + return M.run(scope, run_params) +end + +--- Run a Fabric link inside an already-created link scope. +--- +--- params: +--- link_id?: string +--- link_generation?: integer +--- components: array of { name: string, run: function(scope, link_caps) -> table } +--- policy?: function(link, ev) -> { action = ... } +--- done_queue_len?: integer +--- state_tx?: mailbox sender-like object +--- state_projector_conn?: Connection +--- state_projector_rx?: mailbox receiver-like object +--- state_tx_owned?: boolean +--- +---@param scope Scope +---@param params table +---@return table result +function M.run(scope, params) + if type(scope) ~= 'table' then + error('fabric.link.run: scope required', 2) + end + + if type(params) ~= 'table' then + error('fabric.link.run: params table required', 2) + end + + local components = component_list(params) + + local link_id = params.link_id or 'link' + local link_generation = params.link_generation or 1 + + local initial = initial_snapshot(link_id, link_generation, components) + local model = model_mod.new(initial, { + copy = copy_snapshot, + equals = snapshots_equal, + label = 'fabric.link', + }) + + scope:finally(function (_, status, primary) + model:terminate(primary or status or 'link closed') + end) + + local done_tx, done_rx = mailbox.new( + params.done_queue_len or math.max(DEFAULT_DONE_QUEUE, #components + 4), + { full = 'reject_newest' } + ) + + scope:finally(function () + done_tx:close('link closed') + end) + + local self = setmetatable({ + _scope = scope, + _link_id = link_id, + _link_generation = link_generation, + _model = model, + _state_tx = params.state_tx, + _state_tx_owned = not not params.state_tx_owned, + _state_tx_closed = false, + _state_projector_handle = nil, + _state_projector_done = params.state_projector_rx == nil, + _done_tx = done_tx, + _done_rx = done_rx, + _policy = params.policy, + _components = {}, + _complete = false, + }, Link) + + scope:finally(function (_, status, primary) + close_state_tx(self, primary or status or 'link closed') + end) + + local state_ok, state_err = start_state_projector(self, params) + if state_ok ~= true then + set_state(self, 'failed', state_err or 'state_projector_start_failed') + error(state_err or 'state_projector_start_failed', 0) + end + + publish_state_checked(self) + + start_all_components(self, components) + + return coordinator_loop(self) +end + +return M diff --git a/src/services/fabric/model.lua b/src/services/fabric/model.lua new file mode 100644 index 00000000..411f7ac0 --- /dev/null +++ b/src/services/fabric/model.lua @@ -0,0 +1,45 @@ +-- services/fabric/model.lua +-- +-- Fabric observable models and retained snapshot helpers. +-- +-- The generic observable model mechanics live in devicecode.support.model. This +-- module keeps only Fabric's default shallow copy/equality policy and public +-- service namespace. + +local base_model = require 'devicecode.support.model' +local tablex = require 'shared.table' + +local M = {} + +local function default_copy(v) + if type(v) == 'table' then + return tablex.shallow_copy(v) + end + return v +end + +local function default_equals(a, b) + if rawequal(a, b) then return true end + if type(a) ~= type(b) then return false end + if type(a) ~= 'table' then return a == b end + for k, v in pairs(a) do + if b[k] ~= v then return false end + end + for k in pairs(b) do + if a[k] == nil then return false end + end + return true +end + +function M.new(initial, opts) + opts = opts or {} + return base_model.new(initial, { + copy = opts.copy or default_copy, + equals = opts.equals or default_equals, + label = opts.label or 'fabric.model', + }) +end + +M.Model = base_model.Model + +return M diff --git a/src/services/fabric/protocol.lua b/src/services/fabric/protocol.lua new file mode 100644 index 00000000..ab5d52c3 --- /dev/null +++ b/src/services/fabric/protocol.lua @@ -0,0 +1,781 @@ +-- services/fabric/protocol.lua +-- +-- Pure Fabric wire/semantic protocol helpers. + +local cjson = require 'cjson.safe' +local b64url = require 'shared.encoding.b64url' +local xxhash32 = require 'shared.hash.xxhash32' +local tablex = require 'shared.table' + +local M = {} + +M.PROTO = 'fabric-jsonl/1' +M.DIGEST_ALG = 'xxhash32' +M.DEFAULT_CHUNK_SIZE = 2048 + +function M.proto_supported(proto) + return proto == M.PROTO +end + +function M.is_wire_protocol_error(err) + if err == nil then return false end + + local s = tostring(err) + + return s:match('^decode_failed') ~= nil + or s:match('^encode_failed') ~= nil + or s:match('^invalid_') ~= nil + or s:match('^missing_') ~= nil + or s:match('^unknown_frame_field') ~= nil + or s:match('^unsupported_') ~= nil + or s:match('^chunk_digest_mismatch') ~= nil + or s:match('^non_canonical_base64url') ~= nil + or s:match('^line_must_be_string') ~= nil +end + +-------------------------------------------------------------------------------- +-- Frame vocabulary and validation spec +-------------------------------------------------------------------------------- + +local FRAME_SPECS = { + + hello = { + class = 'session_control', + lane = 'session_control', + fields = { 'type', 'proto', 'sid', 'node', 'identity', 'auth' }, + required = { + { 'proto', 'missing_proto' }, + { 'sid', 'missing_sid' }, + { 'node', 'missing_node' }, + }, + optional_objects = { + identity = true, + auth = true, + }, + }, + + hello_ack = { + class = 'session_control', + lane = 'session_control', + fields = { 'type', 'proto', 'sid', 'node', 'identity', 'auth' }, + required = { + { 'proto', 'missing_proto' }, + { 'sid', 'missing_sid' }, + { 'node', 'missing_node' }, + }, + optional_objects = { + identity = true, + auth = true, + }, + }, + + ping = { + class = 'session_control', + lane = 'session_control', + fields = { 'type', 'sid' }, + required = { + { 'sid', 'missing_sid' }, + }, + }, + + pong = { + class = 'session_control', + lane = 'session_control', + fields = { 'type', 'sid' }, + required = { + { 'sid', 'missing_sid' }, + }, + }, + + pub = { + class = 'rpc', + lane = 'rpc', + fields = { 'type', 'topic', 'retain', 'payload' }, + required = { + { 'topic', 'invalid_topic' }, + { 'retain', 'missing_retain' }, + }, + }, + + unretain = { + class = 'rpc', + lane = 'rpc', + fields = { 'type', 'topic' }, + required = { + { 'topic', 'invalid_topic' }, + }, + }, + + call = { + class = 'rpc', + lane = 'rpc', + fields = { 'type', 'id', 'topic', 'payload' }, + required = { + { 'id', 'missing_id' }, + { 'topic', 'invalid_topic' }, + }, + }, + + reply = { + class = 'rpc', + lane = 'rpc', + fields = { 'type', 'id', 'ok', 'payload', 'err' }, + required = { + { 'id', 'missing_id' }, + { 'ok', 'missing_ok' }, + }, + }, + + xfer_begin = { + class = 'transfer_control', + lane = 'transfer', + fields = { 'type', 'xfer_id', 'target', 'size', 'digest_alg', 'digest', 'meta' }, + required = { + { 'xfer_id', 'missing_xfer_id' }, + { 'target', 'missing_target' }, + { 'size', 'invalid_xfer_size' }, + }, + digest = true, + }, + + xfer_ready = { + class = 'transfer_control', + lane = 'transfer', + fields = { 'type', 'xfer_id' }, + required = { + { 'xfer_id', 'missing_xfer_id' }, + }, + }, + + xfer_need = { + class = 'transfer_control', + lane = 'transfer', + fields = { 'type', 'xfer_id', 'next', 'retry', 'reason', 'counters' }, + required = { + { 'xfer_id', 'missing_xfer_id' }, + { 'next', 'invalid_next' }, + }, + }, + + xfer_chunk = { + class = 'transfer_bulk', + lane = 'transfer', + fields = { 'type', 'xfer_id', 'offset', 'data', 'chunk_digest' }, + required = { + { 'xfer_id', 'missing_xfer_id' }, + { 'offset', 'invalid_offset' }, + { 'data', 'invalid_chunk_data' }, + { 'chunk_digest', 'missing_chunk_digest' }, + }, + semantic = 'chunk', + }, + + xfer_commit = { + class = 'transfer_control', + lane = 'transfer', + fields = { 'type', 'xfer_id', 'size', 'digest_alg', 'digest' }, + required = { + { 'xfer_id', 'missing_xfer_id' }, + { 'size', 'invalid_xfer_size' }, + }, + digest = true, + }, + + xfer_done = { + class = 'transfer_control', + lane = 'transfer', + fields = { 'type', 'xfer_id' }, + required = { + { 'xfer_id', 'missing_xfer_id' }, + }, + }, + + xfer_abort = { + class = 'transfer_control', + lane = 'transfer', + fields = { 'type', 'xfer_id', 'err' }, + required = { + { 'xfer_id', 'missing_xfer_id' }, + }, + }, +} + +local FRAME_TYPES = {} +local FRAME_FIELDS = {} + +for type_name, spec in pairs(FRAME_SPECS) do + FRAME_TYPES[type_name] = true + + local fields = {} + for _, field in ipairs(spec.fields) do + fields[field] = true + end + FRAME_FIELDS[type_name] = fields +end + +function M.known_type(t) + return FRAME_TYPES[t] == true +end + +function M.classify(frame_or_type) + local t = type(frame_or_type) == 'table' + and frame_or_type.type + or frame_or_type + + local spec = FRAME_SPECS[t] + return spec and spec.class or nil +end + +function M.dispatch_lane(frame_or_type) + local t = type(frame_or_type) == 'table' + and frame_or_type.type + or frame_or_type + + local spec = FRAME_SPECS[t] + return spec and spec.lane or nil +end + +-------------------------------------------------------------------------------- +-- Small pure helpers +-------------------------------------------------------------------------------- + +local shallow_copy = tablex.shallow_copy + +function M.copy(frame) + if type(frame) ~= 'table' then + return nil, 'frame_must_be_table' + end + + return shallow_copy(frame), nil +end + +local function is_finite_number(v) + return type(v) == 'number' + and v == v + and v ~= math.huge + and v ~= -math.huge +end + +local function is_nonneg_integer(v) + return is_finite_number(v) + and v >= 0 + and v % 1 == 0 +end + +local function is_nonempty_string(v) + return type(v) == 'string' and v ~= '' +end + +local function scalar_token_ok(v) + if type(v) == 'string' then + return true + end + + if type(v) == 'number' then + return is_finite_number(v) + end + + return false +end + +local function dense_array_len(t) + if type(t) ~= 'table' then + return nil + end + + local n = #t + + for i = 1, n do + if not scalar_token_ok(t[i]) then + return nil + end + end + + for k in pairs(t) do + if type(k) ~= 'number' + or k < 1 + or k % 1 ~= 0 + or k > n + then + return nil + end + end + + return n +end + +function M.validate_topic(topic) + if dense_array_len(topic) == nil then + return nil, 'invalid_topic' + end + + return topic, nil +end + +function M.topic_ok(topic) return dense_array_len(topic) ~= nil end + +local function is_xxhash32_hex(v) + return type(v) == 'string' + and v:match('^[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]$') ~= nil +end + +-------------------------------------------------------------------------------- +-- Digest helpers +-------------------------------------------------------------------------------- + +function M.digest_hex(bytes) + assert(type(bytes) == 'string', 'protocol.digest_hex expects string') + return xxhash32.digest_hex(bytes) +end + +function M.digest_ok(digest) + return is_xxhash32_hex(digest) +end + +function M.verify_digest(bytes, digest) + if type(bytes) ~= 'string' or not is_xxhash32_hex(digest) then + return false + end + + return M.digest_hex(bytes) == digest +end + +function M.chunk_digest(bytes) + return M.digest_hex(bytes) +end + +function M.verify_chunk_digest(bytes, digest) + return M.verify_digest(bytes, digest) +end + +-------------------------------------------------------------------------------- +-- Validation +-------------------------------------------------------------------------------- + +local function validate_frame_header(frame) + if type(frame) ~= 'table' then + return nil, 'invalid_frame_type' + end + + if not is_nonempty_string(frame.type) then + return nil, 'invalid_frame_type' + end + + local spec = FRAME_SPECS[frame.type] + if spec == nil then + return nil, 'invalid_frame_type' + end + + for k in pairs(frame) do + if type(k) ~= 'string' or k == '' then + return nil, 'invalid_frame_type' + end + end + + local allowed = FRAME_FIELDS[frame.type] + for k in pairs(frame) do + if not allowed[k] then + return nil, 'unknown_frame_field: ' .. tostring(k) + end + end + + return spec, nil +end + +local function validate_required_field(frame, field, err) + local v = frame[field] + + if field == 'topic' then + if not M.topic_ok(v) then return nil, err end + return true, nil + end + + if field == 'retain' then + if type(v) ~= 'boolean' then return nil, err end + return true, nil + end + + if field == 'ok' then + if type(v) ~= 'boolean' then return nil, err end + return true, nil + end + + if field == 'size' or field == 'next' or field == 'offset' then + if not is_nonneg_integer(v) then return nil, err end + return true, nil + end + + if field == 'data' then + if type(v) ~= 'string' then return nil, err end + return true, nil + end + + if field == 'chunk_digest' then + if v == nil then return nil, 'missing_chunk_digest' end + if not is_xxhash32_hex(v) then return nil, 'invalid_chunk_digest' end + return true, nil + end + + if not is_nonempty_string(v) then + return nil, err + end + + return true, nil +end + +local function validate_optional_objects(frame, spec) + for field in pairs(spec.optional_objects or {}) do + local v = frame[field] + + if v ~= nil and v ~= cjson.null and type(v) ~= 'table' then + return nil, 'invalid_' .. field + end + end + + return true, nil +end + +local function validate_digest_fields(frame) + if frame.digest_alg ~= M.DIGEST_ALG then + return nil, 'unsupported_digest_alg' + end + + if not is_xxhash32_hex(frame.digest) then + return nil, 'invalid_digest' + end + + return true, nil +end + +local function validate_special_cases(frame) + if frame.type == 'reply' + and frame.ok == false + and frame.err ~= nil + and type(frame.err) ~= 'string' + then + return nil, 'invalid_reply_err' + end + + if frame.type == 'xfer_abort' + and frame.err ~= nil + and type(frame.err) ~= 'string' + then + return nil, 'invalid_xfer_err' + end + + if frame.type == 'xfer_need' then + if frame.retry ~= nil and type(frame.retry) ~= 'boolean' then + return nil, 'invalid_xfer_retry' + end + if frame.reason ~= nil and type(frame.reason) ~= 'string' then + return nil, 'invalid_xfer_reason' + end + if frame.counters ~= nil and type(frame.counters) ~= 'table' then + return nil, 'invalid_xfer_counters' + end + end + + if frame.type == 'xfer_begin' + and frame.meta ~= nil + and type(frame.meta) ~= 'table' + then + return nil, 'invalid_meta' + end + + return true, nil +end + +local function validate_by_spec(frame, opts) + local spec, err = validate_frame_header(frame) + if not spec then + return nil, err + end + + for _, req in ipairs(spec.required or {}) do + local field, field_err = req[1], req[2] + + local ok, verr = validate_required_field(frame, field, field_err) + if not ok then + return nil, verr + end + end + + local ok, oerr = validate_optional_objects(frame, spec) + if not ok then + return nil, oerr + end + + if spec.digest then + ok, oerr = validate_digest_fields(frame) + if not ok then + return nil, oerr + end + end + + ok, oerr = validate_special_cases(frame) + if not ok then + return nil, oerr + end + + if spec.semantic == 'chunk' + and not (opts and opts.wire) + and not M.verify_chunk_digest(frame.data, frame.chunk_digest) + then + return nil, 'chunk_digest_mismatch' + end + + return frame, nil +end + +function M.validate(frame) return validate_by_spec(frame) end +function M.validate_wire(frame) return validate_by_spec(frame, { wire = true }) end + +-- Put near the cjson helpers / small pure helpers section. +function M.is_json_null(v) + return v == cjson.null +end + +function M.normalise_json_null(v) + if v == cjson.null then return nil end + return v +end + +local function copy_json_value(v, seen) + if v == cjson.null then return nil end + if type(v) ~= 'table' then return v end + + seen = seen or {} + if seen[v] then return seen[v] end + + local out = {} + seen[v] = out + + for k, subv in pairs(v) do + local copied = copy_json_value(subv, seen) + if copied ~= nil then out[k] = copied end + end + + return out +end + +function M.copy_json_value(v) return copy_json_value(v) end +function M.normalise_reserved_claim(v) return copy_json_value(v) end +function M.copy_reserved_claim(v) return copy_json_value(v) end + +-- Put after dispatch_lane(). +function M.writer_lane(frame_or_type) + local t = type(frame_or_type) == 'table' + and frame_or_type.type + or frame_or_type + + local lane = M.dispatch_lane(t) + + if lane == 'rpc' then + return 'rpc', nil + end + + if lane == 'transfer' then + if t == 'xfer_chunk' then + return 'bulk', nil + end + return 'control', nil + end + + if lane == 'session_control' then + return nil, 'session_control_frames_are_session_owned' + end + + return nil, 'unknown_frame_lane' +end + + +-------------------------------------------------------------------------------- +-- Constructors +-------------------------------------------------------------------------------- + +local function checked(frame) + local ok, err = M.validate(frame) + if not ok then + return nil, err + end + + return frame, nil +end + +local function put_reserved(frame, identity, auth) + if identity ~= nil then frame.identity = identity end + if auth ~= nil then frame.auth = auth end + return frame +end + +function M.hello(sid, node, identity, auth) + return checked(put_reserved({ + type = 'hello', + proto = M.PROTO, + sid = sid, + node = node, + }, identity, auth)) +end + +function M.hello_ack(sid, node, identity, auth) + return checked(put_reserved({ + type = 'hello_ack', + proto = M.PROTO, + sid = sid, + node = node, + }, identity, auth)) +end + +local CTORS = { + ping = { 'sid' }, + pong = { 'sid' }, + + pub = { 'topic', 'payload', 'retain' }, + unretain = { 'topic' }, + call = { 'id', 'topic', 'payload' }, + reply = { 'id', 'ok', 'payload', 'err' }, + + xfer_begin = { 'xfer_id', 'target', 'size', 'digest_alg', 'digest', 'meta' }, + xfer_ready = { 'xfer_id' }, + xfer_need = { 'xfer_id', 'next', 'retry', 'reason', 'counters' }, + xfer_chunk = { 'xfer_id', 'offset', 'data', 'chunk_digest' }, + xfer_commit = { 'xfer_id', 'size', 'digest_alg', 'digest' }, + xfer_done = { 'xfer_id' }, + xfer_abort = { 'xfer_id', 'err' }, +} + +local function make_ctor(type_name, fields) + return function (...) + local frame = { type = type_name } + + for i = 1, #fields do + local v = select(i, ...) + if v ~= nil then + frame[fields[i]] = v + end + end + + return checked(frame) + end +end + +for type_name, fields in pairs(CTORS) do + M[type_name] = make_ctor(type_name, fields) +end + +-------------------------------------------------------------------------------- +-- Bulk chunk payload encoding +-------------------------------------------------------------------------------- + +local function is_strict_unpadded_b64url(s) + return type(s) == 'string' + and not s:find('=', 1, true) + and s:match('^[A-Za-z0-9_-]*$') ~= nil + and (#s % 4) ~= 1 +end + +function M.encode_chunk(bytes) + assert(type(bytes) == 'string', 'encode_chunk expects string') + return b64url.encode(bytes) +end + +function M.decode_chunk(encoded) + if type(encoded) ~= 'string' then + return nil, 'chunk_must_be_string' + end + + if not is_strict_unpadded_b64url(encoded) then + return nil, 'invalid_base64url_unpadded' + end + + local bytes, err = b64url.decode(encoded) + if bytes == nil then + return nil, err + end + + if M.encode_chunk(bytes) ~= encoded then + return nil, 'non_canonical_base64url' + end + + return bytes, nil +end + +local function to_wire_frame(frame) + if frame.type ~= 'xfer_chunk' then + return frame, nil + end + + local out = shallow_copy(frame) + out.data = M.encode_chunk(frame.data) + + return out, nil +end + +local function from_wire_frame(frame) + if frame.type ~= 'xfer_chunk' then + return frame, nil + end + + local out = shallow_copy(frame) + local bytes, err = M.decode_chunk(out.data) + + if not bytes then + return nil, 'invalid_chunk_encoding: ' .. tostring(err) + end + + -- Do not verify the chunk digest here. Decode-line is a wire parsing + -- boundary; semantic chunk acceptance belongs to the transfer receiver so + -- it can request a same-offset retry instead of treating the line as an + -- unroutable wire error. + out.data = bytes + + return out, nil +end + +-------------------------------------------------------------------------------- +-- Line encoding +-------------------------------------------------------------------------------- + +function M.encode_line(frame) + local ok, err = M.validate(frame) + if not ok then + return nil, err + end + + local wire_frame, wire_err = to_wire_frame(frame) + if not wire_frame then + return nil, wire_err + end + + local line, json_err = cjson.encode(wire_frame) + if not line then + return nil, 'encode_failed: ' .. tostring(json_err) + end + + return line, nil +end + +function M.decode_line(line) + if type(line) ~= 'string' then + return nil, 'line_must_be_string' + end + + local wire_frame, json_err = cjson.decode(line) + if not wire_frame then + return nil, 'decode_failed: ' .. tostring(json_err) + end + + local valid_wire, valid_err = M.validate_wire(wire_frame) + if not valid_wire then + return nil, valid_err + end + + local frame, frame_err = from_wire_frame(valid_wire) + if not frame then + return nil, frame_err + end + + return validate_by_spec(frame, { wire = true }) +end + +return M diff --git a/src/services/fabric/service.lua b/src/services/fabric/service.lua new file mode 100644 index 00000000..9e276cf4 --- /dev/null +++ b/src/services/fabric/service.lua @@ -0,0 +1,1371 @@ +-- services/fabric/service.lua +-- +-- Fabric service coordinator spine. +-- +-- This module supervises link work as scoped children, records completions as +-- semantic data, updates a model, and applies service-level policy. +-- +-- This module owns the configured-link Fabric service runner. It deliberately +-- does not own: +-- * transport opening +-- * bridge routing +-- * transfer attempts +-- * HAL work +-- +-- Coordinator rule: +-- one suspending control point; branches do not suspend. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local priority_event = require 'devicecode.support.priority_event' +local config_watch = require 'devicecode.support.config_watch' +local service_events = require 'devicecode.support.service_events' +local model_mod = require 'services.fabric.model' +local link_mod = require 'services.fabric.link' +local config_mod = require 'services.fabric.config' +local topics = require 'services.fabric.topics' +local transfer_client = require 'services.fabric.transfer_client' +local service_base = require 'devicecode.service_base' +local request_owner = require 'devicecode.support.request_owner' +local dep_slot = require 'devicecode.support.dependency_slot' +local dep_failure = require 'devicecode.support.dependency_failure' +local fabric_deps = require 'services.fabric.dependencies' +local tablex = require 'shared.table' + +local M = {} + +local Service = {} +Service.__index = Service + +local DEFAULT_DONE_QUEUE = 64 + +local shallow_copy = tablex.shallow_copy + +local function copy_link_entry(v) + local out = shallow_copy(v) + + if type(out.result) == 'table' then + out.result = shallow_copy(out.result) + end + + if type(out.snapshot) == 'table' then + out.snapshot = shallow_copy(out.snapshot) + end + + return out +end + +local function copy_links(links) + local out = {} + + for id, v in pairs(links or {}) do + out[id] = copy_link_entry(v) + end + + return out +end + +local function copy_snapshot(s) + local out = shallow_copy(s or {}) + out.links = copy_links(out.links) + return out +end + +local function shallow_value_equal(a, b) + if a == b then return true end + if type(a) ~= 'table' or type(b) ~= 'table' then return false end + + for k, v in pairs(a) do + if b[k] ~= v then return false end + end + + for k in pairs(b) do + if a[k] == nil then return false end + end + + return true +end + +local function snapshots_equal(a, b) + if a == b then return true end + if type(a) ~= 'table' or type(b) ~= 'table' then return false end + + if a.service_id ~= b.service_id then return false end + if a.service_generation ~= b.service_generation then return false end + if a.state ~= b.state then return false end + if a.reason ~= b.reason then return false end + if a.total ~= b.total then return false end + if a.completed ~= b.completed then return false end + + local al = a.links or {} + local bl = b.links or {} + + for id, av in pairs(al) do + local bv = bl[id] + if type(bv) ~= 'table' then return false end + if av.link_generation ~= bv.link_generation then return false end + if av.status ~= bv.status then return false end + if av.primary ~= bv.primary then return false end + if not shallow_value_equal(av.result, bv.result) then return false end + end + + for id in pairs(bl) do + if al[id] == nil then return false end + end + + return true +end + +local function require_params(params) + if type(params) ~= 'table' then + error('fabric.service.run: params table required', 3) + end + + return params +end + +local function compiled_from_params(params) + local compiled = params.compiled_config + if compiled ~= nil then + return compiled, nil + end + + local raw = params.config + if raw == nil then + return nil, nil + end + + local c, err = config_mod.compile(raw) + if not c then + return nil, err or 'fabric config compile failed' + end + return c, nil +end + +local function link_override(params, id, index) + local overrides = params.link_overrides + if type(overrides) ~= 'table' then + return nil + end + return overrides[id] or overrides[index] +end + +local function apply_runtime_defaults(params, copy) + for _, key in ipairs({ + 'conn', + 'open_transport_op', + 'local_rx', 'transfer_admission_rx', + }) do + if copy[key] == nil and params[key] ~= nil then + copy[key] = params[key] + end + end +end + +local function normalise_link_specs(params) + local compiled, cerr = compiled_from_params(params) + if cerr then + error('fabric.service: ' .. tostring(cerr), 3) + end + + local list = (compiled and compiled.links) or params.links + + if type(list) ~= 'table' then + error('fabric.service: links array required', 3) + end + + local out = {} + local seen = {} + + for i, spec in ipairs(list) do + if type(spec) ~= 'table' then + error('fabric.service: link entry must be a table', 3) + end + + local id = spec.link_id + if type(id) ~= 'string' or id == '' then + error('fabric.service: link id must be a non-empty string', 3) + end + + if seen[id] then + error('fabric.service: duplicate link id: ' .. id, 3) + end + seen[id] = true + + local copy = shallow_copy(spec) + copy.link_id = id + copy.link_generation = copy.link_generation or i + + local override = link_override(params, id, i) + if override ~= nil then + if type(override) ~= 'table' then + error('fabric.service: link override must be a table: ' .. id, 3) + end + for k, v in pairs(override) do + copy[k] = v + end + end + + apply_runtime_defaults(params, copy) + fabric_deps.assign_transport_dependency(copy, override) + + out[#out + 1] = copy + out[id] = copy + end + + if #out == 0 then + error('fabric.service: at least one link required', 3) + end + + return out +end + +local function initial_snapshot(service_id, service_generation, link_specs) + local links = {} + + for _, spec in ipairs(link_specs) do + links[spec.link_id] = { + link_generation = spec.link_generation, + status = 'starting', + } + end + + return { + service_id = service_id, + service_generation = service_generation, + state = 'starting', + total = #link_specs, + completed = 0, + links = links, + } +end + +local function public_snapshot(self) + return self._model:snapshot() +end + +local function set_state(self, state, reason) + self._model:update(function (s) + s.state = state + s.reason = reason + end) +end + +local function advance_completion_state(s) + if s.state == 'failed' or s.state == 'cancelling' then + return + end + + if s.completed >= s.total then + s.state = 'completed' + else + s.state = 'running' + end +end + +local function record_link_done(self, ev) + local accepted = false + local reject_reason = nil + + self._model:update(function (s) + local id = ev.link_id + local l = s.links[id] + + if not l then + reject_reason = 'unknown_link' + return + end + + if l.link_generation ~= ev.link_generation + or l.status == 'ok' + or l.status == 'failed' + or l.status == 'cancelled' + then + reject_reason = 'stale_link_completion' + return + end + + l.status = ev.status + + if ev.status == 'ok' then + if type(ev.result) == 'table' then + l.result = shallow_copy(ev.result) + + if type(ev.result.snapshot) == 'table' then + l.snapshot = shallow_copy(ev.result.snapshot) + end + else + l.result = ev.result + end + else + l.primary = ev.primary + end + + s.completed = (s.completed or 0) + 1 + advance_completion_state(s) + accepted = true + end) + + return accepted, reject_reason +end + +local function canonical_dependency_route_failure(ev) + if type(ev) ~= 'table' then return nil end + local primary = ev.primary + if not dep_failure.is(primary) or not dep_failure.is_no_route(primary) then return nil end + local out = shallow_copy(primary) + out.kind = 'dependency_failure' + out.err = 'no_route' + out.dependency_key = dep_failure.key(out) + out.link_id = out.link_id or ev.link_id + return out +end +local function default_policy(_, ev) + if ev.kind ~= 'link_done' then + return { action = 'continue' } + end + + if ev.status == 'ok' then + return { action = 'continue' } + end + + if ev.status == 'failed' then + local dependency_failure = canonical_dependency_route_failure(ev) + if dependency_failure ~= nil then + return { + action = 'fail', + reason = dependency_failure, + } + end + + return { + action = 'fail', + reason = ('link %s failed: %s'):format( + tostring(ev.link_id), + tostring(ev.primary or 'failed') + ), + } + end + + if ev.status == 'cancelled' then + return { + action = 'fail', + reason = ('link %s cancelled unexpectedly: %s'):format( + tostring(ev.link_id), + tostring(ev.primary or 'cancelled') + ), + } + end + + return { + action = 'fail', + reason = ('link %s ended with invalid status: %s'):format( + tostring(ev.link_id), + tostring(ev.status) + ), + } +end + +local function cancel_all_links(self, reason) + for _, rec in pairs(self._links) do + if rec and rec.handle and rec.handle.cancel then + rec.handle:cancel(reason or 'service_cancelled') + end + end +end + +local function apply_policy(self, ev) + local policy = self._policy or default_policy + local decision = policy(self, ev) or { action = 'continue' } + local action = decision.action or 'continue' + + if action == 'continue' then + return + end + + if action == 'fail' then + local reason = decision.reason or 'fabric service policy failure' + set_state(self, 'failed', reason) + error(reason, 0) + end + + if action == 'cancel' then + local reason = decision.reason or 'fabric service policy cancellation' + set_state(self, 'cancelling', reason) + cancel_all_links(self, reason) + return + end + + if action == 'complete' then + local snap = public_snapshot(self) + if (snap.completed or 0) < (snap.total or 0) then + error('fabric.service: complete policy before all links completed', 0) + end + + set_state(self, 'completed', decision.reason) + self._complete = true + return + end + + error('fabric.service: unknown policy action: ' .. tostring(action), 0) +end + +local function make_link_identity(self, spec) + return { + kind = 'link_done', + service_id = self._service_id, + service_generation = self._service_generation, + link_generation = spec.link_generation, + link_id = spec.link_id, + dependency_key = spec.dependency_key, + } +end + +local function make_service_caps(self) + return { + service_id = self._service_id, + service_generation = self._service_generation, + conn = self._conn, + compiled_config = self._compiled_config, + routing = self._compiled_config and self._compiled_config.routing or nil, + svc = self._svc, + + snapshot = public_snapshot(self), + } +end + +local function default_link_runner(link_scope, spec, service_caps) + if service_caps and service_caps.svc then + local out = shallow_copy(spec) + out.state_projector_svc = service_caps.svc + return link_mod.run(link_scope, out) + end + return link_mod.run(link_scope, spec) +end + +local function public_runner_spec(self, spec) + if self._link_runner == nil or self._private_link_runtime then + return spec + end + + local out = shallow_copy(spec) + -- transfer_admission_rx is a service-internal channel used to connect the + -- public transfer-manager capability to the composed link implementation. + -- Custom link runners receive the documented public link spec only. + out.transfer_admission_rx = nil + return out +end + +local function start_link(self, spec) + local runner = self._link_runner or default_link_runner + + local handle, err = scoped_work.start { + lifetime_scope = self._scope, + reaper_scope = self._scope, + report_scope = self._scope, + + identity = make_link_identity(self, spec), + preserve_error_primary = true, + + run = function (link_scope) + return runner(link_scope, public_runner_spec(self, spec), make_service_caps(self)) + end, + + report = service_events.reporter_for( + self._done_tx, + make_link_identity(self, spec), + { label = 'fabric_service_link_completion_report_failed' } + ), + } + + if not handle then + return nil, err + end + + self._links[spec.link_id] = { + link_id = spec.link_id, + link_generation = spec.link_generation, + handle = handle, + } + + return handle, nil +end + +local function start_all_links(self, link_specs) + for _, spec in ipairs(link_specs) do + local h, err = start_link(self, spec) + if not h then + set_state(self, 'failed', err or 'link_start_failed') + error(err or 'link_start_failed', 0) + end + end + + set_state(self, 'running') +end + +local function next_event_op(self) + return self._done_rx:recv_op():wrap(function (ev) + if ev == nil then + return { + kind = 'done_queue_closed', + } + end + + return ev + end) +end + +local function coordinator_loop(self) + while not self._complete do + local ev = fibers.perform(next_event_op(self)) + + if ev.kind == 'done_queue_closed' then + error('fabric service completion queue closed', 0) + end + + if ev.kind == 'link_done' then + if ev.service_id ~= self._service_id or ev.service_generation ~= self._service_generation then + -- Completion for another service instance/generation. Ignore. + else + local accepted = record_link_done(self, ev) + + if accepted then + apply_policy(self, ev) + end + end + else + error('fabric.service: unknown event kind: ' .. tostring(ev.kind), 0) + end + + local snap = public_snapshot(self) + if snap.completed >= snap.total then + self._complete = true + end + end + + return { + role = 'fabric_service', + service_id = self._service_id, + service_generation = self._service_generation, + snapshot = public_snapshot(self), + } +end + +--- Run the Fabric service coordinator inside an existing service scope. +--- +--- params: +--- service_id?: string +--- service_generation?: integer +--- links: array of link specs. Each spec is passed to link.run by default. +--- link_runner?: function(link_scope, link_spec, service_caps) -> result_table +--- policy?: function(service, ev) -> { action = ... } +--- done_queue_len?: integer +--- +---@param scope Scope +---@param params table +---@return table result +function M.run(scope, params) + if type(scope) ~= 'table' then + error('fabric.service.run: scope required', 2) + end + + params = require_params(params) + + local compiled, cerr = compiled_from_params(params) + if cerr then + error('fabric.service: ' .. tostring(cerr), 2) + end + local link_specs = normalise_link_specs(params) + local service_id = params.service_id or (compiled and compiled.service and compiled.service.service_id) or 'fabric' + local service_generation = params.service_generation or 1 + + local initial = initial_snapshot(service_id, service_generation, link_specs) + local model = model_mod.new(initial, { + copy = copy_snapshot, + equals = snapshots_equal, + label = 'fabric.service', + }) + + + scope:finally(function (_, status, primary) + model:terminate(primary or status or 'fabric service closed') + end) + + local done_tx, done_rx = mailbox.new( + params.done_queue_len or math.max(DEFAULT_DONE_QUEUE, #link_specs + 4), + { full = 'reject_newest' } + ) + + scope:finally(function () + done_tx:close('fabric service closed') + end) + + local self = setmetatable({ + _scope = scope, + _service_id = service_id, + _service_generation = service_generation, + _model = model, + _done_tx = done_tx, + _done_rx = done_rx, + _policy = params.policy, + _link_runner = params.link_runner, + _private_link_runtime = not not params._private_link_runtime, + _conn = params.conn, + _compiled_config = compiled, + _links = {}, + _complete = false, + _svc = params.svc, + }, Service) + + start_all_links(self, link_specs) + + return coordinator_loop(self) +end + + +-------------------------------------------------------------------------------- +-- Long-lived public Fabric service shell +-------------------------------------------------------------------------------- + +local function extract_config_payload(ev_or_msg) + if type(ev_or_msg) == 'table' and ev_or_msg.kind == 'cfg' then + return ev_or_msg.raw + end + local payload = ev_or_msg and ev_or_msg.payload or ev_or_msg + if type(payload) == 'table' and payload.data ~= nil then + return payload.data + end + return payload +end + +local retain_transfer_interface +local unretain_transfer_interface +local close_transfer_admissions +local bind_transfer_manager + +local function service_status_payload(state, service_state, extra) + local active = state.active + local payload = { + state = service_state, + ready = active ~= nil, + config_generation = active and active.config_generation or nil, + last_error = state.last_error, + dependencies = state.generation_deps and state.generation_deps:snapshot() or nil, + } + for k, v in pairs(extra or {}) do + payload[k] = v + end + return payload +end + +local function publish_service_lifecycle(state, service_state, extra) + local payload = service_status_payload(state, service_state, extra) + + retain_transfer_interface(state, service_state, extra) + + if service_state == 'starting' then + state.svc:starting(payload) + elseif service_state == 'running' then + state.svc:running(payload) + elseif service_state == 'degraded' then + state.svc:degraded(payload) + elseif service_state == 'failed' then + state.svc:failed(payload.reason or payload.last_error or 'failed', payload) + elseif service_state == 'stopped' then + state.svc:stopped(payload) + else + state.svc:status(service_state, payload) + end + + return payload +end + + +retain_transfer_interface = function(state, service_state, extra) + if not state.conn then return true, nil end + local active = state.active + local links = {} + for link_id in pairs(state.transfer_admissions or {}) do links[#links + 1] = link_id end + table.sort(links) + + local ok, err = bus_cleanup.retain(state.conn, topics.transfer_manager_meta(), { + kind = 'cap.transfer-manager', + class = 'transfer-manager', + id = 'main', + owner = 'fabric', + methods = { 'send-blob' }, + canonical_state = topics.state_root(), + links = links, + }) + if ok ~= true then return nil, err end + + return bus_cleanup.retain(state.conn, topics.transfer_manager_status(), { + state = service_state == 'running' and active ~= nil and 'available' or 'unavailable', + available = service_state == 'running' and active ~= nil, + ready = service_state == 'running' and active ~= nil, + config_generation = active and active.config_generation or nil, + reason = extra and (extra.reason or extra.last_error) or state.last_error, + links = links, + }) +end + +unretain_transfer_interface = function(state) + if not state.conn then return true, nil end + bus_cleanup.unretain(state.conn, topics.transfer_manager_meta()) + bus_cleanup.unretain(state.conn, topics.transfer_manager_status()) + return true, nil +end + +close_transfer_admissions = function(state, reason) + for _, tx in pairs(state.transfer_admissions or {}) do + if tx and type(tx.close) == 'function' then tx:close(reason or 'fabric_generation_closed') end + end + state.transfer_admissions = {} +end + +local function count_transfer_admissions(state) + local n, only = 0, nil + for link_id in pairs(state.transfer_admissions or {}) do + n = n + 1 + only = link_id + end + return n, only +end + +local function fail_request(req, reason) + if req and type(req.fail) == 'function' then return req:fail(reason) end + return nil, reason or 'request_failed' +end + +local function reply_request(req, payload) + if req and type(req.reply) == 'function' then return req:reply(payload) end + return nil, 'request has no reply method' +end + +local function transfer_payload(req) + return type(req) == 'table' and type(req.payload) == 'table' and req.payload or {} +end + +local function transfer_request_params(state, req) + local p = shallow_copy(transfer_payload(req)) + local link_id = p.link_id + if type(link_id) ~= 'string' or link_id == '' then + local n, only = count_transfer_admissions(state) + if n == 1 then link_id = only end + end + if type(link_id) ~= 'string' or link_id == '' then return nil, 'link_id_required' end + local tx = state.transfer_admissions and state.transfer_admissions[link_id] or nil + if not tx then return nil, 'link_not_ready' end + p.link_id = link_id + p.admission_tx = tx + state.transfer_seq = (state.transfer_seq or 0) + 1 + local seq = tostring(state.transfer_seq) + if type(p.request_id) ~= 'string' or p.request_id == '' then + p.request_id = 'fabric-transfer-request-' .. seq + end + if type(p.xfer_id) ~= 'string' or p.xfer_id == '' then + p.xfer_id = 'fabric-xfer-' .. seq + end + return p, nil +end + +local function run_public_transfer_request(scope, state, req, owner) + owner = owner or request_owner.new(req) + local params, perr = transfer_request_params(state, req) + if not params then + owner:fail_once(perr) + return { ok = false, err = perr or 'transfer_request_invalid' } + end + local result, err = transfer_client.run(scope, params) + if not result then + local reason = err or 'transfer_failed' + owner:fail_once(reason) + return { ok = false, err = reason, link_id = params.link_id } + end + local ok, rerr = owner:reply_once({ ok = true, result = result, link_id = params.link_id }) + return { ok = ok == true, err = rerr, result = result, link_id = params.link_id } +end + +bind_transfer_manager = function(scope, state, opts) + if opts and opts.bind_transfer_manager == false then return true, nil end + local ep, err = bus_cleanup.bind(state.conn, topics.transfer_manager_rpc('send-blob'), { + queue_len = opts and opts.transfer_manager_queue_len or 16, + }) + if not ep then return nil, err end + state.transfer_manager_ep = ep + scope:finally(function () + bus_cleanup.unbind(state.conn, ep) + end) + local ok, spawn_err = scope:spawn(function (worker_scope) + while true do + local req = fibers.perform(ep:recv_op()) + if req == nil then return { role = 'transfer_manager_endpoint', reason = 'endpoint_closed' } end + local owner = request_owner.new(req) + local payload = type(req.payload) == 'table' and req.payload or {} + local transfer_id = payload.request_id or payload.xfer_id or tostring(state.transfer_seq or 0) + local handle, serr = scoped_work.start { + lifetime_scope = worker_scope, + reaper_scope = worker_scope, + report_scope = worker_scope, + identity = { + kind = 'public_transfer_request_done', + service_id = state.service_id, + request_id = tostring(transfer_id), + }, + setup = function (request_scope) + request_scope:finally(function (_, status, primary) + owner:finalise_unresolved(primary or status or 'transfer_request_closed') + end) + return { + request_owner = owner, + cancel_owned_now = function (reason) + owner:abandon_unresolved(reason or 'caller_abandoned') + return true + end, + } + end, + cancel_op = owner:caller_cancel_op(), + run = function (request_scope, setup) + return run_public_transfer_request(request_scope, state, req, setup.request_owner) + end, + } + if not handle then + owner:fail_once(serr or 'transfer_request_scope_start_failed') + end + end + end) + if ok ~= true then + bus_cleanup.unbind(state.conn, ep) + return nil, spawn_err or 'transfer_manager_endpoint_start_failed' + end + return true, nil +end + +local function cancel_active_generation(state, reason) + local active = state.active + local why = reason or 'generation_cancelled' + close_transfer_admissions(state, why) + if not active then return false end + if active.stopping then return true end + active.stopping = true + active.stop_reason = why + if active.handle and active.handle.cancel then + active.handle:cancel(why) + end + publish_service_lifecycle(state, 'stopping_generation', { + ready = false, + reason = why, + config_generation = active.config_generation, + }) + return true +end + +local function merged_link_override(state, link_id) + local out = {} + local src = state.link_overrides and state.link_overrides[link_id] or nil + if type(src) == 'table' then + for k, v in pairs(src) do out[k] = v end + end + return out +end + +local function build_generation_overrides(state, compiled) + local overrides = {} + close_transfer_admissions(state, 'generation_replaced') + state.transfer_admissions = {} + + for _, link in ipairs(compiled.links or {}) do + local override = merged_link_override(state, link.link_id) + override.conn = override.conn or state.conn + if override.transfer_admission_rx == nil then + local tx, rx = mailbox.new(state.transfer_admission_queue_len or 16, { full = 'reject_newest' }) + state.transfer_admissions[link.link_id] = tx + override.transfer_admission_rx = rx + end + overrides[link.link_id] = override + end + + return overrides +end + + +local start_generation + +local function terminate_generation_deps(state, reason) + dep_slot.terminate(state, 'generation_deps', reason or 'fabric_generation_dependency_closed') + state.generation_dependency_specs = nil + return true +end + +local function open_generation_deps(state, compiled) + terminate_generation_deps(state, 'fabric_generation_dependency_replaced') + if state.link_runner ~= nil and not state.private_link_runtime then return true, nil end + local specs = fabric_deps.transport_dependencies(compiled, state.link_overrides) + if #specs == 0 then return true, nil end + local ok, err = dep_slot.replace(state, 'generation_deps', state.conn, specs, { + replace_reason = 'fabric_generation_dependency_replaced', + changed_kind = 'fabric_dependency_changed', + closed_kind = 'fabric_dependency_closed', + queue_len = state.dependency_queue_len or 8, + full = 'drop_oldest', + }) + if ok ~= true then return nil, err or 'fabric_dependency_open_failed' end + state.generation_dependency_specs = specs + return true, nil +end +local function all_generation_deps_available(state) + return dep_slot.all_available(state, 'generation_deps', state.generation_dependency_specs) +end + +local function publish_waiting_for_transport(state, reason) + state.last_error = reason or 'transport_unavailable' + publish_service_lifecycle(state, 'waiting_for_dependency', { + ready = false, + reason = state.last_error, + dependency = 'transport', + links = state.pending_compiled and #(state.pending_compiled.links or {}) or nil, + }) + return true +end + + + +local function log_link_inventory(state, links, generation) + if not state.svc then return end + state._logged_link_inventory = state._logged_link_inventory or {} + for _, link in ipairs(links or {}) do + local id = link.link_id or link.id + local tr = link.transport or {} + local transport = tr.id or tr.source or tr.kind or link.transport_id + local peer = link.peer_id or (link.session and link.session.peer_id) or 'peer' + local key = tostring(generation) .. '|' .. tostring(id) .. '|' .. tostring(transport) .. '|' .. tostring(peer) + if id ~= nil and state._logged_link_inventory[id] ~= key then + state._logged_link_inventory[id] = key + state.svc:info('fabric_link_configured', { + summary = string.format('fabric link %s configured transport=%s peer=%s', tostring(id), tostring(transport or 'unknown'), tostring(peer or 'unknown')), + link_id = id, + transport = transport, + peer = peer, + generation = generation, + }) + end + end +end + +local function link_summary_parts(links) + local ids, transports = {}, {} + for _, link in ipairs(links or {}) do + local id = link.link_id or link.id + if id ~= nil then ids[#ids + 1] = tostring(id) end + local tr = link.transport or {} + local tid = tr.id or tr.source or tr.kind or link.transport_id + if tid ~= nil then transports[#transports + 1] = tostring(tid) end + end + table.sort(ids) + table.sort(transports) + return table.concat(ids, ','), table.concat(transports, ',') +end + +local function maybe_start_pending_generation(state, reason) + if state.active ~= nil then return true, nil end + local compiled = state.pending_compiled + if compiled == nil then return true, nil end + if not all_generation_deps_available(state) then + publish_waiting_for_transport(state, reason or state.pending_generation_reason or 'transport_unavailable') + return true, nil + end + state.pending_compiled = nil + state.pending_generation_reason = nil + local active, err = start_generation(state, compiled) + if not active then return nil, err or 'generation_start_failed' end + state.last_error = nil + local link_count = #(compiled.links or {}) + if state.svc and state._last_operator_generation ~= active.config_generation then + local link_ids, transports = link_summary_parts(compiled.links) + state.svc:info('fabric_ready', { + summary = string.format('fabric ready generation=%s links=%d%s%s', tostring(active.config_generation), link_count, link_ids ~= '' and (' ids=' .. link_ids) or '', transports ~= '' and (' transport=' .. transports) or ''), + config_generation = active.config_generation, + links = link_count, + link_ids = link_ids ~= '' and link_ids or nil, + transport = transports ~= '' and transports or nil, + }) + state._last_operator_generation = active.config_generation + end + log_link_inventory(state, compiled.links, active.config_generation) + publish_service_lifecycle(state, 'running', { + ready = true, + config_generation = active.config_generation, + links = link_count, + }) + return true, nil +end + +local function open_deps_and_maybe_start_pending_generation(state, reason) + if state.active ~= nil then return true, nil end + local compiled = state.pending_compiled + if compiled == nil then return true, nil end + + local ok_dep, dep_err = open_generation_deps(state, compiled) + if ok_dep ~= true then + return nil, dep_err or 'dependency_open_failed' + end + + return maybe_start_pending_generation(state, reason or state.pending_generation_reason or 'config_changed') +end + +function start_generation(state, compiled) + state.config_generation_seq = state.config_generation_seq + 1 + local config_generation = state.config_generation_seq + local overrides = build_generation_overrides(state, compiled) + + local handle, err = scoped_work.start { + lifetime_scope = state.scope, + reaper_scope = state.scope, + report_scope = state.scope, + + identity = { + kind = 'generation_done', + service_id = state.service_id, + config_generation = config_generation, + }, + preserve_error_primary = true, + + run = function (gen_scope) + return M.run(gen_scope, { + service_id = state.service_id, + service_generation = config_generation, + compiled_config = compiled, + conn = state.conn, + link_runner = state.link_runner, + _private_link_runtime = state.private_link_runtime, + link_overrides = overrides, + svc = state.svc, + policy = state.config_generation_policy, + done_queue_len = state.config_generation_done_queue_len, + }) + end, + + report = service_events.reporter_for( + state.events_port or state.done_tx, + { + service_id = state.service_id, + source = 'fabric_generation', + source_id = tostring(config_generation), + config_generation = config_generation, + }, + { label = 'fabric_generation_done_report_failed' } + ), + } + + if not handle then + return nil, err or 'generation_start_failed' + end + + state.active = { + config_generation = config_generation, + compiled = compiled, + handle = handle, + overrides = overrides, + } + + return state.active, nil +end + +local function replace_generation(state, compiled, reason) + state.pending_compiled = compiled + state.pending_generation_reason = reason or 'config_changed' + if cancel_active_generation(state, state.pending_generation_reason) then + return true, nil + end + return open_deps_and_maybe_start_pending_generation(state, state.pending_generation_reason) +end + +local function dependency_key_exists(state, key) + return key ~= nil and state.generation_deps ~= nil and state.generation_deps:dependency(key) ~= nil +end + +local function generation_route_failure(state, ev) + local primary = ev and ev.primary + if not dep_failure.is(primary) or not dep_failure.is_no_route(primary) then return nil end + local dep_key = dep_failure.key(primary) + if not dependency_key_exists(state, dep_key) then return nil end + return dep_key, primary +end +local function handle_generation_done(state, ev) + local active = state.active + if not active or ev.config_generation ~= active.config_generation then + return + end + + local completed = active + local was_stopping = active.stopping == true + close_transfer_admissions(state, ev.status == 'ok' and 'generation_completed' or 'generation_failed') + state.active = nil + + if state.pending_compiled ~= nil then + local ok_pending, pending_err = open_deps_and_maybe_start_pending_generation( + state, + state.pending_generation_reason or (was_stopping and active.stop_reason) or 'previous_generation_stopped' + ) + if ok_pending ~= true then + state.last_error = tostring(pending_err or 'generation_start_failed') + publish_service_lifecycle(state, 'degraded', { + reason = 'generation_start_failed', + last_error = state.last_error, + config_generation = ev.config_generation, + }) + end + return + end + + if was_stopping then + state.last_error = nil + publish_service_lifecycle(state, 'stopped_generation', { + ready = false, + reason = active.stop_reason or 'generation_cancelled', + config_generation = ev.config_generation, + }) + return + end + + if ev.status ~= 'ok' and state.generation_deps then + local dep_key, failure = generation_route_failure(state, ev) + if dep_key ~= nil then + state.generation_deps:classify_call_failure(dep_key, failure, nil) + state.pending_compiled = completed and completed.compiled or state.pending_compiled + state.pending_generation_reason = 'transport_route_missing' + publish_waiting_for_transport(state, 'transport_route_missing') + return + end + end + + if ev.status == 'ok' then + state.last_error = nil + publish_service_lifecycle(state, 'running', { + ready = false, + config_generation = ev.config_generation, + last_completed_config_generation = ev.config_generation, + }) + return + end + + state.last_error = tostring(ev.primary or ev.status or 'generation_failed') + publish_service_lifecycle(state, 'degraded', { + reason = 'generation_failed', + last_error = state.last_error, + config_generation = ev.config_generation, + }) +end + +local function config_event_from_watch(ev) + if ev == nil then return nil end + if ev.kind == 'config_closed' then + return { kind = 'cfg_closed', err = ev.err } + end + return { + kind = 'cfg', + generation = ev.generation, + rev = ev.rev, + raw = ev.raw, + msg = ev.msg, + } +end + +local function done_event_from_recv(ev, err) + if ev == nil then + return { kind = 'done_closed', err = err } + end + return ev +end + +local function try_shell_done_now(state) + local ev, err = queue.try_recv_now(state.done_rx) + if ev ~= nil then + return done_event_from_recv(ev, nil) + end + if err ~= 'not_ready' then + return done_event_from_recv(nil, err) + end + return nil +end + +local function try_shell_cfg_now(state) + local ev = state.cfg_watch:try_recv_now() + if ev ~= nil then return config_event_from_watch(ev) end + return nil +end + +local function next_shell_event_op(state) + local sources = { + { + name = 'done', + try_now = function () return try_shell_done_now(state) end, + recv_op = function () + return state.done_rx:recv_op():wrap(done_event_from_recv) + end, + }, + { + name = 'cfg', + try_now = function () return try_shell_cfg_now(state) end, + recv_op = function () + return state.cfg_watch:recv_op():wrap(config_event_from_watch) + end, + }, + } + local deps_source = dep_slot.event_source(state, 'generation_deps', { name = 'dependencies' }) + if deps_source then sources[#sources + 1] = deps_source end + return priority_event.sources_op { + label = 'fabric.service.shell', + pending = state.pending_events, + sources = sources, + } +end + +local function handle_config_event(state, ev_or_msg) + local raw = extract_config_payload(ev_or_msg) + local compiled, err = config_mod.compile(raw) + if not compiled then + state.last_error = tostring(err or 'invalid_config') + publish_service_lifecycle(state, 'degraded', { + reason = 'invalid_config', + last_error = state.last_error, + }) + return + end + + state.pending_compiled = compiled + state.pending_generation_reason = 'config_changed' + + if cancel_active_generation(state, 'config_changed') then + return + end + + local ok_start, gerr = open_deps_and_maybe_start_pending_generation(state, 'config_changed') + if ok_start ~= true then + state.last_error = tostring(gerr or 'generation_start_failed') + publish_service_lifecycle(state, 'degraded', { + reason = 'generation_start_failed', + last_error = state.last_error, + }) + return + end +end + +local function handle_shell_event(state, ev) + if ev.kind == 'cfg' then + handle_config_event(state, ev) + elseif ev.kind == 'generation_done' then + handle_generation_done(state, ev) + elseif ev.kind == 'fabric_dependency_changed' or ev.kind == 'fabric_dependency_closed' then + local ok, err = maybe_start_pending_generation(state, ev.kind == 'fabric_dependency_changed' and 'dependency_available' or 'dependency_closed') + if ok ~= true then error(err or 'fabric dependency handling failed', 0) end + elseif ev.kind == 'cfg_closed' or ev.kind == 'done_closed' then + error(tostring(ev.err or ev.kind), 0) + else + error('fabric.service.start: unknown event kind: ' .. tostring(ev.kind), 0) + end +end + +--- Start the long-lived public Fabric service shell. +--- +--- It watches retained Fabric config and starts, replaces, or cancels configured +--- link generations. It does not expose application-specific entry points. +function M.start(conn, opts) + opts = opts or {} + if conn == nil then + error('fabric.service.start: conn required', 2) + end + + local scope = fibers.current_scope() + if not scope then + error('fabric.service.start must be called inside a fiber', 2) + end + + local svc = service_base.new(conn, { + name = opts.name or 'fabric', + env = opts.env, + meta = opts.meta, + announce = opts.announce, + }) + + local cfg_watch, cfg_err = config_watch.open(conn, 'fabric', { + topic = opts.config_topic or topics.cfg(), + queue_len = opts.config_queue_len or 4, + full = 'reject_newest', + }) + if not cfg_watch then + error(cfg_err or 'fabric config subscribe failed', 2) + end + + local done_tx, done_rx = mailbox.new(opts.done_queue_len or 128, { full = 'reject_newest' }) + + local state = { + scope = scope, + conn = conn, + svc = svc, + service_id = opts.service_id or 'fabric', + cfg_watch = cfg_watch, + done_tx = done_tx, + done_rx = done_rx, + events_port = service_events.port(done_tx, { + service_id = opts.service_id or 'fabric', + source = 'fabric_service', + source_id = opts.service_id or 'fabric', + }, { label = 'fabric_service_event_report_failed' }), + active = nil, + pending_compiled = nil, + pending_generation_reason = nil, + generation_deps = nil, + generation_dependency_specs = nil, + config_generation_seq = 0, + pending_events = {}, + link_runner = opts.link_runner, + private_link_runtime = not not opts._private_link_runtime, + link_overrides = opts.link_overrides, + config_generation_policy = opts.config_generation_policy, + config_generation_done_queue_len = opts.config_generation_done_queue_len, + transfer_admissions = {}, + transfer_admission_queue_len = opts.transfer_admission_queue_len, + dependency_queue_len = opts.dependency_queue_len, + transfer_seq = 0, + last_error = nil, + } + + local transfer_ok, transfer_err = bind_transfer_manager(scope, state, opts) + if transfer_ok ~= true then + error(transfer_err or 'fabric transfer-manager bind failed', 2) + end + + scope:finally(function (_, status, primary) + local reason = primary or status or 'fabric_stop' + cancel_active_generation(state, reason) + terminate_generation_deps(state, reason) + unretain_transfer_interface(state) + done_tx:close(reason) + cfg_watch:close() + end) + + publish_service_lifecycle(state, 'starting', { ready = false }) + + if opts.config ~= nil then + handle_config_event(state, { payload = opts.config }) + end + + while true do + local ev = fibers.perform(next_shell_event_op(state)) + handle_shell_event(state, ev) + end +end + + +M.default_policy = default_policy +M.make_service_caps = make_service_caps +M.Service = Service +M._test = { + generation_route_failure = generation_route_failure, +} + +return M diff --git a/src/services/fabric/session.lua b/src/services/fabric/session.lua new file mode 100644 index 00000000..df5a2dc3 --- /dev/null +++ b/src/services/fabric/session.lua @@ -0,0 +1,779 @@ +-- services/fabric/session.lua +-- Fabric session owner. Reader/link policy owns wire-error budgeting. +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local uuid = require 'uuid' +local queue = require 'devicecode.support.queue' +local priority_event = require 'devicecode.support.priority_event' +local model_mod = require 'services.fabric.model' +local protocol = require 'services.fabric.protocol' +local contracts = require 'devicecode.support.contracts' +local validate = require 'shared.validate' +local trace = require 'services.fabric.trace' + +local M = {} + +local Session = {} +Session.__index = Session + +local OutboundGate = {} +OutboundGate.__index = OutboundGate + +local DEFAULT_HELLO_INTERVAL = 1.0 +local DEFAULT_PING_INTERVAL = 5.0 +local DEFAULT_LIVENESS_TIMEOUT = 15.0 +local DEFAULT_BAD_FRAME_LIMIT = 5 +local DEFAULT_BAD_FRAME_WINDOW_S = 10.0 + +local function require_rx(v, name, level) + return contracts.require_rx(v, name, (level or 1) + 1) +end + +local function require_tx(v, name, level) + return contracts.require_tx(v, name, (level or 1) + 1) +end + +local function positive_number(v, fallback, name) + if v == nil then return fallback end + if type(v) ~= 'number' or v ~= v or v == math.huge or v == -math.huge or v <= 0 then + error('fabric.session: ' .. name .. ' must be a positive finite number', 3) + end + return v +end + +local function positive_integer(v, fallback, name) + if v == nil then return fallback end + return validate.positive_integer(v, 'fabric.session: ' .. name, 2) +end + +function M.new_session_context(args) + if type(args) ~= 'table' then + error('fabric.session.new_session_context: args table required', 2) + end + return { + proto = args.proto, + link_id = args.link_id, + link_generation = args.link_generation, + session_generation = args.session_generation, + local_node = args.local_node, + local_sid = args.local_sid, + peer_node = args.peer_node, + peer_sid = args.peer_sid, + identity_claim = protocol.copy_reserved_claim(args.identity_claim), + auth_claim = protocol.copy_reserved_claim(args.auth_claim), + auth_state = args.auth_state or 'unauthenticated', + authenticated = args.authenticated == true, + established_at = args.established_at, + } +end + +function M.copy_context(ctx) + if type(ctx) ~= 'table' then return nil end + return M.new_session_context(ctx) +end + +function M.context_from_event(ev) + if type(ev) ~= 'table' or type(ev.session) ~= 'table' then return nil end + return M.copy_context(ev.session) +end + +function M.same_session(a, b) + return type(a) == 'table' + and type(b) == 'table' + and a.link_id == b.link_id + and a.link_generation == b.link_generation + and a.session_generation == b.session_generation + and a.peer_sid == b.peer_sid +end + +local function context_from_snapshot(self, cur, at) + return M.new_session_context { + proto = cur.proto, + link_id = self._link_id, + link_generation = self._link_generation, + session_generation = cur.session_generation, + local_node = cur.local_node, + local_sid = cur.local_sid, + peer_node = cur.peer_node, + peer_sid = cur.peer_sid, + identity_claim = cur.peer_identity_claim, + auth_claim = cur.peer_auth_claim, + auth_state = cur.auth_state, + authenticated = cur.authenticated, + established_at = at, + } +end + +local function context_from_peer_frame(self, frame, generation, at) + local cur = self._session_model:snapshot() + return M.new_session_context { + proto = frame.proto, + link_id = self._link_id, + link_generation = self._link_generation, + session_generation = generation, + local_node = cur.local_node, + local_sid = cur.local_sid, + peer_node = frame.node, + peer_sid = frame.sid, + identity_claim = protocol.normalise_reserved_claim(frame.identity), + auth_claim = protocol.normalise_reserved_claim(frame.auth), + auth_state = self._auth_state, + authenticated = self._authenticated, + established_at = at, + } +end + +local function close_unique_txs(txs, reason) + local seen = {} + for _, tx in pairs(txs or {}) do + if tx ~= nil and not seen[tx] then + seen[tx] = true + if type(tx.close) == 'function' then tx:close(reason) end + end + end +end + +function OutboundGate:bind(ctx) + self._session = M.copy_context(ctx) + self._drop_reason = nil + return true, nil +end + +function OutboundGate:drop(reason) + self._session = nil + self._drop_reason = reason or 'no_session' + return true, nil +end + +function OutboundGate:session() + return M.copy_context(self._session) +end + +function OutboundGate:terminate(reason) + if self._closed then return true, nil end + self._closed = true + self._session = nil + self._drop_reason = reason or 'session_outbound_closed' + close_unique_txs(self._lane_txs, self._drop_reason) + return true, nil +end + +function OutboundGate:_admit(ctx, frame, expected_lane, label) + if self._closed then + return nil, tostring(label or 'session_outbound_closed') .. ': closed' + end + local checked, err = protocol.validate(frame) + if not checked then + return nil, tostring(label or 'session_outbound_invalid') .. ': ' .. tostring(err) + end + local lane, lerr = protocol.writer_lane(checked) + if not lane then + return nil, tostring(label or 'session_outbound_invalid_lane') .. ': ' .. tostring(lerr) + end + if expected_lane ~= nil and lane ~= expected_lane then + return nil, tostring(label or 'session_outbound_wrong_lane') .. ': ' .. tostring(lane) + end + local current = self._session + if not current then + return nil, tostring(label or 'session_outbound_no_session') .. ': ' .. tostring(self._drop_reason or 'no_session') + end + if not M.same_session(current, ctx) then + return nil, tostring(label or 'session_outbound_stale_session') .. ': stale_session' + end + local tx = self._lane_txs and self._lane_txs[lane] + if tx == nil then + return nil, tostring(label or 'session_outbound_missing_lane') .. ': ' .. tostring(lane) + end + return queue.try_admit_required(tx, { + kind = 'send_frame', + lane = lane, + frame = checked, + session = M.copy_context(current), + }, label or 'session_outbound_send_failed') +end + +function OutboundGate:send_frame_now(ctx, frame, label) + return self:_admit(ctx, frame, nil, label) +end + +function OutboundGate:send_rpc_frame_now(ctx, frame, label) + return self:_admit(ctx, frame, 'rpc', label) +end + +function OutboundGate:send_transfer_control_frame_now(ctx, frame, label) + return self:_admit(ctx, frame, 'control', label) +end + +function OutboundGate:send_transfer_bulk_frame_now(ctx, frame, label) + return self:_admit(ctx, frame, 'bulk', label) +end + +function M.new_outbound_gate(params) + params = params or {} + return setmetatable({ + _lane_txs = { + control = require_tx(params.tx_control, 'fabric.session.outbound_gate: tx_control', 2), + rpc = require_tx(params.tx_rpc, 'fabric.session.outbound_gate: tx_rpc', 2), + bulk = require_tx(params.tx_bulk, 'fabric.session.outbound_gate: tx_bulk', 2), + }, + _session = nil, + _drop_reason = 'no_session', + _closed = false, + }, OutboundGate) +end + +local function session_equal(a, b) + if a == b then return true end + if type(a) ~= 'table' or type(b) ~= 'table' then return false end + for k, v in pairs(a) do if b[k] ~= v then return false end end + for k in pairs(b) do if a[k] == nil then return false end end + return true +end + +local function session_snapshot(self) + return self._session_model:snapshot() +end + +local function publish_state(self) + if self._state_tx == nil then return true, nil end + local state_mod = require 'services.fabric.state' + return state_mod.admit_component_snapshot_now( + self._state_tx, + self._link_id, + self._link_generation, + self._component_name, + session_snapshot(self), + 'fabric_session_state_admit_failed' + ) +end + +local function update_session(self, mutator) + local changed, err = self._session_model:update(function (cur) + mutator(cur) + return cur + end) + if changed == true then + local ok, perr = publish_state(self) + if ok ~= true then error(perr or 'fabric_session_state_admit_failed', 0) end + end + return changed, err +end + +local function admit_control_frame_now(tx, frame, label) + local checked, err = protocol.validate(frame) + if not checked then + return nil, tostring(label or 'session_control_invalid') .. ': ' .. tostring(err) + end + return queue.try_admit_required(tx, { + kind = 'send_frame', + frame = checked, + }, label or 'session_control_send_failed') +end + +local function must_admit_control_frame_now(tx, frame, label) + local ok, err = admit_control_frame_now(tx, frame, label) + if ok ~= true then error(err or label or 'session_control_send_failed', 0) end + return true +end + +local function is_transient_control_backpressure(err) + err = tostring(err or '') + return err:match(': full$') ~= nil or err:match(': would_block$') ~= nil +end + +local function record_control_send_drop(self, label, err) + trace.error(self._state_tx, { + component = self._component_name or 'session', + link_id = self._link_id, + link_generation = self._link_generation, + }, 'tx', err or label or 'session_control_send_dropped', { event = 'maintenance_control_drop' }) + pcall(function () + update_session(self, function (s) + s.control_send_drops = (s.control_send_drops or 0) + 1 + s.last_control_send_drop = tostring(err or label or 'session_control_send_dropped') + s.last_control_send_drop_at = fibers.now() + end) + end) +end + +local function admit_maintenance_control_frame_now(self, frame, label) + local ok, err = admit_control_frame_now(self._tx_control, frame, label) + if ok == true then return true end + if is_transient_control_backpressure(err) then + record_control_send_drop(self, label, err) + return false, err + end + error(err or label or 'session_control_send_failed', 0) +end + +local function session_event(self, kind, ctx, extra, at) + local ev = { kind = kind, session = M.copy_context(ctx), at = at or fibers.now() } + for k, v in pairs(extra or {}) do ev[k] = v end + return ev +end + +local function peer_session_event(self, ctx, at) + return session_event(self, 'peer_session', ctx, nil, at) +end + +local function peer_session_dropped_event(self, ctx, reason, at) + return session_event(self, 'peer_session_dropped', ctx, { reason = reason or 'session_dropped' }, at) +end + +local function session_frame_event(self, lane, frame, at) + return session_event(self, 'session_frame', context_from_snapshot(self, session_snapshot(self), at), { + lane = lane, + frame = frame, + }, at) +end + +local function publish_to(tx, ev, label) + if tx == nil then return true, nil end + return queue.try_admit_required(tx, ev, label or 'session_event_admit_failed') +end + +local function must_publish_to(tx, ev, label) + local ok, err = publish_to(tx, ev, label) + if ok ~= true then error(err or label or 'session_event_admit_failed', 0) end + return true +end + +local function publish_lifecycle(self, ev) + must_publish_to(self._rpc_tx, ev, 'session_rpc_lifecycle_admit_failed') + must_publish_to(self._transfer_tx, ev, 'session_transfer_lifecycle_admit_failed') + return true +end + +local function publish_session_drop(self, cur, reason, at) + if not cur or cur.established ~= true or cur.peer_sid == nil then return true, nil end + return publish_lifecycle( + self, + peer_session_dropped_event(self, context_from_snapshot(self, cur, at), reason, at) + ) +end + +local function same_peer(cur, frame) + return type(frame.sid) == 'string' + and cur.peer_sid ~= nil + and frame.sid == cur.peer_sid +end + +local function establish_from_peer(self, frame, at) + at = at or fibers.now() + local cur = session_snapshot(self) + local first = cur.established ~= true + local sid_changed = cur.peer_sid ~= frame.sid + local generation = cur.session_generation or 0 + if first or sid_changed then generation = generation + 1 end + if sid_changed and cur.established == true then + publish_session_drop(self, cur, 'peer_sid_changed', at) + end + local ctx = context_from_peer_frame(self, frame, generation, at) + update_session(self, function (s) + s.phase = 'established' + s.established = true + s.peer_sid = frame.sid + s.peer_node = frame.node + s.peer_identity_claim = protocol.normalise_reserved_claim(frame.identity) + s.peer_auth_claim = protocol.normalise_reserved_claim(frame.auth) + s.auth_state = self._auth_state + s.authenticated = self._authenticated + s.proto = frame.proto + s.why = nil + if first or sid_changed then s.session_generation = generation end + end) + self._outbound:bind(ctx) + self._last_peer_at = at + self._next_ping_at = at + self._ping_interval + if first or sid_changed then + publish_lifecycle(self, peer_session_event(self, ctx, at)) + end +end + +local function refresh_peer(self, frame, at) + update_session(self, function (s) + if frame.node ~= nil then s.peer_node = frame.node end + if frame.identity ~= nil then s.peer_identity_claim = protocol.normalise_reserved_claim(frame.identity) end + if frame.auth ~= nil then s.peer_auth_claim = protocol.normalise_reserved_claim(frame.auth) end + end) + self._last_peer_at = at or fibers.now() +end + +local function reset_to_hello(self, reason, now, opts) + now = now or fibers.now() + opts = opts or {} + local rotate_local_sid = opts.rotate_local_sid == true + local cur = session_snapshot(self) + publish_session_drop(self, cur, reason, now) + self._outbound:drop(reason or 'session_dropped') + update_session(self, function (s) + s.phase = 'hello' + -- Liveness timeout starts a fresh local session generation. A + -- bad-frame-limit reset only drops the peer session and keeps the + -- configured local SID stable, preserving existing session tests and + -- avoiding unnecessary local identity churn. + if rotate_local_sid then + s.local_sid = tostring(uuid.new()) + end + s.peer_sid = nil + s.peer_node = nil + s.peer_identity_claim = nil + s.peer_auth_claim = nil + s.auth_state = self._auth_state + s.authenticated = self._authenticated + s.proto = nil + s.established = false + s.why = reason + end) + self._last_peer_at = nil + self._next_hello_at = now + self._next_ping_at = math.huge +end + +local function send_hello(self) + local cur = session_snapshot(self) + admit_maintenance_control_frame_now( + self, + assert(protocol.hello(cur.local_sid, self._local_node, self._identity_claim, self._auth_claim)), + 'session_hello_send_failed' + ) + self._next_hello_at = fibers.now() + self._hello_interval +end + +local function send_hello_ack(self) + local cur = session_snapshot(self) + admit_maintenance_control_frame_now( + self, + assert(protocol.hello_ack(cur.local_sid, self._local_node, self._identity_claim, self._auth_claim)), + 'session_hello_ack_send_failed' + ) +end + +local function send_ping(self) + local cur = session_snapshot(self) + admit_maintenance_control_frame_now(self, assert(protocol.ping(cur.local_sid)), 'session_ping_send_failed') + self._next_ping_at = fibers.now() + self._ping_interval +end + +local function send_pong(self) + local cur = session_snapshot(self) + admit_maintenance_control_frame_now(self, assert(protocol.pong(cur.local_sid)), 'session_pong_send_failed') +end + +local function session_next_deadline(self) + local cur = session_snapshot(self) + local now = fibers.now() + if cur.established ~= true then + return self._next_hello_at or now, 'hello' + end + local ping = self._next_ping_at or math.huge + local live = (self._last_peer_at or now) + self._liveness_timeout + if live <= ping then return live, 'liveness' end + return ping, 'ping' +end + +local function frame_event_from_item(item) + if item == nil then return { kind = 'frame_closed' } end + if type(item) == 'table' and item.kind == 'frame_received' then + return { + kind = 'frame', + frame = item.frame, + at = item.at or fibers.now(), + } + end + + if type(item) == 'table' and item.kind == 'wire_error' then + return { + kind = 'wire_error', + err = item.err or 'wire_error', + at = item.at or fibers.now(), + } + end + return { + kind = 'invalid_frame_item', + err = 'fabric.session frame_rx accepts only frame_received events', + } +end + +local function try_frame_now(self) + if self._frame_closed then return nil end + local item, err = queue.try_recv_now(self._frame_rx) + if item ~= nil then return frame_event_from_item(item) end + if err ~= 'not_ready' then return frame_event_from_item(nil) end + return nil +end + +local function try_session_event_now(self) + if self._pending_frame ~= nil then + local ev = self._pending_frame + self._pending_frame = nil + return ev + end + local ev = try_frame_now(self) + if ev ~= nil then return ev end + if self._pending_timer ~= nil then + ev = self._pending_timer + self._pending_timer = nil + return ev + end + return nil +end + +local function session_event_op(self) + return priority_event.next_op { + label = 'fabric.session', + select_now = function () + return try_session_event_now(self) + end, + wait_op = function () + local deadline, due = session_next_deadline(self) + local dt = deadline - fibers.now() + if dt < 0 then dt = 0 end + return fibers.named_choice { + frame = self._frame_rx:recv_op():wrap(frame_event_from_item), + timer = sleep.sleep_op(dt):wrap(function () + return { kind = 'timer', due = due } + end), + } + end, + store_wake = function (which, ev) + if which == 'frame' and ev ~= nil then + self._pending_frame = ev + elseif which == 'timer' and ev ~= nil then + self._pending_timer = ev + end + end, + } +end + +local function route_downstream(self, lane, frame, at) + if lane == 'rpc' then + return publish_to(self._rpc_tx, session_frame_event(self, lane, frame, at), 'session_rpc_frame_admit_failed') + end + if lane == 'transfer' then + return publish_to(self._transfer_tx, session_frame_event(self, lane, frame, at), 'session_transfer_frame_admit_failed') + end + return true, nil +end + +local function record_bad_frame(self, at) + at = at or fibers.now() + local window_s = self._bad_frame_window_s + local cutoff = at - window_s + local kept = {} + + for _, seen_at in ipairs(self._bad_frame_times or {}) do + if type(seen_at) == 'number' and seen_at >= cutoff then + kept[#kept + 1] = seen_at + end + end + + kept[#kept + 1] = at + self._bad_frame_times = kept + + return #kept +end + +local function handle_wire_error(self, ev) + local at = (type(ev) == 'table' and ev.at) or fibers.now() + local err = (type(ev) == 'table' and ev.err) or 'wire_error' + local count = record_bad_frame(self, at) + + update_session(self, function (s) + s.wire_errors = (s.wire_errors or 0) + 1 + s.bad_frame_count = count + s.last_wire_error = tostring(err) + end) + + if count >= self._bad_frame_limit then + self._bad_frame_times = {} + reset_to_hello(self, 'bad_frame_limit', at, { rotate_local_sid = false }) + end +end + +local function handle_session_frame(self, checked, at) + local cur = session_snapshot(self) + if (checked.type == 'hello' or checked.type == 'hello_ack') + and not protocol.proto_supported(checked.proto) + then + reset_to_hello(self, 'unsupported_proto', at) + return + end + if checked.type == 'hello' then + if not cur.established or cur.phase == 'hello' or not same_peer(cur, checked) then + establish_from_peer(self, checked, at) + else + refresh_peer(self, checked, at) + end + send_hello_ack(self) + elseif checked.type == 'hello_ack' then + if not cur.established or cur.phase == 'hello' then + establish_from_peer(self, checked, at) + elseif same_peer(cur, checked) then + refresh_peer(self, checked, at) + end + elseif checked.type == 'ping' then + if cur.established and same_peer(cur, checked) then + refresh_peer(self, checked, at) + send_pong(self) + end + elseif checked.type == 'pong' then + if cur.established and same_peer(cur, checked) then + refresh_peer(self, checked, at) + end + end +end + +local function handle_non_session_frame(self, checked, lane, at) + local cur = session_snapshot(self) + if cur.established ~= true or cur.peer_sid == nil then return end + self._last_peer_at = at or fibers.now() + local ok, err = route_downstream(self, lane, checked, at) + if ok ~= true then error(err or 'session_downstream_frame_admit_failed', 0) end +end + +local function handle_frame(self, ev) + local checked, err = protocol.validate_wire(ev.frame) + if not checked then error('session invalid frame: ' .. tostring(err), 0) end + local lane = protocol.dispatch_lane(checked) + if lane == 'session_control' then + handle_session_frame(self, checked, ev.at or fibers.now()) + else + handle_non_session_frame(self, checked, lane, ev.at or fibers.now()) + end +end + +local function handle_timer(self, ev) + local now = fibers.now() + local cur = session_snapshot(self) + local due = type(ev) == 'table' and ev.due or nil + if cur.established ~= true then + if now >= (self._next_hello_at or 0) then send_hello(self) end + return + end + if now >= ((self._last_peer_at or now) + self._liveness_timeout) then + reset_to_hello(self, 'liveness_timeout', now, { rotate_local_sid = true }) + return + end + if (due == nil or due == 'ping') and now >= (self._next_ping_at or math.huge) then + send_ping(self) + end +end + +function M.run(scope, params) + if type(scope) ~= 'table' then error('fabric.session.run: scope required', 2) end + if type(params) ~= 'table' then error('fabric.session.run: params table required', 2) end + local frame_rx = require_rx(params.frame_rx, 'fabric.session: frame_rx', 2) + local tx_control = require_tx(params.tx_control, 'fabric.session: tx_control', 2) + local outbound = params.outbound + if type(outbound) ~= 'table' or type(outbound.send_frame_now) ~= 'function' then + error('fabric.session: outbound gate required', 2) + end + local rpc_tx = require_tx(params.rpc_tx, 'fabric.session: rpc_tx', 2) + local transfer_tx = require_tx(params.transfer_tx, 'fabric.session: transfer_tx', 2) + local link_id = params.link_id or 'link' + local link_generation = params.link_generation or 1 + local local_node = params.local_node or link_id + local initial = { + role = 'session', + link_id = link_id, + link_generation = link_generation, + phase = 'hello', + local_node = local_node, + local_sid = params.local_sid or tostring(uuid.new()), + peer_sid = nil, + peer_node = nil, + peer_identity_claim = nil, + peer_auth_claim = nil, + auth_state = 'unauthenticated', + authenticated = false, + proto = nil, + session_generation = params.session_generation or 0, + established = false, + why = nil, + wire_errors = 0, + bad_frame_count = 0, + last_wire_error = nil, + control_send_drops = 0, + last_control_send_drop = nil, + last_control_send_drop_at = nil, + } + + local session_model = model_mod.new(initial, { + copy = protocol.copy_reserved_claim, + equals = session_equal, + label = 'fabric.session', + }) + + scope:finally(function (_, status, primary) + local reason = primary or status or 'session closed' + session_model:terminate(reason) + outbound:terminate(reason) + end) + + local self = setmetatable({ + _link_id = link_id, + _link_generation = link_generation, + _frame_rx = frame_rx, + _tx_control = tx_control, + _outbound = outbound, + _rpc_tx = rpc_tx, + _transfer_tx = transfer_tx, + _session_model = session_model, + _local_node = local_node, + _identity_claim = protocol.normalise_reserved_claim(params.identity_claim), + _auth_claim = protocol.normalise_reserved_claim(params.auth_claim), + _auth_state = 'unauthenticated', + _authenticated = false, + _state_tx = params.state_tx, + _component_name = params.component_name or 'session', + _hello_interval = positive_number(params.hello_interval_s, DEFAULT_HELLO_INTERVAL, 'hello_interval_s'), + _ping_interval = positive_number(params.ping_interval_s, DEFAULT_PING_INTERVAL, 'ping_interval_s'), + _liveness_timeout = positive_number(params.liveness_timeout_s, DEFAULT_LIVENESS_TIMEOUT, 'liveness_timeout_s'), + _bad_frame_limit = positive_integer(params.bad_frame_limit, DEFAULT_BAD_FRAME_LIMIT, 'bad_frame_limit'), + _bad_frame_window_s = positive_number(params.bad_frame_window_s, DEFAULT_BAD_FRAME_WINDOW_S, 'bad_frame_window_s'), + _bad_frame_times = {}, + _next_hello_at = fibers.now(), + _next_ping_at = math.huge, + _last_peer_at = nil, + _frame_closed = false, + _pending_frame = nil, + _pending_timer = nil, + }, Session) + + publish_state(self) + send_hello(self) + + while true do + local ev = fibers.perform(session_event_op(self)) + if ev.kind == 'frame_closed' then + self._frame_closed = true + publish_session_drop(self, session_snapshot(self), 'frame_closed', fibers.now()) + self._outbound:drop('frame_closed') + return { + role = 'session', + link_id = link_id, + snapshot = session_snapshot(self), + reason = 'frame_closed', + } + elseif ev.kind == 'frame' then + handle_frame(self, ev) + elseif ev.kind == 'wire_error' then + handle_wire_error(self, ev) + elseif ev.kind == 'invalid_frame_item' then + error(ev.err or 'fabric.session invalid frame input', 0) + elseif ev.kind == 'timer' then + handle_timer(self, ev) + else + error('fabric.session unknown event: ' .. tostring(ev.kind), 0) + end + end +end + +M.peer_session_event = peer_session_event +M.peer_session_dropped_event = peer_session_dropped_event +M.session_frame_event = session_frame_event +M.Session = Session +M.OutboundGate = OutboundGate + +return M diff --git a/src/services/fabric/state.lua b/src/services/fabric/state.lua new file mode 100644 index 00000000..08d11f6a --- /dev/null +++ b/src/services/fabric/state.lua @@ -0,0 +1,456 @@ +-- services/fabric/state.lua +-- +-- Fabric retained-state projector. +-- +-- Components do not publish retained bus state through callbacks. They emit +-- snapshot events to this owner; this owner is the only Fabric module that maps +-- Fabric runtime snapshots onto the local retained state plane. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local queue = require 'devicecode.support.queue' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local topics = require 'services.fabric.topics' +local contracts = require 'devicecode.support.contracts' +local tablex = require 'shared.table' + +local M = {} + +local DEFAULT_QUEUE_LEN = 64 + +local shallow_copy = tablex.shallow_copy + +local function require_rx(v, name, level) + return contracts.require_rx(v, name, (level or 1) + 1) +end + +local function make_payload(kind, snapshot, opts) + opts = opts or {} + return { + kind = kind, + link_id = opts.link_id, + link_generation = opts.link_generation, + component = opts.component, + state = type(snapshot) == 'table' and snapshot.state or nil, + snapshot = type(snapshot) == 'table' and shallow_copy(snapshot) or snapshot, + ts = fibers.now(), + } +end + +function M.link_topic(link_id) + return topics.state_link(link_id) +end + +function M.component_topic(link_id, component) + return topics.state_link_component(link_id, component) +end + +function M.transfer_topic(xfer_id) + return topics.state_transfer(xfer_id) +end + +function M.link_snapshot_event(link_id, link_generation, snapshot) + return { + kind = 'link_snapshot', + link_id = link_id, + link_generation = link_generation, + snapshot = snapshot, + } +end + +function M.component_snapshot_event(link_id, link_generation, component, snapshot) + return { + kind = 'component_snapshot', + link_id = link_id, + link_generation = link_generation, + component = component, + snapshot = snapshot, + } +end + +function M.wire_trace_event(payload) + return { + kind = 'wire_trace', + payload = type(payload) == 'table' and shallow_copy(payload) or { value = payload }, + } +end + +function M.admit_wire_trace_now(tx, payload, label) + if tx == nil then return true, nil end + return queue.try_admit_required( + tx, + M.wire_trace_event(payload), + label or 'fabric_wire_trace_admit_failed' + ) +end + +function M.clear_link_event(link_id) + return { kind = 'clear_link', link_id = link_id } +end + +function M.clear_component_event(link_id, component) + return { kind = 'clear_component', link_id = link_id, component = component } +end + +function M.admit_link_snapshot_now(tx, link_id, link_generation, snapshot, label) + if tx == nil then return true, nil end + return queue.try_admit_required( + tx, + M.link_snapshot_event(link_id, link_generation, snapshot), + label or 'fabric_state_link_snapshot_admit_failed' + ) +end + +function M.admit_component_snapshot_now(tx, link_id, link_generation, component, snapshot, label) + if tx == nil then return true, nil end + return queue.try_admit_required( + tx, + M.component_snapshot_event(link_id, link_generation, component, snapshot), + label or 'fabric_state_component_snapshot_admit_failed' + ) +end + +function M.new_queue(len) + return mailbox.new(len or DEFAULT_QUEUE_LEN, { full = 'reject_newest' }) +end + + +local function compact_peer(snapshot) + if type(snapshot) ~= 'table' then return nil end + local peer = snapshot.peer_node or snapshot.peer_id or snapshot.peer_sid + if peer ~= nil and peer ~= '' then return tostring(peer) end + local claim = snapshot.peer_identity_claim + if type(claim) == 'table' then + return claim.node or claim.id or claim.name + end + return nil +end + +local function log_component_lifecycle(svc, obs_state, ev) + if type(svc) ~= 'table' or type(svc.info) ~= 'function' then return end + if ev.kind ~= 'component_snapshot' or ev.component ~= 'session' then return end + local snap = type(ev.snapshot) == 'table' and ev.snapshot or {} + obs_state.session_log_keys = obs_state.session_log_keys or {} + local established = snap.established == true + local phase = snap.phase or snap.state + local peer = compact_peer(snap) + local key = tostring(established) .. '|' .. tostring(phase or '') .. '|' .. tostring(peer or '') + local prev = obs_state.session_log_keys[ev.link_id] + if prev == key then return end + obs_state.session_log_keys[ev.link_id] = key + if established then + svc:info('fabric_session_established', { + summary = string.format('fabric session established link=%s%s', tostring(ev.link_id or 'link'), peer and (' peer=' .. tostring(peer)) or ''), + link_id = ev.link_id, + link_generation = ev.link_generation, + peer = peer, + phase = phase, + }) + elseif prev and prev:match('^true|') then + svc:warn('fabric_session_lost', { + summary = string.format('fabric session lost link=%s phase=%s', tostring(ev.link_id or 'link'), tostring(phase or 'unknown')), + link_id = ev.link_id, + link_generation = ev.link_generation, + phase = phase, + }) + end +end + +local function retain(conn, topic, payload, opts) + if conn == nil then return true, nil end + return bus_cleanup.retain(conn, topic, payload, opts) +end + +local function unretain(conn, topic, opts) + if conn == nil then return true, nil end + return bus_cleanup.unretain(conn, topic, opts) +end + + +local function transfer_corr_from_result(result) + result = type(result) == 'table' and result or {} + return { + job_id = result.job_id, + component = result.component, + image_id = result.image_id, + xfer_id = result.xfer_id, + request_id = result.request_id, + } +end + +local function is_transfer_component(name) + -- Historical code used component name "transfer"; current links publish the + -- manager snapshot under the component name "transfer_manager". Treat both + -- as the transfer manager for retained transfer topics and monitor events. + return name == 'transfer' or name == 'transfer_manager' +end + +local function transfer_payload_from_snapshot(link_id, link_generation, snapshot) + if type(snapshot) ~= 'table' then return nil end + local rec = type(snapshot.active) == 'table' and snapshot.active or type(snapshot.last) == 'table' and snapshot.last or nil + if type(rec) ~= 'table' then return nil end + local result = type(rec.result) == 'table' and rec.result or {} + local xfer_id = rec.xfer_id or result.xfer_id + if type(xfer_id) ~= 'string' or xfer_id == '' then return nil end + local meta = type(rec.meta) == 'table' and rec.meta or {} + local corr = transfer_corr_from_result(result) + corr.job_id = corr.job_id or meta.job_id + corr.component = corr.component or meta.component + corr.image_id = corr.image_id or meta.image_id + corr.xfer_id = xfer_id + corr.request_id = corr.request_id or rec.request_id or result.request_id + return { + kind = 'fabric.transfer', + link_id = link_id, + link_generation = link_generation, + xfer_id = xfer_id, + request_id = rec.request_id or result.request_id, + direction = rec.direction, + state = rec.status, + status = rec.status, + target = rec.target or result.target, + size = result.size or rec.size, + sent_bytes = result.sent_bytes, + received_bytes = result.received_bytes, + digest_alg = result.digest_alg or rec.digest_alg, + digest = result.digest or rec.digest, + retransmits = result.retransmits, + chunk_retries = result.chunk_retries, + chunks_sent = result.chunks_sent, + max_frame_queue_ms = result.max_frame_queue_ms, + max_need_to_chunk_ms = result.max_need_to_chunk_ms, + max_source_read_ms = result.max_source_read_ms, + max_send_ms = result.max_send_ms, + progress = type(rec.progress) == 'table' and shallow_copy(rec.progress) or nil, + error = rec.primary, + correlation = corr, + ts = fibers.now(), + } +end + +local function transfer_bytes(progress, payload) + progress = type(progress) == 'table' and progress or {} + payload = type(payload) == 'table' and payload or {} + return payload.sent_bytes + or payload.received_bytes + or progress.sent_bytes + or progress.received_bytes + or progress.last_tx_next + or progress.requested_next + or progress.pending_next + or progress.last_rx_next +end + +local function field(v) + if v == nil then return '-' end + return tostring(v) +end + +local function pct(bytes, total) + if type(bytes) ~= 'number' or type(total) ~= 'number' or total <= 0 then return nil end + return math.floor((bytes * 10000 / total) + 0.5) / 100 +end + +local function compact_transfer_obs_payload(payload) + if type(payload) ~= 'table' then return nil end + local progress = type(payload.progress) == 'table' and payload.progress or {} + local bytes = transfer_bytes(progress, payload) + local total = payload.size or progress.size or progress.total_bytes + local rx = field(progress.last_rx_type) .. ':' .. field(progress.last_rx_next) + local tx = field(progress.last_tx_type) .. ':' .. field(progress.last_tx_offset) .. '->' .. field(progress.last_tx_next) + local out = { + kind = 'fabric.transfer_progress', + xfer_id = payload.xfer_id, + request_id = payload.request_id, + link_id = payload.link_id, + link_generation = payload.link_generation, + target = payload.target, + phase = progress.phase or payload.status or payload.state, + state = payload.state, + status = payload.status, + event = progress.event, + bytes_transferred = bytes, + total_bytes = total, + percent = pct(bytes, total), + last_rx_type = progress.last_rx_type, + last_rx_next = progress.last_rx_next, + last_tx_type = progress.last_tx_type, + last_tx_offset = progress.last_tx_offset, + last_tx_next = progress.last_tx_next, + chunk_len = progress.chunk_len, + chunk_digest = progress.chunk_digest, + chunk_frame_len = progress.chunk_frame_len, + chunks_sent = progress.chunks_sent or payload.chunks_sent, + retransmits = progress.retransmits or payload.retransmits, + retransmit = progress.retransmit, + retry = progress.retry, + retry_reason = progress.reason, + frame_queue_ms = progress.frame_queue_ms, + need_to_chunk_ms = progress.need_to_chunk_ms, + source_read_ms = progress.source_read_ms, + send_ms = progress.send_ms, + max_frame_queue_ms = payload.max_frame_queue_ms or progress.max_frame_queue_ms, + max_need_to_chunk_ms = payload.max_need_to_chunk_ms or progress.max_need_to_chunk_ms, + max_source_read_ms = payload.max_source_read_ms or progress.max_source_read_ms, + max_send_ms = payload.max_send_ms or progress.max_send_ms, + error = payload.error or progress.err, + job_id = type(payload.correlation) == 'table' and payload.correlation.job_id or nil, + component = type(payload.correlation) == 'table' and payload.correlation.component or nil, + image_id = type(payload.correlation) == 'table' and payload.correlation.image_id or nil, + ts = fibers.now(), + } + out.summary = string.format( + 'xfer=%s phase=%s bytes=%s/%s rx=%s tx=%s len=%s digest=%s frame_len=%s retransmit=%s retry=%s reason=%s need_to_chunk_ms=%s frame_queue_ms=%s source_read_ms=%s send_ms=%s', + field(out.xfer_id), field(out.phase), field(bytes), field(total), rx, tx, + field(out.chunk_len), field(out.chunk_digest), field(out.chunk_frame_len), field(out.retransmit), + field(out.retry), field(out.retry_reason), + field(out.need_to_chunk_ms), field(out.frame_queue_ms), field(out.source_read_ms), field(out.send_ms) + ) + return out +end + +local function transfer_obs_key(payload) + if type(payload) ~= 'table' then return nil end + local progress = type(payload.progress) == 'table' and payload.progress or {} + return table.concat({ + field(payload.xfer_id), + field(payload.state or payload.status), + field(progress.event), + field(transfer_bytes(progress, payload)), + field(progress.last_rx_type), + field(progress.last_rx_next), + field(progress.last_tx_type), + field(progress.last_tx_offset), + field(progress.last_tx_next), + field(progress.chunk_len), + field(progress.chunk_digest), + field(progress.chunk_frame_len), + field(progress.chunks_sent or payload.chunks_sent), + field(progress.retransmits or payload.retransmits), + field(progress.retransmit), + field(progress.retry), + field(progress.reason), + field(payload.error or progress.err), + }, '|') +end + +local function publish_transfer_obs(conn, obs_state, payload) + if conn == nil then return true, nil end + local compact = compact_transfer_obs_payload(payload) + if compact == nil or type(compact.xfer_id) ~= 'string' or compact.xfer_id == '' then return true, nil end + local key = transfer_obs_key(payload) + obs_state.transfer_keys = obs_state.transfer_keys or {} + if obs_state.transfer_keys[compact.xfer_id] == key then return true, nil end + obs_state.transfer_keys[compact.xfer_id] = key + conn:publish({ 'obs', 'event', 'fabric', 'transfer_progress' }, compact) + conn:publish({ 'obs', 'v1', 'fabric', 'event', 'transfer_progress' }, compact) + return true, nil +end + +local function handle_event(conn, ev, obs_state, svc) + obs_state = obs_state or {} + if ev.kind == 'link_snapshot' then + return retain( + conn, + M.link_topic(ev.link_id), + make_payload('fabric.link', ev.snapshot, { + link_id = ev.link_id, + link_generation = ev.link_generation, + }) + ) + + elseif ev.kind == 'component_snapshot' then + log_component_lifecycle(svc, obs_state, ev) + local ok, err = retain( + conn, + M.component_topic(ev.link_id, ev.component), + make_payload('fabric.component', ev.snapshot, { + link_id = ev.link_id, + link_generation = ev.link_generation, + component = ev.component, + }) + ) + if ok ~= true then return ok, err end + if is_transfer_component(ev.component) then + local payload = transfer_payload_from_snapshot(ev.link_id, ev.link_generation, ev.snapshot) + if payload ~= nil then + local rok, rerr = retain(conn, M.transfer_topic(payload.xfer_id), payload) + if rok ~= true then return rok, rerr end + return publish_transfer_obs(conn, obs_state, payload) + end + end + return true, nil + + elseif ev.kind == 'wire_trace' then + local payload = type(ev.payload) == 'table' and shallow_copy(ev.payload) or { value = ev.payload } + payload.kind = payload.kind or 'fabric.wire' + payload.ts = payload.ts or fibers.now() + if conn ~= nil then + conn:publish({ 'obs', 'event', 'fabric', 'wire' }, payload) + conn:publish({ 'obs', 'v1', 'fabric', 'event', 'wire' }, payload) + end + return true, nil + + elseif ev.kind == 'clear_link' then + return unretain(conn, M.link_topic(ev.link_id)) + + elseif ev.kind == 'clear_component' then + return unretain(conn, M.component_topic(ev.link_id, ev.component)) + end + + return nil, 'fabric state projector unknown event: ' .. tostring(ev.kind) +end + +function M.run_projector(scope, params) + if type(scope) ~= 'table' then + error('fabric.state.run_projector: scope required', 2) + end + if type(params) ~= 'table' then + error('fabric.state.run_projector: params table required', 2) + end + + local rx = require_rx(params.state_rx, 'fabric.state: state_rx', 2) + local conn = params.conn + local count = 0 + local obs_state = { transfer_keys = {} } + + while true do + local ev = fibers.perform(rx:recv_op()) + if ev == nil then + return { role = 'state_projector', published = count, reason = rx.why and rx:why() or 'closed' } + end + + local ok, err = handle_event(conn, ev, obs_state, params.svc) + if ok ~= true then + error(err or 'fabric state projection failed', 0) + end + count = count + 1 + end +end + +-- Immediate helpers retained for tests and administrative cleanup. Core Fabric +-- components should use the projector event surface above. +function M.publish_link(conn, link_id, link_generation, snapshot, opts) + opts = shallow_copy(opts or {}) + opts.link_id = link_id + opts.link_generation = link_generation + return retain(conn, M.link_topic(link_id), make_payload('fabric.link', snapshot, opts)) +end + +function M.publish_component(conn, link_id, link_generation, component, snapshot, opts) + opts = shallow_copy(opts or {}) + opts.link_id = link_id + opts.link_generation = link_generation + opts.component = component + return retain(conn, M.component_topic(link_id, component), make_payload('fabric.component', snapshot, opts)) +end + +function M.clear_link(conn, link_id) + return unretain(conn, M.link_topic(link_id)) +end + +function M.clear_component(conn, link_id, component) + return unretain(conn, M.component_topic(link_id, component)) +end + +return M diff --git a/src/services/fabric/topics.lua b/src/services/fabric/topics.lua new file mode 100644 index 00000000..15b4be84 --- /dev/null +++ b/src/services/fabric/topics.lua @@ -0,0 +1,381 @@ +-- services/fabric/topics.lua +-- +-- Pure Fabric topic helpers. +-- +-- This module must remain pure: +-- * no fibers.perform +-- * no scopes +-- * no queues +-- * no bus calls +-- +-- It owns only topic construction and lightweight topic validation. + +local topicx = require 'shared.topic' + +local M = {} + +local function scalar_ok(v) + local tv = type(v) + + if tv == 'string' then + return v ~= '' + end + + if tv == 'number' then + return v == v and v ~= math.huge and v ~= -math.huge + end + + return false +end + +local append = topicx.append + +function M.append(base, ...) + if type(base) ~= 'table' then + error('topics.append: base must be a table', 2) + end + + return append(base, ...) +end + +function M.validate(topic) + if type(topic) ~= 'table' then + return nil, 'topic_must_be_table' + end + + local max_i = 0 + local count = 0 + + for k in pairs(topic) do + if type(k) ~= 'number' + or k < 1 + or k % 1 ~= 0 + then + return nil, 'topic_must_be_dense_array' + end + + if k > max_i then + max_i = k + end + + count = count + 1 + end + + if count ~= max_i then + return nil, 'topic_must_be_dense_array' + end + + for i = 1, max_i do + if not scalar_ok(topic[i]) then + return nil, 'invalid_topic_token' + end + end + + return topic, nil +end + +--- Return a collision-safe string key for a literal Fabric topic. +--- +--- The encoding distinguishes token type and token length, so topics such as +--- { "a/b" } and { "a", "b" } cannot collide. +function M.key(topic) + local checked, err = M.validate(topic) + if not checked then + error('topics.key: ' .. tostring(err), 2) + end + + local parts = {} + for i = 1, #checked do + local v = checked[i] + + if type(v) == 'string' then + parts[#parts + 1] = 's' .. #v .. ':' .. v + else + local sv = tostring(v) + parts[#parts + 1] = 'n' .. #sv .. ':' .. sv + end + end + + return table.concat(parts, '|') +end + + +function M.copy(topic) + local checked, err = M.validate(topic) + if not checked then + error('topics.copy: ' .. tostring(err), 2) + end + + local out = {} + for i = 1, #checked do + out[i] = checked[i] + end + return out +end + +function M.starts_with(topic, prefix) + local checked_topic, terr = M.validate(topic) + if not checked_topic then + error('topics.starts_with: topic ' .. tostring(terr), 2) + end + + local checked_prefix, perr = M.validate(prefix) + if not checked_prefix then + error('topics.starts_with: prefix ' .. tostring(perr), 2) + end + + if #checked_prefix > #checked_topic then + return false + end + + for i = 1, #checked_prefix do + if checked_topic[i] ~= checked_prefix[i] then + return false + end + end + + return true +end + +function M.replace_prefix(topic, from_prefix, to_prefix) + local checked_topic, terr = M.validate(topic) + if not checked_topic then + error('topics.replace_prefix: topic ' .. tostring(terr), 2) + end + + local checked_from, ferr = M.validate(from_prefix) + if not checked_from then + error('topics.replace_prefix: from_prefix ' .. tostring(ferr), 2) + end + + local checked_to, toerr = M.validate(to_prefix) + if not checked_to then + error('topics.replace_prefix: to_prefix ' .. tostring(toerr), 2) + end + + if not M.starts_with(checked_topic, checked_from) then + return nil, nil + end + + local out = {} + for i = 1, #checked_to do + out[#out + 1] = checked_to[i] + end + for i = #checked_from + 1, #checked_topic do + out[#out + 1] = checked_topic[i] + end + + return out, nil +end + +local function exact_match(mapped_to, wanted, topic) + if #wanted ~= #topic then + return nil, nil + end + + for i = 1, #topic do + if topic[i] ~= wanted[i] then + return nil, nil + end + end + + return M.copy(mapped_to), nil +end + +local function match_rule(rule, topic, from_field, to_field) + if type(rule) ~= 'table' then + error('topics.match_rule: rule must be a table', 3) + end + + local checked_topic, terr = M.validate(topic) + if not checked_topic then + error('topics.match_rule: topic ' .. tostring(terr), 3) + end + + local to_topic = rule[to_field] + local checked_to, toerr = M.validate(to_topic) + if not checked_to then + error('topics.match_rule: ' .. tostring(to_field) .. ' ' .. tostring(toerr), 3) + end + + if rule.topic then + local wanted, werr = M.validate(rule.topic) + if not wanted then + error('topics.match_rule: topic field ' .. tostring(werr), 3) + end + local mapped = exact_match(checked_to, wanted, checked_topic) + if mapped then + return mapped, rule + end + return nil, nil + end + + local from_topic = rule[from_field] + local checked_from, ferr = M.validate(from_topic) + if not checked_from then + error('topics.match_rule: ' .. tostring(from_field) .. ' ' .. tostring(ferr), 3) + end + + local mapped = M.replace_prefix(checked_topic, checked_from, checked_to) + if mapped then + return mapped, rule + end + + return nil, nil +end + +local function match_rule_set(rules, topic, from_field, to_field) + if rules == nil then + return nil, nil + end + if type(rules) ~= 'table' then + error('topics.match_rule_set: rules must be a table', 3) + end + + for i = 1, #rules do + local mapped, rule = match_rule(rules[i], topic, from_field, to_field) + if mapped then + return mapped, rule + end + end + + return nil, nil +end + +function M.map_local_to_remote(rules, topic) + return match_rule_set(rules, topic, 'local_prefix', 'remote_prefix') +end + +function M.map_remote_to_local(rules, topic) + return match_rule_set(rules, topic, 'remote_prefix', 'local_prefix') +end + +function M.map_local_to_remote_rule(rule, topic) + return match_rule(rule, topic, 'local_prefix', 'remote_prefix') +end + +function M.map_remote_to_local_rule(rule, topic) + return match_rule(rule, topic, 'remote_prefix', 'local_prefix') +end + + +local function match_exact_rule(rule, topic, from_field, to_field) + if type(rule) ~= 'table' then + error('topics.match_exact_rule: rule must be a table', 3) + end + + local checked_topic, terr = M.validate(topic) + if not checked_topic then + error('topics.match_exact_rule: topic ' .. tostring(terr), 3) + end + + local from_topic, ferr = M.validate(rule[from_field]) + if not from_topic then + error('topics.match_exact_rule: ' .. tostring(from_field) .. ' ' .. tostring(ferr), 3) + end + + local to_topic, toerr = M.validate(rule[to_field]) + if not to_topic then + error('topics.match_exact_rule: ' .. tostring(to_field) .. ' ' .. tostring(toerr), 3) + end + + if #checked_topic ~= #from_topic then + return nil, nil + end + + for i = 1, #from_topic do + if checked_topic[i] ~= from_topic[i] then + return nil, nil + end + end + + return M.copy(to_topic), rule +end + +local function match_exact_rule_set(rules, topic, from_field, to_field) + if rules == nil then + return nil, nil + end + if type(rules) ~= 'table' then + error('topics.match_exact_rule_set: rules must be a table', 3) + end + + for i = 1, #rules do + local mapped, rule = match_exact_rule(rules[i], topic, from_field, to_field) + if mapped then + return mapped, rule + end + end + + return nil, nil +end + +function M.map_local_call_to_remote(rules, topic) + return match_exact_rule_set(rules, topic, 'local_topic', 'remote_topic') +end + +function M.map_remote_call_to_local(rules, topic) + return match_exact_rule_set(rules, topic, 'remote_topic', 'local_topic') +end + +function M.map_local_call_to_remote_rule(rule, topic) + return match_exact_rule(rule, topic, 'local_topic', 'remote_topic') +end + +function M.map_remote_call_to_local_rule(rule, topic) + return match_exact_rule(rule, topic, 'remote_topic', 'local_topic') +end + +-------------------------------------------------------------------------------- +-- Service-level topics +-------------------------------------------------------------------------------- + +function M.svc_status() + return { 'svc', 'fabric', 'status' } +end + +function M.svc_meta() + return { 'svc', 'fabric', 'meta' } +end + +function M.cfg() + return { 'cfg', 'fabric' } +end + +-------------------------------------------------------------------------------- +-- Fabric state plane +-------------------------------------------------------------------------------- + +function M.state_root() + return { 'state', 'fabric' } +end + +function M.state_link(link_id, ...) + return append({ 'state', 'fabric', 'link', link_id }, ...) +end + +function M.state_link_component(link_id, component) + return append({ 'state', 'fabric', 'link', link_id, 'component' }, component) +end + +function M.state_transfer(xfer_id) + return { 'state', 'fabric', 'transfer', xfer_id } +end + +-------------------------------------------------------------------------------- +-- Stable public Fabric interfaces +-------------------------------------------------------------------------------- + +function M.transfer_manager_meta(id) + return { 'cap', 'transfer-manager', id or 'main', 'meta' } +end + +function M.transfer_manager_status(id) + return { 'cap', 'transfer-manager', id or 'main', 'status' } +end + +function M.transfer_manager_rpc(method, id) + return { 'cap', 'transfer-manager', id or 'main', 'rpc', method } +end + +return M diff --git a/src/services/fabric/trace.lua b/src/services/fabric/trace.lua new file mode 100644 index 00000000..8e7a0597 --- /dev/null +++ b/src/services/fabric/trace.lua @@ -0,0 +1,115 @@ +-- services/fabric/trace.lua +-- +-- Lightweight opt-in Fabric wire/event tracing. Disabled unless +-- DEVICECODE_FABRIC_TRACE or FABRIC_TRACE is set to a truthy value. +-- Trace events are published through the Fabric state projector to: +-- obs/event/fabric/wire +-- obs/v1/fabric/event/wire +-- Payloads intentionally exclude frame bodies unless preview is explicitly on. + +local fibers = require 'fibers' + +local M = {} + +local function truthy(v) + if v == nil or v == '' then return false end + v = tostring(v):lower() + return not (v == '0' or v == 'false' or v == 'no' or v == 'off') +end + +local function split_set(v) + local out = {} + if type(v) ~= 'string' or v == '' then return out end + for token in v:gmatch('[^,%s]+') do out[token] = true end + return out +end + +M.enabled = truthy(os.getenv('DEVICECODE_FABRIC_TRACE') or os.getenv('FABRIC_TRACE')) +M.preview_enabled = truthy(os.getenv('DEVICECODE_FABRIC_TRACE_PREVIEW') or os.getenv('FABRIC_TRACE_PREVIEW')) +M.types = split_set(os.getenv('DEVICECODE_FABRIC_TRACE_TYPES') or os.getenv('FABRIC_TRACE_TYPES')) + +local function type_allowed(frame_type) + if not M.enabled then return false end + if next(M.types) == nil then return true end + return M.types[frame_type or ''] == true or M.types['*'] == true or M.types.all == true +end + +local function topic_path(topic) + if type(topic) ~= 'table' then return nil end + local out = {} + for i, v in ipairs(topic) do out[i] = tostring(v) end + return table.concat(out, '/') +end + +local function preview(frame) + if not M.preview_enabled then return nil end + local ok, cjson = pcall(require, 'cjson.safe') + if not ok then return nil end + local s = cjson.encode(frame) + if type(s) ~= 'string' then return nil end + if #s > 240 then s = s:sub(1, 240) .. '...' end + return s +end + +local function frame_summary(frame) + local f = type(frame) == 'table' and frame or {} + local t = f.type + return { + frame_type = t, + sid = f.sid, + node = f.node, + xfer_id = f.xfer_id, + offset = f.offset, + next = f.next, + size = f.size, + target = f.target, + call_id = f.id, + ok = f.ok, + err = f.err, + topic = topic_path(f.topic), + data_len = type(f.data) == 'string' and #f.data or nil, + chunk_digest = f.chunk_digest, + preview = preview(frame), + } +end + +local function emit(state_tx, payload) + if state_tx == nil then return true, nil end + local ok, state_mod = pcall(require, 'services.fabric.state') + if not ok then return nil, state_mod end + return state_mod.admit_wire_trace_now(state_tx, payload, 'fabric_trace_admit_failed') +end + +function M.frame(state_tx, base, direction, frame, extra) + local t = type(frame) == 'table' and frame.type or nil + if not type_allowed(t) then return true, nil end + local payload = { + direction = direction, + component = base and base.component, + link_id = base and base.link_id, + link_generation = base and base.link_generation, + lane = extra and extra.lane, + ts = fibers.now(), + } + local s = frame_summary(frame) + for k, v in pairs(s) do payload[k] = v end + for k, v in pairs(extra or {}) do payload[k] = v end + return emit(state_tx, payload) +end + +function M.error(state_tx, base, direction, err, extra) + if not M.enabled then return true, nil end + local payload = { + direction = direction, + component = base and base.component, + link_id = base and base.link_id, + link_generation = base and base.link_generation, + err = tostring(err or 'unknown'), + event = extra and extra.event or 'wire_error', + ts = fibers.now(), + } + for k, v in pairs(extra or {}) do payload[k] = v end + return emit(state_tx, payload) +end + +return M diff --git a/src/services/fabric/transfer.lua b/src/services/fabric/transfer.lua new file mode 100644 index 00000000..c218bd99 --- /dev/null +++ b/src/services/fabric/transfer.lua @@ -0,0 +1,1064 @@ +-- services/fabric/transfer.lua +-- +-- Session-aware transfer manager. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' + +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local service_events = require 'devicecode.support.service_events' +local priority_event = require 'devicecode.support.priority_event' +local model_mod = require 'services.fabric.model' +local resource = require 'devicecode.support.resource' +local session_mod = require 'services.fabric.session' +local state_mod = require 'services.fabric.state' +local protocol = require 'services.fabric.protocol' +local transfer_sender = require 'services.fabric.transfer_sender' +local transfer_receive = require 'services.fabric.transfer_receive' + +local M = {} + +local DEFAULT_DONE_QUEUE = 64 +local DEFAULT_ATTEMPT_FRAME_QUEUE = 16 + +local Manager = {} +Manager.__index = Manager + +local SlotLease = {} +SlotLease.__index = SlotLease + +local function copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +local function ctx(v) + return v and session_mod.copy_context(v) or nil +end + +local function ev_ctx(ev) + return session_mod.context_from_event(ev) +end + +local function same_ctx(a, b) + return session_mod.same_session(a, b) +end + +local function require_ctx(v, where) + local c = ctx(v) + if type(c) ~= 'table' or type(c.session_generation) ~= 'number' then + error(where .. ': session context required', 2) + end + if type(c.peer_sid) ~= 'string' or c.peer_sid == '' then + error(where .. ': peer_sid required', 2) + end + return c +end + +local function req_id(req) + return req.request_id +end + +local function req_gen(req) + return req.request_generation or 1 +end + +local function copy_active(a) + if not a then return nil end + return { + request_id = a.request_id, + request_generation = a.request_generation, + session = ctx(a.session), + status = a.status, + direction = a.direction, + xfer_id = a.xfer_id, + target = a.target, + meta = copy(a.meta), + size = a.size, + digest_alg = a.digest_alg, + digest = a.digest, + progress = copy(a.progress), + } + +end + +local function copy_done(c) + if not c then return nil end + local out = copy(c) + if type(out.result) == 'table' then out.result = copy(out.result) end + return out +end + +local function stat_equal(a, b) + local x, y = a or {}, b or {} + for k, v in pairs(x) do if y[k] ~= v then return false end end + for k in pairs(y) do if x[k] == nil then return false end end + return true +end + +local function snapshot_equal(a, b) + if a == b then return true end + if type(a) ~= 'table' or type(b) ~= 'table' then return false end + if a.manager_id ~= b.manager_id then return false end + + local aa, ba = a.active, b.active + if (aa == nil) ~= (ba == nil) then return false end + if aa and ba then + if aa.request_id ~= ba.request_id then return false end + if aa.request_generation ~= ba.request_generation then return false end + if aa.status ~= ba.status then return false end + if aa.direction ~= ba.direction then return false end + if aa.xfer_id ~= ba.xfer_id then return false end + if not same_ctx(aa.session, ba.session) then return false end + if not stat_equal(aa.progress, ba.progress) then return false end + end + + local al, bl = a.last, b.last + if (al == nil) ~= (bl == nil) then return false end + if al and bl then + if al.request_id ~= bl.request_id then return false end + if al.request_generation ~= bl.request_generation then return false end + if al.status ~= bl.status then return false end + if al.primary ~= bl.primary then return false end + end + + return stat_equal(a.stats, b.stats) +end + +function M.new_state(opts) + opts = opts or {} + + return { + manager_id = opts.manager_id or 'transfer-manager', + active = nil, + last = nil, + stats = { + frames_received = 0, + frames_routed = 0, + accepted = 0, + received = 0, + rejected_busy = 0, + unsupported_target = 0, + completed = 0, + stale = 0, + failed = 0, + cancelled = 0, + released = 0, + deferred_no_session = 0, + }, + } +end + +function M.snapshot(state) + return { + manager_id = state.manager_id, + active = copy_active(state.active), + last = copy_done(state.last), + stats = copy(state.stats), + } +end + +local function active_matches(state, ev) + local a = state.active + return a ~= nil + and a.request_id == ev.request_id + and a.request_generation == ev.request_generation + and same_ctx(a.session, ev_ctx(ev)) +end + +function M.claim_slot(state, rec) + if state.active ~= nil then + state.stats.rejected_busy = state.stats.rejected_busy + 1 + return false, 'slot_busy' + end + + local id = rec.request_id + if type(id) ~= 'string' or id == '' then + error('transfer.claim_slot: request_id required', 2) + end + if type(rec.request_generation) ~= 'number' then + error('transfer.claim_slot: request_generation required', 2) + end + if type(rec.xfer_id) ~= 'string' or rec.xfer_id == '' then + error('transfer.claim_slot: xfer_id required', 2) + end + + state.active = { + request_id = id, + request_generation = rec.request_generation, + session = require_ctx(rec.session, 'transfer.claim_slot'), + status = rec.status or 'leased', + direction = rec.direction or 'send', + xfer_id = rec.xfer_id, + target = rec.target, + meta = copy(rec.meta), + size = rec.size, + digest_alg = rec.digest_alg, + digest = rec.digest, + frame_tx = rec.frame_tx, + lease = rec.lease, + progress = { + phase = rec.status or 'leased', + event = 'slot_claimed', + sent_bytes = 0, + chunks_sent = 0, + retransmits = 0, + at = fibers.now(), + }, + } + state.stats.accepted = state.stats.accepted + 1 + + return true, nil +end + +function M.release_slot(state, ev) + if not active_matches(state, ev) then + state.stats.stale = state.stats.stale + 1 + return false, 'stale_slot_release' + end + + local active = state.active + state.last = { + kind = ev.kind or 'transfer_slot_released', + request_id = ev.request_id, + request_generation = ev.request_generation, + session = ctx(ev_ctx(ev)), + status = 'released', + primary = ev.primary or ev.reason, + progress = active and copy(active.progress) or nil, + } + state.active = nil + state.stats.released = state.stats.released + 1 + + return true, nil, active, state.last +end + +function M.apply_attempt_done(state, ev) + if not active_matches(state, ev) then + state.stats.stale = state.stats.stale + 1 + return false, 'stale_attempt_completion' + end + + local active = state.active + state.last = { + kind = ev.kind or 'transfer_attempt_done', + request_id = ev.request_id, + request_generation = ev.request_generation, + session = ctx(ev_ctx(ev)), + status = ev.status, + report = ev.report, + result = ev.result, + primary = ev.primary, + progress = active and copy(active.progress) or nil, + } + state.active = nil + state.stats.completed = state.stats.completed + 1 + + if ev.status == 'failed' then + state.stats.failed = state.stats.failed + 1 + elseif ev.status == 'cancelled' then + state.stats.cancelled = state.stats.cancelled + 1 + end + + return true, nil, active, state.last +end + +local function manager_snapshot(self) + return M.snapshot(self._state) +end + +local function emit_model(self, force) + if not self._model then return end + + local snap = manager_snapshot(self) + local changed = self._model:set_snapshot(snap) + + if (force or changed) and self._state_tx then + state_mod.admit_component_snapshot_now( + self._state_tx, + self._link_id, + self._link_generation, + self._component_name, + snap, + 'fabric_transfer_state_admit_failed' + ) + end +end + +local function close_feed(active, reason) + if active and active.frame_tx and type(active.frame_tx.close) == 'function' then + active.frame_tx:close(reason or 'transfer attempt closed') + end +end + +local function report(self, ev, label) + if self._events_port and type(self._events_port.emit_required) == 'function' then + return self._events_port:emit_required(ev, label or 'transfer_report_failed') + end + return queue.try_admit_required(self._done_tx, ev, label or 'transfer_report_failed') +end + +local function attempt_identity(req) + return { + kind = 'transfer_attempt_done', + request_id = req_id(req), + request_generation = req_gen(req), + session = ctx(req.session), + } +end + +local function release_identity(lease, reason) + return { + kind = 'transfer_slot_released', + request_id = lease._request_id, + request_generation = lease._request_generation, + session = ctx(lease._session), + xfer_id = lease._xfer_id, + reason = reason, + primary = reason, + } +end + +local function attempt_caps(self, frame_rx, session) + local c = ctx(session or self._session) + local outbound = self._outbound + + return { + manager_id = self._state.manager_id, + frame_rx = frame_rx, + session = ctx(c), + chunk_size = self._chunk_size, + timeout_s = self._timeout_s, + retry_limit = self._retry_limit, + + report_progress_now = function (ev) + ev = copy(ev or {}) + ev.kind = 'transfer_progress' + ev.request_id = ev.request_id or (self._state.active and self._state.active.request_id) + ev.request_generation = ev.request_generation or (self._state.active and self._state.active.request_generation) + ev.session = ctx(c) + ev.at = ev.at or fibers.now() + return report(self, ev, 'transfer_progress_report_failed') + end, + + send_control_frame_now = function (frame, label) + return outbound:send_transfer_control_frame_now(c, frame, label) + end, + + send_bulk_frame_now = function (frame, label) + return outbound:send_transfer_bulk_frame_now(c, frame, label) + end, + } +end + +local function run_attempt(scope, req, caps) + local owner = req.source_owner + if type(owner) ~= 'table' or type(owner.handoff) ~= 'function' then + error('transfer attempt requires source_owner', 0) + end + + local source, err = owner:handoff(function (value) + scope:finally(function (_, status, primary) + resource.terminate_checked( + value, + primary or status or 'transfer attempt closed', + 'transfer source cleanup failed' + ) + end) + return true + end) + if source == nil then error(err or 'source handoff failed', 0) end + + local worker_req = copy(req) + worker_req.source = source + worker_req.source_owner = nil + + local result = transfer_sender.run(scope, worker_req, caps) + if type(result) ~= 'table' then error('transfer attempt must return a result table', 0) end + return result +end + + +local function resolve_receive_target(self, target) + local targets = self._receive_targets + if type(targets) ~= 'table' then return nil end + return targets[target] +end + +local function send_abort_now(self, session, xfer_id, reason) + local c = ctx(session) + local frame, ferr = protocol.xfer_abort(xfer_id, tostring(reason or 'aborted')) + if not frame then return nil, ferr end + return self._outbound:send_transfer_control_frame_now(c, frame, 'transfer_receive_abort_send_failed') +end + +local function receive_attempt_identity(req) + return { + kind = 'transfer_attempt_done', + request_id = req_id(req), + request_generation = req_gen(req), + session = ctx(req.session), + } +end + +local function run_receive_attempt(scope, req, caps) + return transfer_receive.run(scope, req, caps) +end + +local function start_receive_attempt(self, ev) + local frame = ev.frame + local session = require_ctx(ev_ctx(ev), 'transfer receive begin') + local target = resolve_receive_target(self, frame.target) + + if self._state.active ~= nil then + self._state.stats.rejected_busy = self._state.stats.rejected_busy + 1 + send_abort_now(self, session, frame.xfer_id, 'busy') + emit_model(self) + return + end + + if target == nil then + self._state.stats.unsupported_target = self._state.stats.unsupported_target + 1 + send_abort_now(self, session, frame.xfer_id, 'unsupported_target') + emit_model(self) + return + end + + local frame_tx, frame_rx = mailbox.new(self._attempt_frame_queue_len, { full = 'reject_newest' }) + local request_id = 'receive:' .. tostring(frame.xfer_id) + local rec = { + request_id = request_id, + request_generation = 1, + session = session, + xfer_id = frame.xfer_id, + frame_tx = frame_tx, + frame_rx = frame_rx, + status = 'receiving', + direction = 'receive', + target = frame.target, + meta = frame.meta, + size = frame.size, + digest_alg = frame.digest_alg, + digest = frame.digest, + } + + local ok, reason = M.claim_slot(self._state, rec) + if not ok then + frame_tx:close(reason or 'slot_busy') + send_abort_now(self, session, frame.xfer_id, reason or 'slot_busy') + emit_model(self) + return + end + + self._state.stats.received = self._state.stats.received + 1 + + local req = { + request_id = request_id, + request_generation = 1, + session = session, + xfer_id = frame.xfer_id, + initial_frame = frame, + target = target, + timeout_s = self._timeout_s, + } + + local raw, serr = scoped_work.start { + lifetime_scope = self._scope, + reaper_scope = self._scope, + report_scope = self._scope, + identity = receive_attempt_identity(req), + run = function (attempt_scope) + return run_receive_attempt(attempt_scope, req, attempt_caps(self, frame_rx, session)) + end, + report = function (done_ev) + return report(self, done_ev, 'transfer_receive_attempt_report_failed') + end, + } + + if not raw then + self._state.active = nil + frame_tx:close(serr or 'transfer_receive_start_failed') + send_abort_now(self, session, frame.xfer_id, serr or 'transfer_receive_start_failed') + end + + emit_model(self) +end + +function SlotLease:release(reason) + if self._released then return true, nil end + + self._released = true + if self._frame_tx then self._frame_tx:close(reason or 'transfer slot released') end + + local manager = self._manager + if not manager or not manager._done_tx then return true, nil end + + local ok, err = report(manager, release_identity(self, reason or 'transfer slot released'), + 'transfer_slot_release_report_failed') + + if ok == true then return true, nil end + if type(err) == 'string' and err:match('closed') then return true, nil end + return nil, err +end + +function SlotLease:terminate(reason) + return self:release(reason) +end + +function SlotLease:_poison(reason) + if self._released then return true, nil end + + self._released = true + self._invalid_reason = reason or 'transfer slot no longer active' + if self._frame_tx then self._frame_tx:close(self._invalid_reason) end + + return true, nil +end + +local function lease_active(lease) + local active = lease._manager and lease._manager._state and lease._manager._state.active + return active ~= nil + and active.request_id == lease._request_id + and active.request_generation == lease._request_generation + and same_ctx(active.session, lease._session) +end + +function SlotLease:start_attempt(request_scope, req) + if self._invalid_reason then return nil, self._invalid_reason end + if self._released then return nil, 'transfer slot lease already released' end + if self._attempt_started then return nil, 'transfer slot lease already used' end + if not lease_active(self) then return nil, 'transfer slot lease no longer active' end + + self._attempt_started = true + + local manager = self._manager + local local_tx, local_rx = mailbox.new(1, { full = 'reject_newest' }) + local attempt_req = copy(req or {}) + + local active = manager and manager._state and manager._state.active or nil + if active ~= nil + and active.request_id == self._request_id + and active.request_generation == self._request_generation + then + active.status = 'sending' + active.target = req and req.target or active.target + active.meta = req and req.meta or active.meta + active.size = req and req.size or active.size + active.digest_alg = req and req.digest_alg or active.digest_alg + active.digest = req and req.digest or active.digest + emit_model(manager) + end + + attempt_req.request_id = self._request_id + attempt_req.request_generation = self._request_generation + attempt_req.session = ctx(self._session) + attempt_req.xfer_id = self._xfer_id + + request_scope:finally(function (_, status, primary) + local_tx:close(primary or status or 'transfer attempt observer closed') + end) + + local send_attempt_result = function (ev) + local ok, rerr = report(manager, ev, 'transfer_attempt_report_failed') + if ok ~= true then return nil, rerr end + queue.try_send_now(local_tx, ev) + return true, nil + end + + local raw, err = scoped_work.start { + lifetime_scope = request_scope, + reaper_scope = request_scope, + report_scope = manager._scope, + identity = attempt_identity(attempt_req), + + run = function (scope) + return run_attempt(scope, attempt_req, attempt_caps(manager, self._frame_rx, self._session)) + end, + + report = send_attempt_result, + } + + if not raw then + self._attempt_started = false + local_tx:close(err or 'transfer attempt start failed') + return nil, err + end + + return { + cancel = function (_, reason) return raw:cancel(reason) end, + outcome = function () return raw:outcome() end, + identity = function () return raw:identity() end, + + outcome_op = function () + return local_rx:recv_op():wrap(function (ev, recv_err) + if ev ~= nil then return ev end + + return { + kind = 'transfer_attempt_done', + request_id = attempt_req.request_id, + request_generation = attempt_req.request_generation, + session = ctx(attempt_req.session), + status = 'failed', + primary = recv_err or 'transfer attempt observer closed', + } + end) + end, + }, nil +end + +local function new_lease(self, rec) + local c = ctx(rec.session) + + return setmetatable({ + request_id = rec.request_id, + request_generation = rec.request_generation, + session = ctx(c), + xfer_id = rec.xfer_id, + + _manager = self, + _request_id = rec.request_id, + _request_generation = rec.request_generation, + _session = ctx(c), + _xfer_id = rec.xfer_id, + _frame_tx = rec.frame_tx, + _frame_rx = rec.frame_rx, + _released = false, + _attempt_started = false, + }, SlotLease) +end + +function M.start_attempt(request_scope, lease, req) + if type(lease) ~= 'table' or type(lease.start_attempt) ~= 'function' then + return nil, 'transfer.start_attempt: lease required' + end + return lease:start_attempt(request_scope, req) +end + +local function reply_slot(req, value) + if type(req.reply) ~= 'function' then return nil, 'slot request has no reply method' end + return req:reply(value) +end + +local function fail_slot(req, reason) + if type(req.fail) ~= 'function' then return nil, 'slot request has no fail method' end + return req:fail(reason) +end + +local function defer_slot_request(self, req) + self._pending_slots = self._pending_slots or {} + self._pending_slots[#self._pending_slots + 1] = req + self._state.stats.deferred_no_session = (self._state.stats.deferred_no_session or 0) + 1 + emit_model(self) + return true, nil +end + +local function shift_pending_slot(self) + local pending = self._pending_slots + if type(pending) ~= 'table' or #pending == 0 then return nil end + local req = pending[1] + table.remove(pending, 1) + return req +end + +local drain_pending_slots + +local function handle_slot_request_now(self, req, opts) + opts = opts or {} + local id = req_id(req) + if type(id) ~= 'string' or id == '' then + error('transfer slot request requires request_id', 2) + end + + if self._session == nil then + if opts.defer_on_no_session ~= false then + defer_slot_request(self, req) + else + fail_slot(req, 'no_session') + emit_model(self) + end + return + end + + local frame_tx, frame_rx = mailbox.new(self._attempt_frame_queue_len, { full = 'reject_newest' }) + local rec = { + request_id = id, + request_generation = req_gen(req), + session = ctx(self._session), + xfer_id = req.xfer_id, + target = req.target, + meta = req.meta, + size = req.size, + digest_alg = req.digest_alg, + digest = req.digest, + frame_tx = frame_tx, + frame_rx = frame_rx, + } + + local ok, reason = M.claim_slot(self._state, rec) + if not ok then + frame_tx:close(reason or 'slot_busy') + fail_slot(req, reason or 'slot_busy') + emit_model(self) + return + end + + local lease = new_lease(self, rec) + self._state.active.lease = lease + + local replied, err = reply_slot(req, { ok = true, lease = lease }) + if replied ~= true then lease:release(err or 'slot admission reply failed') end + + emit_model(self) +end + +local function handle_slot_request(self, req) + return handle_slot_request_now(self, req, { defer_on_no_session = true }) +end + +drain_pending_slots = function(self) + if self._session == nil or self._state.active ~= nil then return false end + local req = shift_pending_slot(self) + if req == nil then return false end + handle_slot_request_now(self, req, { defer_on_no_session = true }) + return true +end + +local function active_done(self, reason, session) + local active = self._state.active + + if session ~= nil and active ~= nil and not same_ctx(active.session, session) then + return + end + + if active ~= nil then + -- Outbound transfers grant a caller-visible lease. A session drop poisons + -- that lease immediately because no later receiver can make progress. + -- + -- Inbound receive attempts are different: once xfer_done has been admitted + -- to the writer, the receive worker may already have completed and merely + -- be waiting for its scoped-work completion to be reported. Do not + -- overwrite that successful completion with a synthetic cancellation just + -- because the session queue closes at the same time. Close the frame feed + -- and let the owned receive worker report the authoritative result. + if active.direction == 'receive' then + close_feed(active, reason or 'session_dropped') + emit_model(self) + return + end + + if active.lease and type(active.lease._poison) == 'function' then + active.lease:_poison(reason or 'session_dropped') + else + close_feed(active, reason or 'session_dropped') + end + + self._state.last = { + kind = 'transfer_session_dropped', + request_id = active.request_id, + request_generation = active.request_generation, + session = ctx(active.session), + status = 'cancelled', + primary = reason or 'session_dropped', + progress = active and copy(active.progress) or nil, + } + + self._state.active = nil + self._state.stats.cancelled = self._state.stats.cancelled + 1 + end + + emit_model(self) +end + +local function handle_progress(self, ev) + if not active_matches(self._state, ev) then + self._state.stats.stale = self._state.stats.stale + 1 + emit_model(self) + return + end + + local active = self._state.active + active.progress = active.progress or {} + for k, v in pairs(ev or {}) do + if k ~= 'kind' and k ~= 'request_id' and k ~= 'request_generation' and k ~= 'session' then + active.progress[k] = v + end + end + emit_model(self) +end + +local function handle_attempt_done(self, ev) + local accepted, _, active = M.apply_attempt_done(self._state, ev) + if accepted then close_feed(active, 'transfer attempt completed') end + emit_model(self) + drain_pending_slots(self) +end + +local function handle_slot_released(self, ev) + local accepted, _, active = M.release_slot(self._state, ev) + if accepted then close_feed(active, ev.reason or 'transfer slot released') end + emit_model(self) + drain_pending_slots(self) +end + +local function handle_frame(self, ev) + self._state.stats.frames_received = self._state.stats.frames_received + 1 + + local active = self._state.active + local frame = ev.frame + + if type(frame) == 'table' and frame.type == 'xfer_begin' then + start_receive_attempt(self, ev) + return + end + + if not active + or type(frame) ~= 'table' + or frame.xfer_id ~= active.xfer_id + or not same_ctx(active.session, ev_ctx(ev)) + then + self._state.stats.stale = self._state.stats.stale + 1 + emit_model(self) + return + end + + local ok, err = queue.try_admit_required(active.frame_tx, ev, + 'transfer_attempt_frame_admission_failed') + if ok ~= true then error(err or 'transfer_attempt_frame_admission_failed', 0) end + + self._state.stats.frames_routed = self._state.stats.frames_routed + 1 + emit_model(self) +end + +local function handle_peer_session(self, ev) + local c = require_ctx(ev_ctx(ev), 'transfer peer_session') + + if self._session ~= nil and not same_ctx(self._session, c) then + active_done(self, 'new_peer_session') + end + + self._session = ctx(c) + emit_model(self) + drain_pending_slots(self) +end + +local function handle_peer_session_dropped(self, ev) + local c = require_ctx(ev_ctx(ev), 'transfer peer_session_dropped') + if not same_ctx(self._session, c) then return end + + active_done(self, ev.reason or 'session_dropped', c) + self._session = nil + emit_model(self) +end + +local function map_done(ev) + return ev or { kind = 'done_queue_closed' } +end + +local function map_admission(req) + if req == nil then return { kind = 'admission_queue_closed' } end + return { kind = 'slot_request', req = req } +end + +local function map_session(item) + if item == nil then return { kind = 'session_queue_closed' } end + if type(item) ~= 'table' then error('transfer expected session event', 0) end + + if item.kind == 'peer_session' or item.kind == 'peer_session_dropped' then + require_ctx(ev_ctx(item), 'transfer session event') + return item + end + + if item.kind ~= 'session_frame' then error('transfer expected session event', 0) end + if item.lane ~= 'transfer' then + error('transfer received non-transfer session frame: ' .. tostring(item.lane), 0) + end + if type(item.frame) ~= 'table' then error('transfer session_frame missing frame', 0) end + + return { + kind = 'transfer_frame', + frame = item.frame, + at = item.at, + session = require_ctx(ev_ctx(item), 'transfer session_frame'), + } +end + +local function try_rx(open, rx, mapper) + if not open or rx == nil then return nil end + + local item, err = queue.try_recv_now(rx) + if item ~= nil then return mapper(item) end + if err ~= 'not_ready' then return mapper(nil) end + return nil +end + +local function next_event_op(self) + return priority_event.sources_op { + label = 'fabric.transfer.manager', + pending = self._event_pending, + sources = { + { + name = 'done', + try_now = function () return try_rx(true, self._done_rx, map_done) end, + recv_op = function () return self._done_rx:recv_op():wrap(map_done) end, + }, + { + name = 'session', + enabled = function () return self._session_open end, + try_now = function () return try_rx(self._session_open, self._session_rx, map_session) end, + recv_op = function () return self._session_rx:recv_op():wrap(map_session) end, + }, + { + name = 'admission', + enabled = function () return self._admission_open end, + try_now = function () return try_rx(self._admission_open, self._admission_rx, map_admission) end, + recv_op = function () return self._admission_rx:recv_op():wrap(map_admission) end, + }, + }, + } +end + +local function dispatch(self, ev) + if ev.kind == 'peer_session' then + handle_peer_session(self, ev) + + elseif ev.kind == 'peer_session_dropped' then + handle_peer_session_dropped(self, ev) + + elseif ev.kind == 'slot_request' then + handle_slot_request(self, ev.req) + + elseif ev.kind == 'admission_queue_closed' then + self._admission_open = false + + elseif ev.kind == 'session_queue_closed' then + self._session_open = false + active_done(self, 'session_closed') + self._session = nil + + elseif ev.kind == 'transfer_frame' then + handle_frame(self, ev) + + elseif ev.kind == 'transfer_progress' then + handle_progress(self, ev) + + elseif ev.kind == 'transfer_attempt_done' then + handle_attempt_done(self, ev) + + elseif ev.kind == 'transfer_slot_released' then + handle_slot_released(self, ev) + + elseif ev.kind == 'done_queue_closed' then + error('transfer manager done queue closed', 0) + + else + error('transfer manager unknown event kind: ' .. tostring(ev.kind), 0) + end +end + +local function should_finish(self) + return not self._admission_open + and not self._session_open + and self._state.active == nil +end + +local function cancel_active(self, reason) + local active = self._state.active + if active and active.lease and type(active.lease._poison) == 'function' then + active.lease:_poison(reason or 'transfer manager closing') + else + close_feed(active, reason or 'transfer manager closing') + end +end + +local function coordinator_loop(self) + while not should_finish(self) do + dispatch(self, fibers.perform(next_event_op(self))) + end + + return { + role = 'transfer_manager', + manager_id = self._state.manager_id, + snapshot = manager_snapshot(self), + } +end + +function M.run(scope, params) + if type(scope) ~= 'table' then error('transfer.run: scope required', 2) end + if type(params) ~= 'table' then error('transfer.run: params table required', 2) end + if not params.admission_rx then error('transfer.run: admission_rx required', 2) end + if not params.session_rx then error('transfer.run: session_rx required', 2) end + + local outbound = params.outbound + if type(outbound) ~= 'table' + or type(outbound.send_transfer_control_frame_now) ~= 'function' + or type(outbound.send_transfer_bulk_frame_now) ~= 'function' + then + error('transfer.run: fabric.session outbound gate required', 2) + end + + local state = M.new_state { manager_id = params.manager_id } + local model = model_mod.new(M.snapshot(state), { + copy = M.snapshot, + equals = snapshot_equal, + label = 'fabric.transfer', + }) + + scope:finally(function (_, status, primary) + model:terminate(primary or status or 'transfer manager closed') + end) + + local done_tx, done_rx = mailbox.new(params.done_queue_len or DEFAULT_DONE_QUEUE, + { full = 'reject_newest' }) + + scope:finally(function () + done_tx:close('transfer manager closed') + end) + + local self = setmetatable({ + _scope = scope, + _state = state, + _model = model, + _state_tx = params.state_tx, + _component_name = params.component_name or 'transfer_manager', + _link_id = params.link_id, + _link_generation = params.link_generation, + + _admission_rx = params.admission_rx, + _session_rx = params.session_rx, + _admission_open = true, + _session_open = true, + + _done_tx = done_tx, + _done_rx = done_rx, + _events_port = service_events.port(done_tx, { + source = 'fabric_transfer', + source_id = params.manager_id or params.link_id or 'transfer_manager', + link_id = params.link_id, + link_generation = params.link_generation, + }, { label = 'fabric_transfer_event_report_failed' }), + _outbound = outbound, + _receive_targets = params.receive_targets, + _attempt_frame_queue_len = params.attempt_frame_queue_len or DEFAULT_ATTEMPT_FRAME_QUEUE, + _chunk_size = params.chunk_size, + _timeout_s = params.timeout_s, + _retry_limit = params.retry_limit, + _session = nil, + _event_pending = {}, + }, Manager) + + emit_model(self, true) + + scope:finally(function (_, status, primary) + cancel_active(self, primary or status or 'transfer manager closed') + end) + + return coordinator_loop(self) +end + +M.make_attempt_caps = attempt_caps +M.SlotLease = SlotLease +M.Manager = Manager + +return M diff --git a/src/services/fabric/transfer_client.lua b/src/services/fabric/transfer_client.lua new file mode 100644 index 00000000..5a3b663c --- /dev/null +++ b/src/services/fabric/transfer_client.lua @@ -0,0 +1,206 @@ +-- services/fabric/transfer_client.lua +-- +-- Caller-side helper for one outbound Fabric byte transfer. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local sleep = require 'fibers.sleep' + +local queue = require 'devicecode.support.queue' +local resource = require 'devicecode.support.resource' +local blob = require 'devicecode.blob_source' +local protocol = require 'services.fabric.protocol' +local transfer = require 'services.fabric.transfer' +local contracts = require 'devicecode.support.contracts' + +local M = {} + +local function require_tx(v, name) + return contracts.require_tx(v, name, 2) +end + +local function pos_timeout(v) + v = v or 30.0 + if type(v) ~= 'number' or v <= 0 then return nil, 'timeout' end + return v, nil +end + +local function source_owner(scope, p) + if type(p.source_owner) == 'table' and type(p.source_owner.handoff) == 'function' then + return p.source_owner, nil + end + + local src = p.source + if src == nil and type(p.data) == 'string' then + src = blob.from_string(p.data) + end + if type(src) ~= 'table' or type(src.read_chunk_op) ~= 'function' then + return nil, 'source_required' + end + + local owner = resource.owned(src, { + label = 'fabric transfer client source cleanup failed', + }) + + scope:finally(function (_, status, primary) + owner:terminate_checked(primary or status or 'transfer client closed') + end) + + return owner, nil +end + +local function normalise_request(p) + local id = p.request_id + if type(id) ~= 'string' or id == '' then return nil, 'request_id_required' end + if type(p.xfer_id) ~= 'string' or p.xfer_id == '' then return nil, 'xfer_id_required' end + if type(p.target) ~= 'string' or p.target == '' then return nil, 'target_required' end + + local size = p.size + if size == nil and type(p.data) == 'string' then size = #p.data end + if type(size) ~= 'number' or size < 0 or size % 1 ~= 0 then + return nil, 'size_required' + end + + local digest_alg = p.digest_alg or protocol.DIGEST_ALG + if digest_alg ~= protocol.DIGEST_ALG then return nil, 'unsupported_digest_alg' end + + local digest = p.digest + if digest == nil and type(p.data) == 'string' then digest = protocol.digest_hex(p.data) end + if type(digest) ~= 'string' or not protocol.digest_ok(digest) then + return nil, 'digest_required' + end + + return { + request_id = id, + request_generation = p.request_generation or 1, + xfer_id = p.xfer_id, + target = p.target, + size = size, + digest_alg = digest_alg, + digest = digest, + meta = p.meta, + chunk_size = p.chunk_size, + }, nil +end + +local function remaining(deadline) + local n = deadline - fibers.now() + return n > 0 and n or 0 +end + +local function make_slot_request(req, reply_tx) + local slot = { + kind = 'transfer_slot_request', + request_id = req.request_id, + request_generation = req.request_generation, + xfer_id = req.xfer_id, + target = req.target, + meta = req.meta, + size = req.size, + digest_alg = req.digest_alg, + digest = req.digest, + } + + function slot:reply(value) + return queue.try_admit_required(reply_tx, { ok = true, value = value }, + 'transfer_client_slot_reply_failed') + end + + function slot:fail(reason) + return queue.try_admit_required(reply_tx, { ok = false, err = reason }, + 'transfer_client_slot_fail_failed') + end + + return slot +end + +local function await_reply(rx, deadline, label) + local which, reply = fibers.perform(fibers.named_choice { + reply = rx:recv_op(), + timeout = sleep.sleep_op(remaining(deadline)), + }) + + if which == 'timeout' then return nil, 'timeout' end + if reply == nil then return nil, label or 'closed' end + if reply.ok ~= true then return nil, reply.err or label or 'failed' end + return reply.value, nil +end + +local function run_in_scope(scope, params) + if type(params) ~= 'table' then error('transfer_client.run: params table required', 2) end + + local admission_tx = require_tx(params.admission_tx, 'transfer_client.run: admission_tx') + local timeout_s, terr = pos_timeout(params.timeout_s) + if not timeout_s then return nil, terr end + + local req, rerr = normalise_request(params) + if not req then return nil, rerr end + + local owner, serr = source_owner(scope, params) + if not owner then return nil, serr end + + local deadline = fibers.now() + timeout_s + local reply_tx, reply_rx = mailbox.new(1, { full = 'reject_newest' }) + + scope:finally(function (_, status, primary) + reply_tx:close(primary or status or 'transfer client closed') + end) + + local ok, err = queue.try_admit_required( + admission_tx, + make_slot_request(req, reply_tx), + 'transfer_client_slot_request_failed' + ) + if ok ~= true then return nil, err or 'slot_admission_failed' end + + local admitted, aerr = await_reply(reply_rx, deadline, 'slot_admission_closed') + if not admitted then return nil, aerr end + + local lease = admitted.lease + if type(lease) ~= 'table' or type(lease.start_attempt) ~= 'function' then + return nil, 'slot_admission_missing_lease' + end + + local lease_owner = resource.owned(lease, { + terminate = function (value, reason) return value:release(reason) end, + label = 'fabric transfer client lease release failed', + }) + + scope:finally(function (_, status, primary) + lease_owner:terminate_checked(primary or status or 'transfer client closed') + end) + + req.source_owner = owner + req.timeout_s = timeout_s + + local handle, herr = transfer.start_attempt(scope, lease, req) + if not handle then return nil, herr or 'transfer_attempt_start_failed' end + + local handed, handoff_err = lease_owner:handoff(function () return true end) + if not handed then return nil, handoff_err or 'transfer_slot_lease_handoff_failed' end + + local which, ev = fibers.perform(fibers.named_choice { + attempt = handle:outcome_op(), + timeout = sleep.sleep_op(remaining(deadline)), + }) + + if which == 'timeout' then + handle:cancel('timeout') + return nil, 'timeout' + end + + if ev == nil then return nil, 'attempt_outcome_missing' end + if ev.status == 'ok' then return ev.result, nil end + return nil, ev.primary or ev.status or 'transfer_attempt_failed' +end + +function M.run(_, params) + local st, _, result, err = fibers.run_scope(function (scope) + return run_in_scope(scope, params) + end) + + if st == 'ok' then return result, err end + return nil, result or st or 'transfer_client_failed' +end + +return M diff --git a/src/services/fabric/transfer_receive.lua b/src/services/fabric/transfer_receive.lua new file mode 100644 index 00000000..a50a82f4 --- /dev/null +++ b/src/services/fabric/transfer_receive.lua @@ -0,0 +1,298 @@ +-- services/fabric/transfer_receive.lua +-- +-- Receive-side Fabric transfer worker. +-- +-- This is deliberately below the public bus surface. Fabric owns the wire +-- transfer protocol; application services register target objects at runtime. +-- A target receives a complete stream through a finaliser-safe sink interface. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' + +local protocol = require 'services.fabric.protocol' +local xxhash32 = require 'shared.hash.xxhash32' + +local M = {} + +local DEFAULT_TIMEOUT = 1.0 +local DEFAULT_RETRY_LIMIT = 12 + +local function positive(v, fallback, name) + v = v or fallback + if type(v) ~= 'number' or v <= 0 or v ~= v or v == math.huge or v == -math.huge then + error('transfer_receive: ' .. name .. ' must be positive', 0) + end + return v +end + +local function nonneg_int(v, fallback, name) + if v == nil then v = fallback end + if type(v) ~= 'number' or v < 0 or v % 1 ~= 0 then + error('transfer_receive: ' .. name .. ' must be a non-negative integer', 0) + end + return v +end + +local function construct(label, fn, ...) + local frame, err = fn(...) + if not frame then error(label .. ': ' .. tostring(err), 0) end + return frame +end + +local function send_control(caps, frame, label) + local fn = caps.send_control_frame_now + if type(fn) ~= 'function' then + error('transfer_receive: missing session-bound control sender', 0) + end + local ok, err = fn(frame, label) + if ok ~= true then error((label or 'transfer_receive_send_failed') .. ': ' .. tostring(err), 0) end + return true +end + +local function need_frame(xfer_id, next, retry, reason) + return construct('xfer_need', protocol.xfer_need, xfer_id, next, retry == true or nil, reason) +end + +local function try_abort(caps, xfer_id, reason) + if type(caps.send_control_frame_now) ~= 'function' then return true, nil end + local frame, err = protocol.xfer_abort(xfer_id, tostring(reason or 'aborted')) + if not frame then return nil, err end + return caps.send_control_frame_now(frame, 'transfer_receive_abort_send_failed') +end + +local function terminate_sink(sink, reason) + if type(sink) ~= 'table' then return true, nil end + if type(sink.abort) == 'function' then return sink:abort(reason or 'transfer_aborted') end + if type(sink.terminate) == 'function' then return sink:terminate(reason or 'transfer_aborted') end + if type(sink.close) == 'function' then return sink:close(reason or 'transfer_aborted') end + return true, nil +end + +local function fail(caps, xfer_id, sink, reason, send_abort) + local err = tostring(reason or 'transfer_receive_failed') + terminate_sink(sink, err) + if send_abort ~= false then + local ok, aerr = try_abort(caps, xfer_id, err) + if ok ~= true then err = err .. '; abort_failed: ' .. tostring(aerr) end + end + error(err, 0) +end + +local function wait_frame_op(rx, deadline) + local dt = deadline - fibers.now() + if dt < 0 then dt = 0 end + return fibers.named_choice { + frame = rx:recv_op(), + timeout = sleep.sleep_op(dt), + } +end + +local function open_sink(target, req) + local sink, err + if type(target.open_sink_op) == 'function' then + sink, err = fibers.perform(target:open_sink_op(req)) + elseif type(target.create_sink_op) == 'function' then + sink, err = fibers.perform(target:create_sink_op(req)) + elseif type(target.begin_transfer_op) == 'function' then + sink, err = fibers.perform(target:begin_transfer_op(req)) + elseif type(target.open_sink) == 'function' then + sink, err = target:open_sink(req) + elseif type(target.create_sink) == 'function' then + sink, err = target:create_sink(req) + end + if sink == nil then return nil, err or 'transfer_target_unavailable' end + if type(sink) ~= 'table' then return nil, 'transfer_target_returned_non_sink' end + if type(sink.append_op) ~= 'function' + and type(sink.write_chunk_op) ~= 'function' + and type(sink.write_op) ~= 'function' + then + return nil, 'transfer_sink_missing_append_op' + end + if type(sink.commit_op) ~= 'function' then + return nil, 'transfer_sink_missing_commit_op' + end + return sink, nil +end + +local function append_chunk(sink, chunk) + local ok, err + if type(sink.append_op) == 'function' then + ok, err = fibers.perform(sink:append_op(chunk)) + elseif type(sink.write_chunk_op) == 'function' then + ok, err = fibers.perform(sink:write_chunk_op(chunk)) + else + ok, err = fibers.perform(sink:write_op(chunk)) + end + if ok == nil or ok == false then return nil, err or 'write_failed' end + return true, nil +end + +local function commit_sink(sink, req) + local result, err = fibers.perform(sink:commit_op(req)) + if result == nil or result == false then return nil, err or 'commit_failed' end + return result, nil +end + +local function require_begin(req) + local begin = req.initial_frame + if type(begin) ~= 'table' or begin.type ~= 'xfer_begin' then + error('transfer_receive: initial xfer_begin required', 0) + end + if begin.digest_alg ~= protocol.DIGEST_ALG then + error('unsupported_digest_alg', 0) + end + if type(begin.digest) ~= 'string' or not protocol.digest_ok(begin.digest) then + error('invalid_digest', 0) + end + return begin +end + +function M.run(scope, req, caps) + if type(scope) ~= 'table' then error('transfer_receive.run: scope required', 2) end + if type(req) ~= 'table' then error('transfer_receive.run: request table required', 2) end + if type(caps) ~= 'table' then error('transfer_receive.run: caps table required', 2) end + local rx = caps.frame_rx + if type(rx) ~= 'table' or type(rx.recv_op) ~= 'function' then + error('transfer_receive: frame_rx required', 0) + end + + local begin = require_begin(req) + local xfer_id = begin.xfer_id + local target_id = begin.target + local size = begin.size + local timeout_s = positive(req.timeout_s or caps.timeout_s, DEFAULT_TIMEOUT, 'timeout_s') + local retry_limit = nonneg_int(req.retry_limit or caps.retry_limit, DEFAULT_RETRY_LIMIT, 'retry_limit') + local target = req.target + if type(target) ~= 'table' then + fail(caps, xfer_id, nil, 'unsupported_target', true) + end + + local sink, serr = open_sink(target, { + xfer_id = xfer_id, + target = target_id, + size = size, + digest_alg = begin.digest_alg, + digest = begin.digest, + meta = protocol.copy_json_value(begin.meta), + session = req.session, + }) + if sink == nil then + fail(caps, xfer_id, nil, serr or 'transfer_target_unavailable', true) + end + + local finished = false + scope:finally(function (_, status, primary) + if not finished then + terminate_sink(sink, primary or status or 'transfer_receive_closed') + end + end) + + send_control(caps, construct('xfer_ready', protocol.xfer_ready, xfer_id), 'transfer_receive_ready_send_failed') + send_control(caps, need_frame(xfer_id, 0, false), 'transfer_receive_need_send_failed') + + local received = 0 + local digest_state = xxhash32.new(0) + local deadline = fibers.now() + timeout_s + local retries_at_offset = 0 + local chunk_retries = 0 + + local function retry_current(reason, label) + if retries_at_offset >= retry_limit then return false end + retries_at_offset = retries_at_offset + 1 + chunk_retries = chunk_retries + 1 + deadline = fibers.now() + timeout_s + send_control(caps, need_frame(xfer_id, received, true, reason), label or 'transfer_receive_retry_need_send_failed') + return true + end + + while true do + local which, item = fibers.perform(wait_frame_op(rx, deadline)) + if which == 'timeout' then + if not retry_current('idle', 'transfer_receive_idle_need_send_failed') then + fail(caps, xfer_id, sink, 'timeout', true) + end + + elseif item == nil then + fail(caps, xfer_id, sink, 'transfer_receive_frame_feed_closed', false) + + else + local frame = item.frame or item + + if type(frame) ~= 'table' or frame.xfer_id ~= xfer_id then + -- Manager normally filters these. + + elseif frame.type == 'xfer_abort' then + fail(caps, xfer_id, sink, frame.err or 'remote_abort', false) + + elseif frame.type == 'xfer_chunk' then + if frame.offset ~= received then + local reason = frame.offset < received and 'duplicate_or_stale_offset' or 'future_offset' + if not retry_current(reason, 'transfer_receive_offset_need_send_failed') then + fail(caps, xfer_id, sink, 'unexpected_offset', true) + end + else + local chunk = frame.data + if type(chunk) ~= 'string' then fail(caps, xfer_id, sink, 'invalid_chunk_data', true) end + if received + #chunk > size then fail(caps, xfer_id, sink, 'size_overrun', true) end + if not protocol.verify_chunk_digest(chunk, frame.chunk_digest) then + if not retry_current('chunk_digest_mismatch', 'transfer_receive_retry_need_send_failed') then + fail(caps, xfer_id, sink, 'chunk_digest_mismatch', true) + end + else + local ok, werr = append_chunk(sink, chunk) + if ok ~= true then fail(caps, xfer_id, sink, werr or 'write_failed', true) end + xxhash32.update(digest_state, chunk) + received = received + #chunk + retries_at_offset = 0 + deadline = fibers.now() + timeout_s + -- Acknowledge every accepted chunk, including the final one. The + -- sender waits for xfer_need next == size before sending xfer_commit. + -- This keeps commit ordered after the receiver has actually processed + -- the last bulk frame, even when the writer uses separate control and + -- bulk lanes. + send_control(caps, need_frame(xfer_id, received, false), 'transfer_receive_need_send_failed') + end + end + + elseif frame.type == 'xfer_commit' then + if frame.size ~= size or frame.size ~= received then + fail(caps, xfer_id, sink, 'short_transfer', true) + end + if frame.digest_alg ~= protocol.DIGEST_ALG then + fail(caps, xfer_id, sink, 'unsupported_digest_alg', true) + end + local got = xxhash32.digest_hex_state(digest_state) + if frame.digest ~= begin.digest or frame.digest ~= got then + fail(caps, xfer_id, sink, 'digest_mismatch', true) + end + local commit_result, cerr = commit_sink(sink, { + xfer_id = xfer_id, + target = target_id, + size = size, + digest_alg = frame.digest_alg, + digest = frame.digest, + meta = protocol.copy_json_value(begin.meta), + session = req.session, + }) + if commit_result == nil then fail(caps, xfer_id, sink, cerr or 'commit_failed', true) end + finished = true + send_control(caps, construct('xfer_done', protocol.xfer_done, xfer_id), 'transfer_receive_done_send_failed') + return { + job_id = type(begin.meta) == 'table' and begin.meta.job_id or nil, + component = type(begin.meta) == 'table' and begin.meta.component or nil, + image_id = type(begin.meta) == 'table' and begin.meta.image_id or nil, + target = target_id, + xfer_id = xfer_id, + digest_alg = frame.digest_alg, + digest = frame.digest, + received_bytes = received, + size = size, + commit_result = commit_result, + chunk_retries = chunk_retries, + } + end + end + end +end + +return M diff --git a/src/services/fabric/transfer_sender.lua b/src/services/fabric/transfer_sender.lua new file mode 100644 index 00000000..7f7a8058 --- /dev/null +++ b/src/services/fabric/transfer_sender.lua @@ -0,0 +1,501 @@ +-- services/fabric/transfer_sender.lua +-- +-- Send-side Fabric transfer attempt worker. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' + +local protocol = require 'services.fabric.protocol' + +local M = {} + +local DEFAULT_TIMEOUT = 1.0 +local DEFAULT_CHUNK_SIZE = protocol.DEFAULT_CHUNK_SIZE or 2048 +local DEFAULT_BEGIN_RETRY = 1.0 +local DEFAULT_COMMIT_RETRY_LIMIT = 8 + +local function nonempty(v) + return type(v) == 'string' and v ~= '' +end + +local function nonneg_int(v) + return type(v) == 'number' + and v == v and v ~= math.huge and v ~= -math.huge + and v >= 0 and v % 1 == 0 +end + +local function positive(v, fallback, name, integer) + v = v or fallback + if type(v) ~= 'number' or v <= 0 or v ~= v or v == math.huge or v == -math.huge then + error('transfer_sender: ' .. name .. ' must be positive', 0) + end + if integer and v % 1 ~= 0 then + error('transfer_sender: ' .. name .. ' must be an integer', 0) + end + return v +end + +local function require_source(req) + local source = req.source + if type(source) ~= 'table' or type(source.read_chunk_op) ~= 'function' then + error('transfer_sender: source with read_chunk_op required', 0) + end + return source +end + +local function require_request(req) + local xfer_id = req.xfer_id + if not nonempty(xfer_id) then error('transfer_sender: xfer_id required', 0) end + if not nonempty(req.target) then error('transfer_sender: target required', 0) end + if not nonneg_int(req.size) then error('transfer_sender: size must be a non-negative integer', 0) end + + local alg = req.digest_alg or protocol.DIGEST_ALG + if alg ~= protocol.DIGEST_ALG then + error('transfer_sender: unsupported digest_alg: ' .. tostring(alg), 0) + end + if not nonempty(req.digest) or not protocol.digest_ok(req.digest) then + error('transfer_sender: digest must be xxhash32 hex', 0) + end + + return xfer_id, req.target, req.size, alg, req.digest +end + +local function construct(label, fn, ...) + local frame, err = fn(...) + if not frame then error(label .. ': ' .. tostring(err), 0) end + return frame +end + +local function encoded_frame_len(frame) + local line = protocol.encode_line(frame) + if type(line) == 'string' then return #line end + return nil +end + +local function elapsed_ms(start) + if start == nil then return nil end + local dt = (fibers.now() - start) * 1000 + if dt < 0 then dt = 0 end + return math.floor(dt + 0.5) +end + +local function observe(caps, ev) + if type(caps) ~= 'table' or type(caps.report_progress_now) ~= 'function' then + return true, nil + end + ev.at = ev.at or fibers.now() + return caps.report_progress_now(ev) +end + +local function send(caps, lane, frame, label) + local fn = lane == 'bulk' and caps.send_bulk_frame_now or caps.send_control_frame_now + if type(fn) ~= 'function' then + error('transfer_sender: missing session-bound sender for ' .. tostring(lane), 0) + end + + local ok, err = fn(frame, label) + if ok ~= true then error((label or 'transfer_send_failed') .. ': ' .. tostring(err), 0) end + return true +end + +local function try_abort(caps, xfer_id, reason) + if type(caps.send_control_frame_now) ~= 'function' then return true, nil end + local frame, err = protocol.xfer_abort(xfer_id, tostring(reason or 'aborted')) + if not frame then return nil, err end + return caps.send_control_frame_now(frame, 'transfer_abort_send_failed') +end + +local function fail(caps, xfer_id, reason, send_abort) + local err = tostring(reason or 'transfer_failed') + + if send_abort ~= false then + local ok, aerr = try_abort(caps, xfer_id, err) + if ok ~= true then err = err .. '; abort_failed: ' .. tostring(aerr) end + end + + error(err, 0) +end + +local function wait_frame_op(rx, deadline, retry_at) + local now = fibers.now() + local dt = deadline - now + if dt < 0 then dt = 0 end + + local choices = { + frame = rx:recv_op(), + timeout = sleep.sleep_op(dt), + } + if type(retry_at) == 'number' and retry_at < deadline then + local rt = retry_at - now + if rt < 0 then rt = 0 end + choices.retry = sleep.sleep_op(rt) + end + + return fibers.named_choice(choices) +end + +local function read_chunk(source, n) + local started = fibers.now() + local chunk, err = fibers.perform(source:read_chunk_op(n)) + if err ~= nil then return nil, err, elapsed_ms(started) end + return chunk, nil, elapsed_ms(started) +end + +local function send_commit(caps, xfer_id, size, alg, digest, timeout_s) + local frame = construct('xfer_commit', protocol.xfer_commit, xfer_id, size, alg, digest) + send(caps, 'control', frame, 'transfer_commit_send_failed') + return 'committing', fibers.now() + timeout_s +end + +local function make_next_chunk(caps, source, xfer_id, offset, size, chunk_size) + local want = math.min(chunk_size, size - offset) + local chunk, err, source_read_ms = read_chunk(source, want) + + if err ~= nil then fail(caps, xfer_id, err, true) end + if type(chunk) ~= 'string' or #chunk == 0 then fail(caps, xfer_id, 'short_source', true) end + if offset + #chunk > size then fail(caps, xfer_id, 'source_overrun', true) end + + local chunk_digest = protocol.chunk_digest(chunk) + local frame = construct( + 'xfer_chunk', + protocol.xfer_chunk, + xfer_id, + offset, + chunk, + chunk_digest + ) + + return { + offset = offset, + next = offset + #chunk, + len = #chunk, + digest = chunk_digest, + frame_len = encoded_frame_len(frame), + source_read_ms = source_read_ms, + frame = frame, + } +end + +local function send_chunk(caps, pending) + local started = fibers.now() + send(caps, 'bulk', pending.frame, 'transfer_chunk_send_failed') + return true, elapsed_ms(started) +end + +function M.run(scope, req, caps) + if type(scope) ~= 'table' then error('transfer_sender.run: scope required', 2) end + if type(req) ~= 'table' then error('transfer_sender.run: request table required', 2) end + if type(caps) ~= 'table' then error('transfer_sender.run: caps table required', 2) end + + local rx = caps.frame_rx + if type(rx) ~= 'table' or type(rx.recv_op) ~= 'function' then + error('transfer_sender: frame_rx required', 0) + end + + local source = require_source(req) + local xfer_id, target, size, alg, digest = require_request(req) + local timeout_s = positive(req.timeout_s or caps.timeout_s, DEFAULT_TIMEOUT, 'timeout_s') + local chunk_size = positive(req.chunk_size or caps.chunk_size, DEFAULT_CHUNK_SIZE, 'chunk_size', true) + local begin_retry_s = positive(req.begin_retry_s or caps.begin_retry_s or DEFAULT_BEGIN_RETRY, DEFAULT_BEGIN_RETRY, 'begin_retry_s') + local commit_retry_limit = positive(req.commit_retry_limit or caps.commit_retry_limit or DEFAULT_COMMIT_RETRY_LIMIT, DEFAULT_COMMIT_RETRY_LIMIT, 'commit_retry_limit', true) + local chunks_sent = 0 + local retransmits = 0 + local begin_retries = 0 + local commit_retries = 0 + local max_frame_queue_ms = 0 + local max_need_to_chunk_ms = 0 + local max_source_read_ms = 0 + local max_send_ms = 0 + + local function note_max(name, value) + if type(value) ~= 'number' then return end + if name == 'frame_queue' and value > max_frame_queue_ms then max_frame_queue_ms = value end + if name == 'need_to_chunk' and value > max_need_to_chunk_ms then max_need_to_chunk_ms = value end + if name == 'source_read' and value > max_source_read_ms then max_source_read_ms = value end + if name == 'send' and value > max_send_ms then max_send_ms = value end + end + + local function report(ev) + ev = ev or {} + ev.xfer_id = xfer_id + ev.target = target + ev.size = size + ev.digest_alg = alg + ev.digest = digest + ev.sent_bytes = ev.sent_bytes or 0 + ev.chunks_sent = chunks_sent + ev.retransmits = retransmits or 0 + ev.begin_retries = begin_retries or 0 + ev.commit_retries = commit_retries or 0 + ev.max_frame_queue_ms = max_frame_queue_ms + ev.max_need_to_chunk_ms = max_need_to_chunk_ms + ev.max_source_read_ms = max_source_read_ms + ev.max_send_ms = max_send_ms + return observe(caps, ev) + end + + local begin = construct('xfer_begin', protocol.xfer_begin, + xfer_id, target, size, alg, digest, req.meta) + + local begin_send_start = fibers.now() + send(caps, 'control', begin, 'transfer_begin_send_failed') + note_max('send', elapsed_ms(begin_send_start)) + report { event = 'xfer_begin_tx', phase = 'waiting_ready', last_tx_type = 'xfer_begin' } + + -- `sent` is the receiver-acknowledged offset. `pending` is the one + -- outstanding chunk that may be resent if the receiver asks again for the + -- same offset. This deliberately stays stop-and-wait; there is no seek, + -- resume, or sliding window in v1. + local sent = 0 + local pending = nil + local state = 'waiting_ready' + local deadline = fibers.now() + timeout_s + local next_begin_retry = fibers.now() + begin_retry_s + + while true do + local retry_at = state == 'waiting_ready' and next_begin_retry or nil + local which, item = fibers.perform(wait_frame_op(rx, deadline, retry_at)) + if which == 'retry' then + if state == 'waiting_ready' then + local started = fibers.now() + send(caps, 'control', begin, 'transfer_begin_send_failed') + note_max('send', elapsed_ms(started)) + begin_retries = begin_retries + 1 + report { event = 'xfer_begin_tx', phase = state, last_tx_type = 'xfer_begin', retransmit = true, sent_bytes = sent } + next_begin_retry = fibers.now() + begin_retry_s + end + else + if which == 'timeout' then + report { event = 'timeout', phase = state, err = 'timeout', sent_bytes = sent, pending_offset = pending and pending.offset or nil, pending_next = pending and pending.next or nil } + fail(caps, xfer_id, 'timeout', true) + end + if item == nil then error('transfer_sender_frame_feed_closed', 0) end + + local frame = item.frame or item + local frame_queue_ms = nil + if type(item) == 'table' and type(item.at) == 'number' then + frame_queue_ms = elapsed_ms(item.at) + note_max('frame_queue', frame_queue_ms) + end + + if type(frame) ~= 'table' or frame.xfer_id ~= xfer_id then + -- Manager normally filters these. + + elseif frame.type == 'xfer_abort' then + report { + event = 'xfer_abort_rx', + phase = state, + last_rx_type = 'xfer_abort', + err = frame.err or 'remote_abort', + frame_queue_ms = frame_queue_ms, + sent_bytes = sent, + } + fail(caps, xfer_id, frame.err or 'remote_abort', false) + + elseif frame.type == 'xfer_ready' then + if state == 'waiting_ready' then + state = 'sending' + next_begin_retry = nil + deadline = fibers.now() + timeout_s + end + report { + event = 'xfer_ready_rx', + phase = state, + last_rx_type = 'xfer_ready', + frame_queue_ms = frame_queue_ms, + sent_bytes = sent, + } + + elseif frame.type == 'xfer_need' then + if state ~= 'sending' then + if state == 'committing' and frame.retry == true and frame.next == size then + if commit_retries >= commit_retry_limit then + fail(caps, xfer_id, 'commit_retry_limit_exceeded', true) + end + commit_retries = commit_retries + 1 + retransmits = retransmits + 1 + local started = fibers.now() + send(caps, 'control', construct('xfer_commit', protocol.xfer_commit, xfer_id, size, alg, digest), 'transfer_commit_send_failed') + note_max('send', elapsed_ms(started)) + deadline = fibers.now() + timeout_s + report { + event = 'xfer_commit_tx', + phase = state, + last_rx_type = 'xfer_need', + last_rx_next = frame.next, + requested_next = frame.next, + retry = true, + reason = frame.reason, + frame_queue_ms = frame_queue_ms, + retransmit = true, + last_tx_type = 'xfer_commit', + sent_bytes = sent, + } + elseif state == 'committing' then + report { + event = 'xfer_need_stale_ignored', + phase = state, + last_rx_type = 'xfer_need', + last_rx_next = frame.next, + requested_next = frame.next, + retry = frame.retry == true, + reason = frame.reason, + frame_queue_ms = frame_queue_ms, + sent_bytes = sent, + } + else + fail(caps, xfer_id, 'unexpected_need', true) + end + else + local need_at = fibers.now() + report { + event = 'xfer_need_rx', + phase = state, + last_rx_type = 'xfer_need', + last_rx_next = frame.next, + requested_next = frame.next, + retry = frame.retry == true, + reason = frame.reason, + pending_offset = pending and pending.offset or nil, + pending_next = pending and pending.next or nil, + frame_queue_ms = frame_queue_ms, + sent_bytes = sent, + } + + if pending ~= nil then + if frame.next == pending.offset then + -- Receiver rejected or lost the last chunk before advancing. + -- Resend the cached frame without reading from the source again. + local _, send_ms = send_chunk(caps, pending) + note_max('send', send_ms) + local need_to_chunk_ms = elapsed_ms(need_at) + note_max('need_to_chunk', need_to_chunk_ms) + retransmits = retransmits + 1 + report { + event = 'xfer_chunk_tx', + phase = state, + last_tx_type = 'xfer_chunk', + last_tx_offset = pending.offset, + last_tx_next = pending.next, + chunk_len = pending.len, + chunk_digest = pending.digest, + chunk_frame_len = pending.frame_len, + send_ms = send_ms, + need_to_chunk_ms = need_to_chunk_ms, + retransmit = true, + sent_bytes = sent, + } + deadline = fibers.now() + timeout_s + + elseif frame.next == pending.next then + sent = pending.next + pending = nil + if sent >= size then + state, deadline = send_commit(caps, xfer_id, size, alg, digest, timeout_s) + report { + event = 'xfer_commit_tx', + phase = state, + last_tx_type = 'xfer_commit', + sent_bytes = sent, + } + else + pending = make_next_chunk(caps, source, xfer_id, sent, size, chunk_size) + note_max('source_read', pending.source_read_ms) + local _, send_ms = send_chunk(caps, pending) + note_max('send', send_ms) + chunks_sent = chunks_sent + 1 + local need_to_chunk_ms = elapsed_ms(need_at) + note_max('need_to_chunk', need_to_chunk_ms) + report { + event = 'xfer_chunk_tx', + phase = state, + last_tx_type = 'xfer_chunk', + last_tx_offset = pending.offset, + last_tx_next = pending.next, + chunk_len = pending.len, + chunk_digest = pending.digest, + chunk_frame_len = pending.frame_len, + source_read_ms = pending.source_read_ms, + send_ms = send_ms, + need_to_chunk_ms = need_to_chunk_ms, + sent_bytes = sent, + } + deadline = fibers.now() + timeout_s + end + + else + fail(caps, xfer_id, 'unexpected_offset', true) + end + + else + if frame.next ~= sent then fail(caps, xfer_id, 'unexpected_offset', true) end + if sent >= size then + state, deadline = send_commit(caps, xfer_id, size, alg, digest, timeout_s) + report { + event = 'xfer_commit_tx', + phase = state, + last_tx_type = 'xfer_commit', + sent_bytes = sent, + } + else + pending = make_next_chunk(caps, source, xfer_id, sent, size, chunk_size) + note_max('source_read', pending.source_read_ms) + local _, send_ms = send_chunk(caps, pending) + note_max('send', send_ms) + chunks_sent = chunks_sent + 1 + local need_to_chunk_ms = elapsed_ms(need_at) + note_max('need_to_chunk', need_to_chunk_ms) + report { + event = 'xfer_chunk_tx', + phase = state, + last_tx_type = 'xfer_chunk', + last_tx_offset = pending.offset, + last_tx_next = pending.next, + chunk_len = pending.len, + chunk_digest = pending.digest, + chunk_frame_len = pending.frame_len, + source_read_ms = pending.source_read_ms, + send_ms = send_ms, + need_to_chunk_ms = need_to_chunk_ms, + sent_bytes = sent, + } + deadline = fibers.now() + timeout_s + end + end + + end + + elseif frame.type == 'xfer_done' and state == 'committing' then + report { + event = 'xfer_done_rx', + phase = 'done', + last_rx_type = 'xfer_done', + frame_queue_ms = frame_queue_ms, + sent_bytes = sent, + } + return { + request_id = req.request_id, + job_id = type(req.meta) == 'table' and req.meta.job_id or nil, + component = type(req.meta) == 'table' and req.meta.component or nil, + image_id = type(req.meta) == 'table' and req.meta.image_id or nil, + target = target, + xfer_id = xfer_id, + digest_alg = alg, + digest = digest, + sent_bytes = sent, + size = size, + retransmits = retransmits, + begin_retries = begin_retries, + commit_retries = commit_retries, + chunks_sent = chunks_sent, + max_frame_queue_ms = max_frame_queue_ms, + max_need_to_chunk_ms = max_need_to_chunk_ms, + max_source_read_ms = max_source_read_ms, + max_send_ms = max_send_ms, + } + end + end + end +end + +return M diff --git a/src/services/geo.lua b/src/services/geo.lua deleted file mode 100644 index 3b323b54..00000000 --- a/src/services/geo.lua +++ /dev/null @@ -1,13 +0,0 @@ -local fiber = require 'fibers.fiber' -local sleep = require 'fibers.sleep' - -local geo_service = { - name = "geo" -} -geo_service.__index = geo_service - -function geo_service:start(bus_connection) - self.bus_connection = bus_connection -end - -return geo_service diff --git a/src/services/gsm.lua b/src/services/gsm.lua index 51f48728..308a048f 100644 --- a/src/services/gsm.lua +++ b/src/services/gsm.lua @@ -1,875 +1,1672 @@ -local fiber = require "fibers.fiber" +-- services/gsm.lua +-- +-- GSM service (new fibers): +-- - consumes HAL modem capabilities +-- - runs per-modem child scopes for autoconnect + metrics +-- - publishes derived observability metrics and canonical modem state +-- +-- Topics consumed: +-- cfg/ service config (retained) +-- cap/modem/+/state HAL modem capability add/remove events +-- +-- Topics produced: +-- svc//announce service lifecycle announce +-- svc//status service lifecycle status +-- obs/v1/gsm/metric/ per-modem observability metrics +-- obs/v1/gsm/event/ per-modem observability events +-- state/gsm/modem//connected retained: true when APN connected, false otherwise +-- state/gsm/modem//wwan-iface retained: kernel wwan interface name, false when unknown +-- state/gsm/uplink/ retained: canonical cellular uplink record for NET + +local fibers = require "fibers" local op = require "fibers.op" -local channel = require "fibers.channel" -local context = require "fibers.context" local sleep = require "fibers.sleep" -local service = require "service" -local log = require "services.log" -local new_msg = require "bus".new_msg -local apn = require "services.gsm.apn" +local pulse = require "fibers.pulse" + +local perform = fibers.perform + +local base = require 'devicecode.service_base' +local cap_sdk = require 'services.hal.sdk.cap' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local retained_domain = require 'devicecode.support.retained_domain' +local apns = require "services.gsm.apn" +local gsm_topics = require 'services.gsm.topics' +local apn_model = require 'services.gsm.apn_model' +local apn_store_control = require 'services.gsm.apn_store_control_store' +local tablex = require 'shared.table' + +local copy = tablex.deep_copy -local gsm_service = { - name = 'GSM' -} -gsm_service.__index = {} local REQUEST_TIMEOUT = 10 -local DEFAULT_RETRY_TIMEOUT = 5 - --- for now we only hold configs about modems but later may need to hold --- sim and sim switching configs as well -local configs = { - modem = { - default = {}, - imei = {}, - device = {} - } -} +local DEFAULT_RETRY_TIMEOUT = 20 +local DEFAULT_METRICS_INTERVAL = 120 +local DEFAULT_SIGNAL_FREQ = 5 +local APN_SETTLE_TIMEOUT = 3 -- seconds to wait for modem to enter 'connecting' after a failed attempt --- store active modems by preferred id field, imei by default -local modems = { - imei = {}, - device = {}, -} +local SCHEMA_STANDARD = "devicecode.config/gsm/1" local SCOREMAP = { - cdma1x = { rssi = { -110, -100, -86, -70, 1000000 } }, - evdo = { rssi = { -110, -100, -86, -70, 1000000 } }, - gsm = { rssi = { -110, -100, -86, -70, 1000000 } }, - umts = { rscp = { -124, -95, -85, -75, 1000000 } }, - lte = { rsrp = { -115, -105, -95, -85, 1000000 } }, - ["5g"] = { rsrp = { -115, -105, -95, -85, 1000000 } } + cdma1x = { rssi = { -110, -100, -86, -70, 1000000 } }, + evdo = { rssi = { -110, -100, -86, -70, 1000000 } }, + gsm = { rssi = { -110, -100, -86, -70, 1000000 } }, + umts = { rscp = { -124, -95, -85, -75, 1000000 } }, + lte = { rsrp = { -115, -105, -95, -85, 1000000 } }, + ["5g"] = { rsrp = { -115, -105, -95, -85, 1000000 } }, } --- Access technology detection rules: list of required tokens -> access tech name --- Rules are checked in order, first match wins local ACCESS_TECH_MAP = { - { tokens = { 'lte', '5gnr' }, tech = '5g' }, -- Non-standalone 5G - { tokens = { '5gnr' }, tech = '5g' }, -- Standalone 5G NR - { tokens = { '5g' }, tech = '5g' }, - { tokens = { 'lte' }, tech = 'lte' }, - { tokens = { 'umts' }, tech = 'umts' }, - { tokens = { 'gsm' }, tech = 'gsm' }, - { tokens = { 'evdo' }, tech = 'evdo' }, - { tokens = { 'cdma1x' }, tech = 'cdma1x' }, + { tokens = { 'lte', '5gnr' }, tech = '5g' }, + { tokens = { '5gnr' }, tech = '5g' }, + { tokens = { '5g' }, tech = '5g' }, + { tokens = { 'lte' }, tech = 'lte' }, + { tokens = { 'umts' }, tech = 'umts' }, + { tokens = { 'gsm' }, tech = 'gsm' }, + { tokens = { 'evdo' }, tech = 'evdo' }, + { tokens = { 'cdma1x' }, tech = 'cdma1x' }, } -local function purge_sub(ctx, sub) - local msg - while not ctx:err() do - local next_msg, err = sub:next_msg_with_context_op(ctx):perform_alt(function() - return nil, "all messages purged" - end) - if err then break end - msg = next_msg - end - return msg +local STATE_CONNECTED = 'connected' + +-- Topic helpers (centralized so we can remap if needed) +---@param name string +---@return table +local function t_cfg(name) + return { 'cfg', name } end ----@class Modem ----@field conn Connection ----@field ctx Context ----@field idx number ----@field imei string ----@field name string ----@field cfg table -local Modem = {} -Modem.__index = Modem - ---- Autoconnect function attempts to connect the modem to the network using APNs ---- @param ctx Context ---- @param cutoff number ---- @return any apn ---- @return string? error -function Modem:_apn_connect(ctx, cutoff) - cutoff = cutoff or 4 - - -- Subscribe to various modem information topics on the bus - local mcc_sub = self.conn:subscribe( - { 'hal', 'capability', 'modem', self.idx, 'info', 'nas', 'home-network', 'mcc' } - ) - -- This line commonly fails on first try - local mcc_msg, mcc_err = mcc_sub:next_msg_with_context_op(context.with_timeout(ctx, REQUEST_TIMEOUT)):perform() - mcc_sub:unsubscribe() - if mcc_err then return nil, "mcc: " .. mcc_err end - local mcc = mcc_msg.payload - - local mnc_sub = self.conn:subscribe( - { 'hal', 'capability', 'modem', self.idx, 'info', 'nas', 'home-network', 'mnc' } - ) - local mnc_msg, mnc_err = mnc_sub:next_msg_with_context_op(context.with_timeout(ctx, REQUEST_TIMEOUT)):perform() - mnc_sub:unsubscribe() - if mnc_err then return nil, "mnc: " .. mnc_err end - local mnc = mnc_msg.payload - - local imsi_sub = self.conn:subscribe( - { 'hal', 'capability', 'modem', self.idx, 'info', 'sim', 'properties', 'imsi' } - ) - local imsi_msg, imsi_err = imsi_sub:next_msg_with_context_op(context.with_timeout(ctx, REQUEST_TIMEOUT)):perform() - imsi_sub:unsubscribe() - if imsi_err then return nil, "imsi: " .. imsi_err end - local imsi = imsi_msg.payload - - -- local spn, spn_err = self.modem_capability:get_spn() -- this is needed for giffgaff but for now im not doing it because I don't want to - local spn = nil - - local gid1_sub = self.conn:subscribe( - { 'hal', 'capability', 'modem', self.idx, 'info', 'gids', 'gid1' } - ) - local gid1_msg, gid1_err = gid1_sub:next_msg_with_context_op(context.with_timeout(ctx, REQUEST_TIMEOUT)):perform() - gid1_sub:unsubscribe() - if gid1_err then return nil, "gid: " .. gid1_err end - local gid1 = gid1_msg.payload - - -- todo: how to feed apns from configs into this fiber? - -- (this fiber cannot directly access memory from the gsm_manager) - - local status_sub = self.conn:subscribe( - { 'hal', 'capability', 'modem', self.idx, 'info', 'state' } - ) - - -- get apns by rank and iterate through all apns within rank requirements - -- attempt connection on each apn - local apns, ranks = apn.get_ranked_apns(mcc, mnc, imsi, spn, gid1) - for _, n in ipairs(ranks) do - if n.rank > cutoff then break end - local apn_connect_string, string_err = apn.build_connection_string(apns[n.name], self.cfg.roaming_allow) - if string_err == nil then - local connect_sub = self.conn:request(new_msg( - { 'hal', 'capability', 'modem', self.idx, 'control', 'connect' }, - { apn_connect_string } - )) - local connect_msg, ctx_err = connect_sub:next_msg_with_context_op(context.with_timeout(ctx, REQUEST_TIMEOUT)) - :perform() - if ctx_err then - return nil, "connect: " .. ctx_err - end - local connect_err = connect_msg.payload.err - if connect_err == nil then - status_sub:unsubscribe() - return apns[n.name], nil - else - if connect_msg.payload.result and string.find(connect_msg.payload.result, "pdn-ipv4-call-throttled") then - status_sub:unsubscribe() - return nil, "pdn-ipv4-call-throttled" - else - log.debug(string.format( - "%s - %s: Failed to connect to APN '%s', reason: %s (%s)", - ctx:value("service_name"), - ctx:value("fiber_name"), - n.name, - connect_msg.payload.result, - connect_err - )) - end - end - - -- need to wait for current connection attempt to finish before trying another - local modem_status_msg = purge_sub(ctx, status_sub) - local modem_status = modem_status_msg and modem_status_msg.payload and modem_status_msg.payload.curr_state or - nil - while modem_status == 'connecting' do - local status_msg, status_err = status_sub:next_msg_with_context_op(ctx):perform() - if status_err then - status_sub:unsubscribe() - log.debug(status_err); return - end - modem_status = status_msg and status_msg.payload and status_msg.payload.curr_state or modem_status - end - if modem_status == 'connected' then - return apns[n.name], nil - end - end - end - status_sub:unsubscribe() - return nil, "no apn connected" -end - ----Connect modem and set signal report frequency ----@param ctx Context ----@return string? ----@return number? timeout -function Modem:_connect(ctx) - local active_apn, apn_err = self:_apn_connect(ctx) - if apn_err then - local timeout = 20 - if apn_err == "pdn-ipv4-call-throttled" then - timeout = 360 - end - return apn_err, timeout - end - - log.info(string.format( - "%s - %s: CONNECTED", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - local signal_freq = self.cfg.signal_freq or 5 - local signal_sub = self.conn:request(new_msg( - { 'hal', 'capability', 'modem', self.idx, 'control', 'set_signal_update_freq' }, - { signal_freq } - )) - local result, ctx_err = signal_sub:next_msg_with_context_op(ctx):perform() - signal_sub:unsubscribe() - if ctx_err then return ctx_err end - local signal_update_err = result.payload.err - if signal_update_err then return signal_update_err end - - self.cfg.apn = active_apn -end - ----Enable the modem ----@param ctx Context ----@return string? -function Modem:_enable(ctx) - local enable_sub = self.conn:request(new_msg( - { 'hal', 'capability', 'modem', self.idx, 'control', 'enable' }, - {} - )) - local ret_msg, ctx_err = enable_sub:next_msg_with_context_op(ctx):perform() - enable_sub:unsubscribe() - if ctx_err then return ctx_err end +---@param name string +---@param field string +---@return table +local function t_state_gsm_modem(name, field) + return { 'state', 'gsm', 'modem', name, field } +end - local enable_err = ret_msg.payload.err - if enable_err then return enable_err end +local function t_state_gsm_uplink(name) + return { 'state', 'gsm', 'uplink', name } end -function Modem:_enable_failed(ctx) - log.warn(string.format( - "%s - %s: Modem may have failed to register", - ctx:value("service_name"), - ctx:value("fiber_name") - )) + +---@param cap CapabilityReference +---@param method string +---@param payload table? +---@param timeout number? +---@return any +---@return string +local function call_modem_rpc(cap, method, payload, timeout) + local reply, err = perform(cap:call_control_op(method, payload or {}, { + timeout = timeout or REQUEST_TIMEOUT, + })) + if not reply then + return nil, err or "rpc failed" + end + if reply.ok ~= true then + return nil, reply.reason or 'rpc failed' + end + return reply.reason, "" end ----Start warm swap to detect sim insertion ----@param ctx any ----@return string? -function Modem:_detect_sim(ctx) - local sim_detect_sub = self.conn:request(new_msg( - { 'hal', 'capability', 'modem', self.idx, 'control', 'sim_detect' }, - {} - )) - -- this will return a successful dispatch of a detect command, but not if the sim has actually been detected, - -- for that we must listen to the state change - local ret_msg, ctx_err = sim_detect_sub:next_msg_with_context_op(ctx):perform() - sim_detect_sub:unsubscribe() - if ctx_err then return ctx_err end - - local sim_err = ret_msg.payload.err - if sim_err then return sim_err end -end - ----Move a modem out of failed state ----@param ctx Context ----@return string? -function Modem:_fix_failure(ctx) - local fix_failure_sub = self.conn:request(new_msg( - { 'hal', 'capability', 'modem', self.idx, 'control', 'fix_failure' }, - {} - )) - -- this will return a successful dispatch of a fix_failure command, but not if the failure has actually - -- been fixed, for that we must listen to the state change - local ret_msg, ctx_err = fix_failure_sub:next_msg_with_context_op(ctx):perform() - fix_failure_sub:unsubscribe() - if ctx_err then return ctx_err end - - local fix_err = ret_msg.payload.err - if fix_err then return fix_err end -end - -function Modem:_autounlock(ctx) - return "not implemented" -end - ----State machine to automatically connect the modem to an apn and network interface ----@param ctx Context -function Modem:_autoconnect(ctx) - log.trace(string.format( - "%s - %s: Started", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - local states = { - locked = self._autounlock, - failed = self._fix_failure, - no_sim = self._detect_sim, - disabled = self._enable, - enabled = self._enable_failed, - registered = self._connect - } - - local state_monitor_sub = self.conn:subscribe( - { 'hal', 'capability', 'modem', self.idx, 'info', 'state' } - ) - local state_carry_forward = nil - while not ctx:err() do - -- Purge messages that may have been sent during state action - local state_info_msg, monitor_err = purge_sub(ctx, state_monitor_sub) - local state_info = nil - if not state_carry_forward then - if not state_info_msg then - state_info_msg, monitor_err = state_monitor_sub:next_msg_with_context_op(ctx):perform() - end - if monitor_err then - log.debug(monitor_err) - break - end - state_info = state_info_msg.payload - else - state_info = state_carry_forward - state_carry_forward = nil - end - if state_info == nil then - ctx:cancel('state monitor closed') - return - end - log.trace(string.format( - "%s - %s: State recieved: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - state_info.curr_state - )) - - -- Get the relavent function to transition to the next state, if any - local state_func = states[state_info.curr_state] - if state_func then - -- Call the state function and check for errors - local state_err, retry_timeout = state_func(self, ctx) - if state_err then - log.error(string.format( - "%s - %s: State function for state %s failed, reason: %s, retrying after %s seconds", - ctx:value("service_name"), - ctx:value("fiber_name"), - state_info.curr_state, - state_err, - retry_timeout or DEFAULT_RETRY_TIMEOUT - )) - sleep.sleep(retry_timeout or DEFAULT_RETRY_TIMEOUT) - state_carry_forward = state_info - end - end - end - state_monitor_sub:unsubscribe() - log.trace(string.format( - "%s - %s: Closing, reason: '%s'", - ctx:value("service_name"), - ctx:value("fiber_name"), - ctx:err() - )) -end - -local function get_band_class(ctx, conn, idx) - local band_class_sub = conn:subscribe({ 'hal', - 'capability', - 'modem', - idx, - 'info', - 'band', - 'band-information', - 'active-band-class' - }) - local band_class_msg, band_err = band_class_sub:next_msg_with_context_op(ctx):perform() - band_class_sub:unsubscribe() - if band_err then return nil, band_err end - return band_class_msg.payload +---@param cap CapabilityReference +---@param field string +---@param timeout number? +---@param timescale number? +---@return any +---@return string +local function modem_get_field(cap, field, timeout, timescale) + local opts, opts_err = cap_sdk.args.new.ModemGetOpts(field, timescale) + if not opts then + return nil, opts_err or "invalid modem get opts" + end + return call_modem_rpc(cap, 'get', opts, timeout) +end + +---@param cap CapabilityReference +---@param freq number +---@return any +---@return string +local function modem_set_signal_freq(cap, freq) + local opts, opts_err = cap_sdk.args.new.ModemSignalUpdateOpts(freq) + if not opts then + return false, opts_err or "invalid signal update opts" + end + return call_modem_rpc(cap, 'set_signal_update_freq', opts, REQUEST_TIMEOUT) +end + +---@param tbl table? +---@return table +local function shallow_copy(tbl) + local out = {} + if tbl then + for k, v in pairs(tbl) do out[k] = v end + end + return out +end + +---@param cfg table +---@param defaults table +local function apply_defaults(cfg, defaults) + for key, value in pairs(defaults) do + if cfg[key] == nil then + cfg[key] = value + elseif type(value) == 'table' and type(cfg[key]) == 'table' then + apply_defaults(cfg[key], value) + end + end +end + +---@param value any +---@return boolean +local function is_plain_table(value) + return type(value) == 'table' and getmetatable(value) == nil +end + +---@param access_techs any +---@return string +local function derive_access_tech(access_techs) + local techs = {} + if type(access_techs) == 'string' then + for token in string.gmatch(access_techs, "([^,]+)") do + techs[token] = true + end + elseif is_plain_table(access_techs) then + for _, token in ipairs(access_techs) do + techs[token] = true + end + end + + for _, rule in ipairs(ACCESS_TECH_MAP) do + local all_match = true + for _, required_token in ipairs(rule.tokens) do + if not techs[required_token] then + all_match = false + break + end + end + if all_match then + return rule.tech + end + end + + return "" end +---@param access_tech string +---@return string local function get_access_family(access_tech) - local accessfamdict = { - cdma1x = '3G', - evdo = '3G', - gsm = '2G', - umts = '3G', - lte = '4G', - ['5g'] = '5G', - } - if not accessfamdict[access_tech] then - local err = "error invalid access tech" - return nil, err - end - return accessfamdict[access_tech], nil -end - -local function get_imei(ctx, conn, idx) - local imei_sub = conn:subscribe({ 'hal', 'capability', 'modem', idx, 'info', 'modem', 'generic', - 'equipment-identifier' }) - local imei_msg, imei_err = imei_sub:next_msg_with_context_op(ctx):perform() - imei_sub:unsubscribe() - if imei_err then return nil, imei_err end - return imei_msg.payload -end - -function Modem:_publish_static_data(ctx) - local band, band_err = get_band_class(ctx, self.conn, self.idx) - if band_err then log.warn(band_err) end - self.conn:publish(new_msg({ 'gsm', 'modem', self.name, 'band' }, band, { retained = true })) - - local imei, imei_err = get_imei(ctx, self.conn, self.idx) - if imei_err then log.warn(imei_err) end - self.conn:publish(new_msg({ 'gsm', 'modem', self.name, 'imei' }, imei, { retained = true })) + local map = { + cdma1x = '3G', + evdo = '3G', + gsm = '2G', + umts = '3G', + lte = '4G', + ['5g'] = '5G', + } + return map[access_tech] or "" end +---@param access_tech string +---@param signal_type string +---@param signal number +---@return number +---@return string local function get_signal_bars(access_tech, signal_type, signal) - local tech_map = SCOREMAP[access_tech] - if not tech_map then return 0, "error invalid access tech" end - - local map = tech_map[signal_type] - if not map then return 0, "error invalid signal type" end - - for i, threshold in ipairs(map) do - if signal < threshold then - return i, nil - end - end - - return 0, nil -end - -function Modem:_publish_dynamic_data(ctx) - local access_tech = nil - local function publish_access_tech(msg) - -- Parse comma-separated access technologies string - local tech_string = msg.payload - if not tech_string or tech_string == '--' then - return - end - - -- Split the comma-separated string into a set - local technologies = {} - for tech in string.gmatch(tech_string, "([^,]+)") do - technologies[tech] = true - end - - -- Match against ACCESS_TECH_MAP rules - for _, rule in ipairs(ACCESS_TECH_MAP) do - local all_match = true - for _, required_token in ipairs(rule.tokens) do - if not technologies[required_token] then - all_match = false - break - end - end - if all_match then - access_tech = rule.tech - break - end - end - - -- If no rule matched, log and don't publish - if not access_tech then - log.debug(string.format( - "%s - %s: No access tech rule matched for technologies: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - tech_string - )) - return - end - - self.conn:publish(new_msg({ 'gsm', 'modem', self.name, 'access-tech' }, access_tech, { retained = true })) - local access_family, family_err = get_access_family(access_tech) - if family_err then - log.warn(family_err) - return - end - self.conn:publish(new_msg({ 'gsm', 'modem', self.name, 'access-family' }, access_family, { retained = true })) - end - local access_tech_sub = self.conn:subscribe({ 'hal', - 'capability', - 'modem', - self.idx, - 'info', - 'modem', - 'generic', - 'access-technologies' - }) - - local function publish_sim_present(msg) - local state = "--" - if msg.payload and msg.payload ~= '--' then - state = "present" - end - self.conn:publish(new_msg( - { 'gsm', 'modem', self.name, 'sim' }, - state, - { retained = true } - )) - end - local sim_present_sub = self.conn:subscribe({ 'hal', 'capability', 'modem', self.idx, 'info', 'modem', 'generic', - 'sim' }) - - local function publish_signal(msg) - local signal = tonumber(msg.payload) - if not signal then - log.warn(string.format( - "%s - %s: Signal strength (%s) is not a number", - ctx:value("service_name"), - ctx:value("fiber_name"), - msg.payload - )) - return - end - self.conn:publish(new_msg( - { 'gsm', 'modem', self.name, 'signal', msg.topic[8] }, - signal - )) - - local signal_type = msg.topic[#msg.topic] - if signal_type == 'rssi' or signal_type == 'rsrp' or signal_type == 'rscp' then - local bars, bars_err = get_signal_bars(access_tech, signal_type, signal) - - if bars_err then - return - end - - self.conn:publish(new_msg( - { 'gsm', 'modem', self.name, 'bars' }, - bars, - { retained = true } - )) - end - end - local signal_sub = self.conn:subscribe({ 'hal', 'capability', 'modem', self.idx, 'info', 'modem', 'signal', '+' }) - - local function publish_operator(msg) - self.conn:publish(new_msg( - { 'gsm', 'modem', self.name, 'operator' }, - msg.payload or '--', - { retained = true } - )) - end - local operator_sub = self.conn:subscribe({ 'hal', 'capability', 'modem', self.idx, 'info', 'modem', '3gpp', - 'operator-name' }) - - local function publish_iccid(msg) - self.conn:publish(new_msg( - { 'gsm', 'modem', self.name, 'iccid' }, - msg.payload, - { retained = true } - )) - end - local iccid_sub = self.conn:subscribe({ 'hal', 'capability', 'modem', self.idx, 'info', 'sim', 'properties', - 'iccid' }) - - local function publish_state(msg) - self.conn:publish(new_msg( - { 'gsm', 'modem', self.name, 'state' }, - msg.payload, - { retained = true } - )) - end - local state_sub = self.conn:subscribe({ 'hal', 'capability', 'modem', self.idx, 'info', 'state' }) - - local function publish_firmware_version(msg) - self.conn:publish(new_msg( - { 'gsm', 'modem', self.name, 'firmware' }, - msg.payload, - { retained = true } - )) - end - local firmware_sub = self.conn:subscribe({ 'hal', 'capability', 'modem', self.idx, 'info', 'modem', 'firmware' }) - - local function publish_bytes(msg) - self.conn:publish(new_msg( - { 'gsm', 'modem', self.name, msg.topic[#msg.topic] }, - msg.payload - )) - end - local bytes_sub = self.conn:subscribe({ 'hal', 'capability', 'modem', self.idx, 'info', 'modem', 'net', '+' }) - - while not ctx:err() do - op.choice( - access_tech_sub:next_msg_op():wrap(publish_access_tech), - sim_present_sub:next_msg_op():wrap(publish_sim_present), - signal_sub:next_msg_op():wrap(publish_signal), - operator_sub:next_msg_op():wrap(publish_operator), - iccid_sub:next_msg_op():wrap(publish_iccid), - state_sub:next_msg_op():wrap(publish_state), - firmware_sub:next_msg_op():wrap(publish_firmware_version), - bytes_sub:next_msg_op():wrap(publish_bytes), - ctx:done_op() - ):perform() - end - access_tech_sub:unsubscribe() - sim_present_sub:unsubscribe() - signal_sub:unsubscribe() - operator_sub:unsubscribe() - iccid_sub:unsubscribe() - state_sub:unsubscribe() - firmware_sub:unsubscribe() - bytes_sub:unsubscribe() - log.trace(string.format( - "%s - %s: Closing, reason: '%s'", - ctx:value("service_name"), - ctx:value("fiber_name"), - ctx:err() - )) -end - ----Retrieve and publish the network interface to the bus ----@param ctx Context + local tech_map = SCOREMAP[access_tech] + if not tech_map then + return 0, "invalid access tech" + end + + local thresholds = tech_map[signal_type] + if not thresholds then + return 0, "invalid signal type" + end + + for index, threshold in ipairs(thresholds) do + if signal < threshold then + return index, "" + end + end + + return 0, "" +end + +---@param value any ---@return string? -function Modem:_publish_iface(ctx) - local net_iface_sub = self.conn:subscribe( - { 'hal', 'capability', 'modem', self.idx, 'info', 'modem', 'generic', 'ports'} - ) - - local net_interface_msg, interface_err = net_iface_sub:next_msg_with_context_op( - context.with_timeout(ctx, 1) - ):perform() - net_iface_sub:unsubscribe() - - if interface_err then - return interface_err - end - - -- ports is now a comma-separated string like "cdc-wdm0 (qmi),ttyUSB0 (ignored),...,wwan0 (net)" - local ports_string = net_interface_msg.payload - local net_interface = ports_string:match("([^,%(%)%s]+)%s*%(net%)") - - if not net_interface then - return "no net interface found in ports" - end - - self.conn:publish(new_msg( - { 'gsm', 'modem', self.name, 'interface' }, - net_interface, - { retained = true } - )) -end - ---- Either starts or ends autoconnection and updates modem name ---- @param config table -function Modem:update_config(config) - self.cfg = config - - -- every modem is initialised with a name either from config or from the imei - self.name = self.cfg.name or self.imei - -- stop autoconnect if it is enabled and the new config is disabled - if config.autoconnect == false and self.autoconnect_ctx and not self.autoconnect_ctx:err() then - self.autoconnect_ctx:cancel() - -- start autoconnect if it is disabled and the new config is enabled - elseif config.autoconnect == true and (not self.autoconnect_ctx or self.autoconnect_ctx:err()) then - self.autoconnect_ctx = context.with_cancel(self.ctx) - service.spawn_fiber( - string.format("Autoconnect (%s)", self.name), - self.conn, - self.autoconnect_ctx, - function(fiber_ctx) - self:_autoconnect(fiber_ctx) - end - ) - end - fiber.spawn(function() - self:_publish_iface(self.ctx) - self:_publish_static_data(self.ctx) - end) - service.spawn_fiber( - string.format("Reporting (%s)", self.name), - self.conn, - self.ctx, - function(fiber_ctx) - self:_publish_dynamic_data(fiber_ctx) - end - ) -end - ---- Creates a New Modem Capability Class ----@param ctx Context +local function normalize_optional_string(value) + if type(value) ~= 'string' then return nil end + if value == '' or value == '--' then return nil end + return value +end + +local function modem_state_implies_sim_present(modem_state) + return modem_state == 'enabled' + or modem_state == 'searching' + or modem_state == 'registered' + or modem_state == 'connecting' + or modem_state == 'connected' + or modem_state == 'disconnecting' +end + +---@param sim_value any +---@param sim_lock any +---@param sim_lock_retries table? +---@param modem_state string? +---@return table +local function sim_payload_from_value(sim_value, sim_lock, sim_lock_retries, modem_state) + local value_type = type(sim_value) + local present + local state + local lock = nil + local retries = nil + + if value_type == 'table' then + present = sim_value.present + state = normalize_optional_string(sim_value.state) + or normalize_optional_string(sim_value.status) + or normalize_optional_string(sim_value.sim_state) + lock = normalize_optional_string(sim_value.lock or sim_value.sim_lock) + retries = sim_value.lock_retries or sim_value.sim_lock_retries + elseif sim_value == false or sim_value == '--' or sim_value == 'absent' then + present = false + state = 'absent' + elseif sim_value == nil then + present = nil + state = 'unknown' + elseif sim_value == true or sim_value == 'present' then + present = true + state = 'present' + elseif type(sim_value) == 'string' and sim_value ~= '' then + present = true + state = sim_value + else + present = nil + state = 'unknown' + end + + if state == 'missing' or state == 'none' or state == 'not-present' then + state = 'absent' + present = false + end + + if present ~= false and modem_state == 'locked' then + state = 'locked' + present = true + lock = lock or normalize_optional_string(sim_lock) + retries = retries or sim_lock_retries + elseif present == nil and modem_state_implies_sim_present(modem_state) then + state = 'present' + present = true + end + + if present == false then + state = 'absent' + lock = nil + retries = nil + elseif state == nil or state == '' then + state = present == true and 'present' or 'unknown' + end + + return { + present = present, + state = state or 'unknown', + legacy_state = present == false and '--' or nil, + lock = lock, + lock_retries = lock and copy(retries) or nil, + } +end + +---@param sim_value any +---@return string +local function normalize_sim_presence(sim_value) + local sim = sim_payload_from_value(sim_value, nil, nil, nil) + return sim.present == false and '--' or (sim.present == true and 'present' or 'unknown') +end + +---@param connected boolean +---@param modem_state string? +---@param sim_lock string? +---@param sim_state string? +---@return string +local function uplink_state_for_modem(connected, modem_state, _sim_lock, sim_state) + if sim_state == '--' then return 'sim_absent' end + if modem_state == 'locked' then return 'locked' end + if connected == true then return 'connected' end + if modem_state == 'registered' then return 'registered' end + return 'disconnected' +end + +---@param sim_state any +---@param sim_lock string? +---@param sim_lock_retries table? +---@param modem_state string? +---@return table +local function build_sim_payload(sim_state, sim_lock, sim_lock_retries, modem_state) + return sim_payload_from_value(sim_state, sim_lock, sim_lock_retries, modem_state) +end + +---@param access_techs any +---@param rssi any +---@param rsrp any +---@param rscp any +---@return string +---@return number +---@return string +local function select_signal_for_bars(access_techs, rssi, rsrp, rscp) + if not access_techs then + return "", 0, "access tech unavailable" + end + + local access_tech = derive_access_tech(access_techs) + if access_tech == "" then + return "", 0, "access tech unknown" + end + + if access_tech == 'umts' and rscp ~= nil then + local rscp_value = tonumber(rscp) + if rscp_value then + return access_tech, rscp_value, "rscp" + end + end + + if (access_tech == 'lte' or access_tech == '5g') and rsrp ~= nil then + local rsrp_value = tonumber(rsrp) + if rsrp_value then + return access_tech, rsrp_value, "rsrp" + end + end + + if rssi ~= nil then + local rssi_value = tonumber(rssi) + if rssi_value then + return access_tech, rssi_value, "rssi" + end + end + + return access_tech, 0, "" +end + +---@param cfg table? +---@return table +---@return string +local function normalize_config(cfg) + if not is_plain_table(cfg) then + return {}, "config must be a table" + end + ---@cast cfg table + if cfg.schema ~= SCHEMA_STANDARD then + return {}, "config schema must be " .. SCHEMA_STANDARD + end + local modems_cfg = rawget(cfg, 'modems') + if not is_plain_table(modems_cfg) then + return {}, "config.modems must be a table" + end + local modems_default = rawget(modems_cfg, 'default') + if not is_plain_table(modems_default) then + return {}, "config.modems.default must be a table" + end + local modems_known = rawget(modems_cfg, 'known') + if modems_known ~= nil and type(modems_known) ~= 'table' then + return {}, "config.modems.known must be a list" + end + local apn_store = rawget(cfg, 'apn_store') + if apn_store ~= nil and not is_plain_table(apn_store) then + return {}, "config.apn_store must be a table" + end + local out = shallow_copy(cfg) + out.schema = nil + return out, "" +end + +---@param cfg table +---@param imei string|number +---@param device string +---@return table +---@return string +---@return string +local function get_modem_config(cfg, imei, device) + local modem_base = shallow_copy(cfg.modems and cfg.modems.default or {}) + local known = cfg.modems and cfg.modems.known + if type(known) == 'table' then + for _, entry in ipairs(known) do + if is_plain_table(entry) then + local id_field = entry.id_field or 'imei' + if (id_field == 'device' and device ~= "" and entry.device == device) + or (id_field ~= 'device' and (entry.imei == imei)) + then + local merged = shallow_copy(entry) + apply_defaults(merged, modem_base) + return merged, (merged.name or ""), "" + end + end + end + end + + return modem_base, "", "" +end + +--- Waits for a modem connection attempt to settle. +--- Phase 1: waits for the modem to enter 'connecting'. If the modem never enters +--- 'connecting' within APN_SETTLE_TIMEOUT seconds (ModemManager rejected the command +--- immediately), we consider it settled and return. Phase 2: once connecting, waits +--- (without a timeout) until the modem leaves 'connecting'. +--- Expects that the caller has already drained the retained current-state message from +--- the subscription so only live transitions are seen here. +---@param name string +---@param state_sub Subscription +---@param log_fn function? +---@return boolean ok +---@return string error +local function wait_for_connection(name, state_sub, log_fn) + log_fn = log_fn or function(_, _) end + + -- Phase 1: wait for 'connecting', or give up after APN_SETTLE_TIMEOUT seconds. + while true do + local which, msg = perform(op.named_choice({ + state = state_sub:recv_op(), + settle = sleep.sleep_op(APN_SETTLE_TIMEOUT), + })) + + if which == 'settle' then + -- Modem never entered 'connecting'; ModemManager likely rejected the command + -- immediately (e.g. modem busy, wrong state). Treat as settled. + log_fn('debug', { what = 'connection_settled', modem = name }) + return true, "" + end + + if not msg then + return false, "state subscription interrupted" + end + + local state = msg.payload and msg.payload.to + log_fn('debug', { what = 'connection_progress', modem = name, modem_state = tostring(state) }) + + if state == 'connecting' then + break + end + -- Any other state (e.g. a transient non-connecting event): keep waiting. + end + + -- Phase 2: modem entered 'connecting'; wait until it leaves. + while true do + local msg, err = state_sub:recv() + if err then + return false, "state subscription interrupted" + end + local state = msg.payload and msg.payload.to + log_fn('debug', { what = 'connection_progress', modem = name, modem_state = tostring(state) }) + if state and state ~= 'connecting' then + return true, "" + end + end +end + +---@class GsmModem +---@field conn Connection +---@field cap CapabilityReference +---@field id string|number +---@field name string +---@field cfg table +---@field device string +---@field scope Scope? +---@field config_pulse Pulse +---@field svc ServiceBase + +local function id_suffix(id) + id = tostring(id or '') + if #id <= 4 then return id end + return '…' .. id:sub(-4) +end + +local function modem_label(self) + local role = tostring(self.name or self.id or 'modem') + local suffix = id_suffix(self.id) + if suffix ~= '' and suffix ~= role then return role .. ' ' .. suffix end + return role +end + +local function normalised_sim_string(v) + if v == nil then return nil end + local n = normalize_sim_presence(v) + if n == nil then return tostring(v) end + return tostring(n) +end + +local function log_modem_detected(modem, fields) + if not modem or not modem.svc then return end + fields = fields or {} + local key = table.concat({ + tostring(modem.name or ''), + tostring(modem.id or ''), + tostring(modem.device or ''), + tostring(fields.firmware or ''), + tostring(fields.sim or ''), + tostring(fields.ifname or modem.wwan_iface or ''), + }, '|') + if modem._last_inventory_key == key then return end + modem._last_inventory_key = key + local parts = { tostring(modem.name or 'modem'), id_suffix(modem.id), 'detected' } + if fields.firmware then parts[#parts + 1] = 'fw=' .. tostring(fields.firmware) end + if fields.sim ~= nil then parts[#parts + 1] = 'sim=' .. tostring(fields.sim) end + if fields.ifname or modem.wwan_iface then parts[#parts + 1] = 'ifname=' .. tostring(fields.ifname or modem.wwan_iface) end + if modem.device and modem.device ~= '' then parts[#parts + 1] = 'device=' .. tostring(modem.device):match('[^/]+$') end + modem.svc:info('modem_detected', { + summary = table.concat(parts, ' '), + modem = modem.name, + modem_id = tostring(modem.id), + device = modem.device, + firmware = fields.firmware, + sim = fields.sim, + ifname = fields.ifname or modem.wwan_iface, + }) +end + + +local function modem_summary_state(modem) + if not modem then return 'missing' end + if modem.current_state then return tostring(modem.current_state) end + if modem.connected then return 'connected' end + return 'detected' +end + +local function expected_role_count(svc) + local roles = svc and svc._gsm_expected_roles or nil + if type(roles) ~= 'table' then return 0 end + local n = 0 + for _ in pairs(roles) do n = n + 1 end + return n +end + +local function derive_expected_roles(cfg) + local roles = {} + local known = cfg and cfg.modems and cfg.modems.known + if type(known) == 'table' then + for i, entry in ipairs(known) do + if is_plain_table(entry) then + roles[tostring(entry.name or ('modem' .. tostring(i)))] = true + end + end + end + return roles +end + +local function log_gsm_summary(svc, modems, reason) + if not svc then return end + local parts = {} + local roles = {} + for _, modem in pairs(modems or {}) do roles[#roles + 1] = tostring(modem.name or modem.id or 'modem') end + table.sort(roles) + local expected_n = expected_role_count(svc) + if expected_n > 0 and #roles < expected_n then return end + for _, role in ipairs(roles) do + for _, modem in pairs(modems or {}) do + if tostring(modem.name or modem.id or 'modem') == role then + local piece = role .. '=' .. modem_summary_state(modem) + if modem.wwan_iface then piece = piece .. '(' .. tostring(modem.wwan_iface) .. ')' end + parts[#parts + 1] = piece + end + end + end + if #parts == 0 then parts[#parts + 1] = 'modems=none' end + local summary = 'gsm summary ' .. table.concat(parts, ' ') + local tnow = fibers.now() + if svc._gsm_summary_key == summary and (tnow - (svc._gsm_summary_at or 0)) < 600 then return end + svc._gsm_summary_key = summary + svc._gsm_summary_at = tnow + svc:info('gsm_summary', { summary = summary, reason = reason }) +end + +local GsmModem = {} +GsmModem.__index = GsmModem + +---@param cap CapabilityReference +---@param svc ServiceBase +---@return GsmModem +function GsmModem.new(cap, svc) + local self = setmetatable({}, GsmModem) + self.cap = cap + self.conn = cap.conn + self.id = cap.id + self.name = tostring(cap.id) + self.cfg = {} + self.device = "" + self.connected = false + self.wwan_iface = nil + self.modem_state = nil + self.sim_state = nil + self.sim_lock = nil + self.sim_lock_retries = nil + self.last_access = nil + self.last_signal = nil + self.current_state = nil + self.uplink_generation = 0 + self.scope = nil + self.config_pulse = pulse.new() + self.svc = svc + self.domain = retained_domain.new(self.conn, { prefix = { 'state', 'gsm' } }) + return self +end + +---@return nil +function GsmModem:_signal_config_change() + self.config_pulse:signal() +end + +---@param cfg table +---@param name string? +---@return nil +function GsmModem:apply_config(cfg, name) + self.cfg = cfg or {} + if name and name ~= "" then + self.name = name + end + self:_signal_config_change() +end + +---@param key string +---@param value any +---@return nil +function GsmModem:_emit_metric(key, value) + if value == nil then + return + end + local ns_name + if self.name == "primary" then + ns_name = "1" + elseif self.name == "secondary" then + ns_name = "2" + else + return + end + self.svc:obs_metric(key, { + value = value, + namespace = { 'modem', ns_name, key }, + }) +end + +---@param key string +---@param value any +---@return nil +function GsmModem:_emit_event(key, value) + if value == nil then + return + end + self.svc:obs_event(key, { modem = self.name, value = value }) +end + +---@param timescale number? +---@return nil +function GsmModem:_refresh_sim_lock_fields(timescale) + local modem_state, state_err = modem_get_field(self.cap, 'modem_state', REQUEST_TIMEOUT, timescale) + if state_err == "" then + self.modem_state = normalize_optional_string(modem_state) + end + + if self.modem_state ~= 'locked' then + self.sim_lock = nil + self.sim_lock_retries = nil + return + end + + local sim_lock, lock_err = modem_get_field(self.cap, 'sim_lock', REQUEST_TIMEOUT) + if lock_err == "" then + self.sim_lock = normalize_optional_string(sim_lock) + end + + local retries, retries_err = modem_get_field(self.cap, 'sim_lock_retries', REQUEST_TIMEOUT) + if retries_err == "" then + self.sim_lock_retries = retries + end +end + +---@return nil +function GsmModem:_publish_modem_fields() + local state = uplink_state_for_modem(self.connected == true, self.modem_state, self.sim_lock, self.sim_state) + self.domain:retain({ 'modem', self.name, 'state' }, state) + self.domain:retain({ 'modem', self.name, 'sim-state' }, build_sim_payload( + self.sim_state, + self.sim_lock, + self.sim_lock_retries, + self.modem_state + )) +end + +---@return nil + +---@param connected boolean? +---@param iface string? +---@return nil +function GsmModem:_publish_uplink_state(connected, iface) + -- Transitional GSM domain model boundary. HAL owns raw modem facts and + -- cache policy. GSM owns cellular semantics: SIM absent/locked/present, + -- registration, connection, operator validity and APN readiness. UI and NET + -- consume state/gsm/uplink/* rather than HAL fields or obs metrics. + if connected ~= nil then self.connected = connected == true end + if type(iface) == 'string' and iface ~= '' then self.wwan_iface = iface end + self.uplink_generation = (self.uplink_generation or 0) + 1 + + local ifname = self.wwan_iface + local sim = build_sim_payload(self.sim_state, self.sim_lock, self.sim_lock_retries, self.modem_state) + local state = uplink_state_for_modem(self.connected == true, self.modem_state, self.sim_lock, self.sim_state) + local access = {} + local signal_payload = nil + + if sim.present == true and state ~= 'locked' and state ~= 'sim_absent' then + local access_techs, access_err = modem_get_field(self.cap, 'access_techs', REQUEST_TIMEOUT, 0) + local access_tech = access_err == '' and derive_access_tech(access_techs) or '' + if access_tech ~= '' then + access.tech = access_tech + access.family = get_access_family(access_tech) + end + + local operator, operator_err = modem_get_field(self.cap, 'operator', REQUEST_TIMEOUT, 0) + operator = operator_err == '' and normalize_optional_string(operator) or nil + if operator then + access.operator = operator + self.domain:retain({ 'modem', self.name, 'operator' }, operator) + else + self.domain:unretain({ 'modem', self.name, 'operator' }) + end + + local signal, signal_err = modem_get_field(self.cap, 'signal', REQUEST_TIMEOUT, 0) + if signal_err == '' and type(signal) == 'table' then + signal_payload = signal + self.domain:retain({ 'modem', self.name, 'signal' }, signal) + else + self.domain:unretain({ 'modem', self.name, 'signal' }) + end + else + self.domain:unretain({ 'modem', self.name, 'operator' }) + self.domain:unretain({ 'modem', self.name, 'signal' }) + end + + local payload = { + schema = 'devicecode.gsm.uplink/1', + id = self.name, + role = self.name, + state = state, + connected = self.connected == true, + available = self.connected == true and type(ifname) == 'string' and ifname ~= '', + generation = self.uplink_generation, + linux = { ifname = ifname }, + modem = { + id = tostring(self.id), + role = self.name, + device = self.device, + }, + sim = sim, + access = access, + signal = signal_payload, + at = self.svc and self.svc.wall and self.svc:wall() or nil, + } + self:_publish_modem_fields() + if type(ifname) == 'string' and ifname ~= '' then + self.domain:retain({ 'modem', self.name, 'wwan-iface' }, ifname) + else + self.domain:unretain({ 'modem', self.name, 'wwan-iface' }) + end + self.domain:retain({ 'uplink', self.name }, payload) +end + +function GsmModem:_emit_metrics_once() + -- Derived metrics only; HAL remains the source of truth. + local inventory = {} + local access_techs, access_err = modem_get_field(self.cap, 'access_techs', REQUEST_TIMEOUT) + if access_err == "" then + local access_tech = derive_access_tech(access_techs) + if access_tech ~= "" then + inventory.access_tech = access_tech + self:_emit_metric('access_tech', access_tech) + local access_family = get_access_family(access_tech) + if access_family ~= "" then + self:_emit_metric('access_fam', access_family) + end + end + end + + local band, band_err = modem_get_field(self.cap, 'active_band_class', REQUEST_TIMEOUT) + if band_err == "" then + self:_emit_metric('band', band) + end + + local imei, imei_err = modem_get_field(self.cap, 'imei', REQUEST_TIMEOUT) + if imei_err == "" then + inventory.imei = imei + self:_emit_metric('imei', imei) + end + + local operator, operator_err = modem_get_field(self.cap, 'operator', REQUEST_TIMEOUT) + if operator_err == "" then + inventory.operator = operator + self:_emit_metric('operator', operator) + end + + local sim, sim_err = modem_get_field(self.cap, 'sim', REQUEST_TIMEOUT) + if sim_err == "" then + inventory.sim = normalised_sim_string(sim) + self.sim_state = normalize_sim_presence(sim) + self:_emit_metric('sim', self.sim_state) + end + + self:_refresh_sim_lock_fields(0) + self:_emit_metric('modem_state', self.modem_state) + self:_emit_metric('sim_lock', self.sim_lock or '--') + self:_emit_metric('sim_lock_retries', self.sim_lock_retries or {}) + + local iccid, iccid_err = modem_get_field(self.cap, 'iccid', REQUEST_TIMEOUT) + if iccid_err == "" then + self:_emit_metric('iccid', iccid) + end + + local firmware, firmware_err = modem_get_field(self.cap, 'firmware', REQUEST_TIMEOUT) + if firmware_err == "" then + inventory.firmware = firmware + self:_emit_metric('fw_version', firmware) + end + + local state_sub = self.cap:get_state_sub('card') + local state_msg, msg_err = state_sub:recv() + state_sub:unsubscribe() + if msg_err then + self.svc:obs_log('debug', { what = 'state_recv_error', modem = self.name, err = tostring(msg_err) }) + end + + local state = state_msg.payload and state_msg.payload + ---@cast state ModemStateEvent + if state then + self.modem_state = normalize_optional_string(state.to) + self:_emit_metric('state', state.to) + end + + local net_ports, net_ports_err = modem_get_field(self.cap, 'net_ports', REQUEST_TIMEOUT) + if net_ports_err == "" then + local interface = net_ports and net_ports[1] + if interface then + self.wwan_iface = interface + inventory.ifname = interface + self:_emit_metric('wann_type', interface) + self.domain:retain({ 'modem', self.name, 'wwan-iface' }, interface) + self:_publish_uplink_state(nil, interface) + else + self.svc:obs_log('debug', { what = 'no_net_ports', modem = self.name }) + end + end + + local rx_bytes, rx_err = modem_get_field(self.cap, 'rx_bytes', REQUEST_TIMEOUT) + if rx_err == "" then + self:_emit_metric('rx_bytes', rx_bytes) + end + + local tx_bytes, tx_err = modem_get_field(self.cap, 'tx_bytes', REQUEST_TIMEOUT) + if tx_err == "" then + self:_emit_metric('tx_bytes', tx_bytes) + end + + local signal, signal_err = modem_get_field(self.cap, 'signal', REQUEST_TIMEOUT) + local rssi, rsrp, rsrq, rscp = nil, nil, nil, nil + if signal_err == "" then + rssi = signal.rssi + rsrp = signal.rsrp + rsrq = signal.rsrq + rscp = signal.rscp + end + + if rsrp then + self:_emit_metric('rsrp', rsrp) + end + + if rsrq then + self:_emit_metric('rsrq', rsrq) + end + + if rssi then + self:_emit_metric('rssi', rssi) + end + + local bars_access_tech, signal_value, signal_type = select_signal_for_bars( + access_techs, + rssi, + rsrp, + rscp + ) + + if bars_access_tech ~= "" and signal_type ~= "" then + local bars, bars_err = get_signal_bars(bars_access_tech, signal_type, signal_value) + if bars_err == "" then + self:_emit_metric('bars', bars) + end + end + log_modem_detected(self, inventory) +end + +---@return nil +function GsmModem:_metrics_loop() + local seen = self.config_pulse:version() + local interval = tonumber(self.cfg.metrics_interval) or DEFAULT_METRICS_INTERVAL + + while true do + local which, ver = perform(op.named_choice({ + tick = sleep.sleep_op(interval), + config = self.config_pulse:changed_op(seen), + })) + + if which == 'config' then + if not ver then + return + end + seen = ver + interval = tonumber(self.cfg.metrics_interval) or DEFAULT_METRICS_INTERVAL + else + self:_emit_metrics_once() + end + end +end + +---@return table|nil apn +---@return string error +---@return number? retry_timeout +function GsmModem:_apn_connect() + -- Fetch network/SIM identifiers from HAL + local mcc, mcc_err = modem_get_field(self.cap, 'mcc', REQUEST_TIMEOUT) + if mcc_err ~= "" then + return nil, "mcc: " .. mcc_err, DEFAULT_RETRY_TIMEOUT + end + + local mnc, mnc_err = modem_get_field(self.cap, 'mnc', REQUEST_TIMEOUT) + if mnc_err ~= "" then + return nil, "mnc: " .. mnc_err, DEFAULT_RETRY_TIMEOUT + end + + local imsi, imsi_err = modem_get_field(self.cap, 'imsi', REQUEST_TIMEOUT) + if imsi_err ~= "" then + return nil, "imsi: " .. imsi_err, DEFAULT_RETRY_TIMEOUT + end + + local gid1, gid1_err = modem_get_field(self.cap, 'gid1', REQUEST_TIMEOUT) + if gid1_err ~= "" then + return nil, "gid1: " .. gid1_err, DEFAULT_RETRY_TIMEOUT + end + + -- Get ranked APNs + local rank_cutoff = tonumber(self.cfg.apn_rank_cutoff) or 4 + local ranked_apns, rankings = apns.get_ranked_apns(mcc, mnc, imsi, nil, gid1, self.svc and self.svc.custom_apns) + + -- Iterate through ranked APNs + for _, ranking in ipairs(rankings) do + if ranking.rank > rank_cutoff then break end + + local apn_table = ranked_apns[ranking.name] + local conn_str, build_err = apns.build_connection_string(apn_table, self.cfg.roaming_allow) + + if not build_err and conn_str then + -- Build and send connect RPC + local opts, opts_err = cap_sdk.args.new.ModemConnectOpts(conn_str) + if opts then + -- Subscribe before connecting so we capture real state transitions. + -- Retained messages are delivered synchronously into the mailbox during + -- subscribe(), so recv() returns immediately with the stale current state. + -- Draining it here ensures wait_for_connection only sees live transitions. + local state_sub = self.cap:get_state_sub('card', { + queue_len = 4, + full = 'drop_oldest', + }) + state_sub:recv() -- drain the retained current state + + self.svc:obs_log('debug', + { what = 'apn_connect_attempt', modem = self.name, apn = ranking.name, conn_str = conn_str }) + + local _, conn_err = call_modem_rpc(self.cap, 'connect', opts, REQUEST_TIMEOUT) + + self.svc:obs_log('debug', + { what = 'apn_connect_rpc', modem = self.name, apn = ranking.name, err = conn_err }) + if conn_err == "" then + -- Connect succeeded + self.svc:obs_log('info', { what = 'apn_connected', modem = self.name, apn = ranking.name }) + state_sub:unsubscribe() + return apn_table, "", nil + end + + -- Check for throttled error + if string.find(conn_err, "pdn-ipv4-call-throttled") then + self.svc:obs_log('warn', { what = 'apn_throttled', modem = self.name }) + state_sub:unsubscribe() + return nil, conn_err, 360 -- 6-minute backoff + end + + -- Connection attempt failed; wait for modem state to stabilize before trying next APN. + self.svc:obs_log('debug', + { what = 'apn_connect_failed', modem = self.name, apn = ranking.name, err = conn_err }) + + local ok, wait_err = wait_for_connection( + self.name, state_sub, + function(level, payload) self.svc:obs_log(level, payload) end + ) + state_sub:unsubscribe() + if not ok then + self.svc:obs_log('warn', { what = 'wait_for_connection_failed', modem = self.name, err = wait_err }) + return nil, wait_err, DEFAULT_RETRY_TIMEOUT + end + + self.svc:obs_log('warn', { what = 'apn_attempt_failed', modem = self.name, apn = ranking.name }) + else + self.svc:obs_log('debug', + { what = 'apn_invalid_opts', modem = self.name, apn = ranking.name, err = opts_err }) + end + else + self.svc:obs_log('debug', + { what = 'apn_build_failed', modem = self.name, apn = ranking.name, err = build_err or 'nil' }) + end + end + + return nil, "no apn connected", DEFAULT_RETRY_TIMEOUT +end + +-- Autoconnect loop: listens to modem state changes and reacts with enable/fix/connect. +-- Retry logic with exponential backoff and state change preemption. +---@return nil +function GsmModem:_autoconnect_loop() + self.svc:obs_log('debug', self.name .. ": starting autoconnect loop") + local seen = self.config_pulse:version() + local state_sub = self.cap:get_state_sub('card', { + queue_len = 1, + full = 'drop_oldest', + }) + local sim_state_sub = self.cap:get_state_sub('sim_state', { + queue_len = 1, + full = 'drop_oldest', + }) + + local current_state = nil + local connected = false + local backoff = math.huge + + while true do + local which, msg_or_ver = perform(op.named_choice({ + state = state_sub:recv_op(), + sim = sim_state_sub:recv_op(), + backoff = sleep.sleep_op(backoff), + config = self.config_pulse:changed_op(seen), + })) + + if which == 'config' then + if not msg_or_ver then + -- pulse closed, exit + break + end + seen = msg_or_ver + backoff = math.huge + elseif which == 'state' then + local msg = msg_or_ver + if msg then + current_state = msg.payload.to + self.current_state = current_state + self.modem_state = normalize_optional_string(current_state) + if current_state == 'locked' then + self:_refresh_sim_lock_fields(0) + else + self.sim_lock = nil + self.sim_lock_retries = nil + end + self:_emit_metric('modem_state', self.modem_state) + self:_emit_metric('sim_lock', self.sim_lock or '--') + self:_emit_metric('sim_lock_retries', self.sim_lock_retries or {}) + self:_publish_uplink_state(nil, self.wwan_iface) + if current_state ~= self._last_operator_state then + self._last_operator_state = current_state + local level = current_state == 'failed' and 'warn' or 'info' + local what = current_state == 'failed' and 'modem_failed' or 'modem_state_changed' + self.svc:obs_log(level, { + what = what, + summary = string.format('%s modem state -> %s', modem_label(self), tostring(current_state)), + modem = self.name, + modem_id = self.id, + modem_suffix = id_suffix(self.id), + state = current_state, + }) + if self.summary_fn then self.summary_fn('modem_state_changed') end + end + end + elseif which == 'sim' then + local msg = msg_or_ver + if msg then + local sim = sim_payload_from_value(msg.payload, self.sim_lock, self.sim_lock_retries, self.modem_state) + self.sim_state = sim.present == false and '--' or (sim.present == true and 'present' or 'unknown') + if sim.present == false then + self.connected = false + connected = false + self.sim_lock = nil + self.sim_lock_retries = nil + end + self:_emit_metric('sim', self.sim_state) + self:_publish_uplink_state(nil, self.wwan_iface) + end + elseif which == 'backoff' then + self.svc:obs_log('warn', { what = 'autoconnect_retry', modem = self.name, state = current_state }) + end + + -- Act on current_state + if current_state and self.cfg.autoconnect then + local err, retry_timeout + + if current_state == 'disabled' then + local _, err_inner = call_modem_rpc(self.cap, 'enable', {}, REQUEST_TIMEOUT) + err = err_inner + elseif current_state == 'failed' then + local _, err_inner = call_modem_rpc(self.cap, 'listen_for_sim', {}, REQUEST_TIMEOUT) + err = err_inner + elseif current_state == 'registered' then + local _, err_inner, retry_inner = self:_apn_connect() + err = err_inner + retry_timeout = retry_inner + local signal_freq = tonumber(self.cfg.signal_freq) or DEFAULT_SIGNAL_FREQ + local _, sig_err = modem_set_signal_freq(self.cap, signal_freq) + if sig_err ~= "" then + self.svc:obs_log('debug', { what = 'set_signal_freq_failed', modem = self.name, err = sig_err }) + end + end + + local connection_changed = (connected and current_state ~= STATE_CONNECTED) + or (not connected and current_state == STATE_CONNECTED) + + if connection_changed then + connected = current_state == STATE_CONNECTED + self.connected = connected + self.domain:retain({ 'modem', self.name, 'connected' }, connected) + self:_publish_uplink_state(connected, self.wwan_iface) + self.svc:obs_event('connection_changed', { modem = self.name, connected = connected }) + end + + if err and err ~= "" then + backoff = retry_timeout or DEFAULT_RETRY_TIMEOUT + self.svc:obs_log('error', + { + what = 'autoconnect_failed', + modem = self.name, + state = current_state, + err = err, + retry_after = + backoff + }) + else + backoff = math.huge + end + end + end + + state_sub:unsubscribe() + sim_state_sub:unsubscribe() +end + +---@param parent_scope Scope +---@return boolean +---@return string +function GsmModem:start(parent_scope) + if self.scope then + return true, "" + end + + if self.config_pulse:is_closed() then + self.config_pulse = pulse.new() + end + + local child, err = parent_scope:child() + if not child then + return false, err or "failed to create modem scope" + end + + self.scope = child + + child:finally(function() + self.svc:obs_log('debug', { what = 'modem_scope_closed', modem = self.name }) + end) + + local ok, spawn_err = child:spawn(function() + self:_metrics_loop() + end) + if not ok then + return false, spawn_err or "failed to spawn metrics loop" + end + + ok, spawn_err = child:spawn(function() + self:_autoconnect_loop() + end) + if not ok then + return false, spawn_err or "failed to spawn autoconnect loop" + end + + return true, "" +end + +---@param reason string? +---@param close_pulse boolean? +---@return nil +function GsmModem:stop(reason, close_pulse, clear_state) + if not self.scope then + if clear_state == true then self.domain:clear() end + return + end + + if close_pulse then + self.config_pulse:close(reason or 'modem stopped') + end + + self.scope:cancel(reason or 'modem stopped') + perform(self.scope:join_op()) + self.scope = nil + + if clear_state == true then + self.domain:clear() + return + end + + if self.name and self.name ~= "" then + self.connected = false + self.domain:retain({ 'modem', self.name, 'connected' }, false) + self:_publish_uplink_state(false, self.wwan_iface) + end +end + +---@class GsmService +---@field name string +local GsmService = {} + ---@param conn Connection ----@param imei string ----@param index number ----@return Modem -local function new_modem(ctx, conn, imei, device, index) - local self = setmetatable({}, Modem) - self.ctx = context.with_cancel(ctx) - self.conn = conn - - self.idx = index - self.imei = imei - self.device = device - - return self -end - -local function get_modem_config(imei, device) - -- get the modem config from the configs table, if not found use default - local modem_config = configs.modem.imei[imei] or configs.modem.device[device] - local id_field, key - if modem_config then - id_field = modem_config.id_field - key = modem_config[id_field] - else - -- default modems will be stored by imei - id_field = 'imei' - key = imei - modem_config = configs.modem.default - end - - return modem_config, id_field, key -end - -local function modem_capability_remove(modem) - local imei = modem.imei - local device = modem.device - - local modem_config, id_field, key = get_modem_config(imei, device) - - if modems[id_field][key] then - modems[id_field][key].ctx:cancel('modem disconnected') - modems[id_field][key] = nil - log.info(string.format('Modem Capability %s removed', modem.idx)) - end -end - -local function modem_capability_add(modem) - local imei = modem.imei - local device = modem.device - - local modem_config, id_field, key = get_modem_config(imei, device) - - modem:update_config(modem_config) - modems[id_field][key] = modem - log.info(string.format('Modem Capability %s detected', modem.idx)) -end - - -local function init_modem_capability(ctx, conn, modem_add_channel, modem_remove_channel, modem_capability_msg) - if modem_capability_msg.payload == nil then - log.error("GSM - Modem capability message is nil") - return - end - local modem_capability = modem_capability_msg.payload - - -- get imei and device (port) info from bus - local imei, imei_err = get_imei(ctx, conn, modem_capability.index) - if imei_err then - log.error(string.format( - "%s - %s: Error getting modem imei: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - imei_err - )) - return - end - - local modem_device_sub = conn:subscribe({ 'hal', 'capability', 'modem', modem_capability.index, 'info', - 'modem', 'generic', 'device' }) - local device_msg, device_err = modem_device_sub:next_msg_with_context_op(context.with_timeout(ctx, 10)):perform() - modem_device_sub:unsubscribe() - if device_err then - log.error(string.format( - "%s - %s: Error getting modem device: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - device_err - )) - return - end - local device = device_msg.payload - - local modem = new_modem(ctx, conn, imei, device, modem_capability.index) - - if modem_capability.connected then - modem_add_channel:put(modem) - else - modem_remove_channel:put(modem) - end -end - ---- Merge custom config with default to avoid any missing fields ----@param config table custom config ----@param defaults table default config -local function apply_defaults(config, defaults) - if not defaults then return end - for k, v in pairs(defaults) do - if config[k] == nil then - config[k] = v - end - end -end - ---- Applies configs to modems and updates the configs table ----@param config_msg table -local function config_handler(config_msg) - log.trace("GSM received config") - if config_msg and config_msg.payload then - local modem_configs = config_msg.payload.modems - local default_config = modem_configs.default - if not default_config then - log.error("GSM - Config: default config not set") - return - end - configs.modem.default = default_config - - for _, known_config in ipairs(modem_configs.known) do - local id_field = known_config.id_field - if id_field then - apply_defaults(known_config, default_config) - configs.modem[id_field][known_config[id_field]] = known_config - - if modems[id_field][known_config[id_field]] then - modems[id_field][known_config[id_field]]:update_config(known_config) - end - else - log.error('GSM - Config: id_field is not set') - end - end - - for _, modem in ipairs(modems.imei) do - if not modem.cfg.name then - modem:update_config(default_config) - end - end - end -end - ---- Manages creation/removal of modem capabilities and application of configs ----@param ctx Context ----@param conn Connection Bus Connection -local function gsm_manager(ctx, conn) - local capability_sub = conn:subscribe({ 'hal', 'capability', 'modem', '+' }) - local config_sub = conn:subscribe({ 'config', 'gsm' }) - - local modem_add_channel = channel.new() - local modem_remove_channel = channel.new() - - -- load config before anything else - local config, config_err = config_sub:next_msg_with_context_op(ctx):perform() - if config_err then - log.trace(config_err) - config_sub:unsubscribe() - capability_sub:unsubscribe() - return - end - config_handler(config) - - while not ctx:err() do - op.choice( - capability_sub:next_msg_op():wrap(function(capability_msg) - fiber.spawn(function() - init_modem_capability(ctx, conn, modem_add_channel, modem_remove_channel, capability_msg) - end) - end), - modem_add_channel:get_op():wrap(modem_capability_add), - modem_remove_channel:get_op():wrap(modem_capability_remove), - config_sub:next_msg_op():wrap(config_handler), - ctx:done_op() - ):perform() - end - capability_sub:unsubscribe() - config_sub:unsubscribe() - log.trace(string.format("GSM: Manager Closing, reason: '%s'", ctx:err())) -end - ---- Initialise GSM service ----@param ctx Context ----@param conn Connection Bus Connection -function gsm_service:start(ctx, conn) - log.trace("Starting GSM Service") - service.spawn_fiber('Manager', conn, ctx, function(fiber_ctx) - gsm_manager(fiber_ctx, conn) - end) -end - -return gsm_service +---@param opts table? +---@return nil +function GsmService.start(conn, opts) + opts = opts or {} + local name = opts.name or 'gsm' + local heartbeat_s = (type(opts.heartbeat_s) == 'number') and opts.heartbeat_s or 30.0 + + local svc = base.new(conn, { name = name, env = opts.env }) + + svc:obs_state('boot', { at = svc:wall(), ts = svc:now(), state = 'entered' }) + svc:obs_log('debug', 'service start() entered') + svc:announce({}) + svc:starting() + svc:spawn_heartbeat(heartbeat_s, 'tick') + + local current_cfg = {} + local config_ready = false + local custom_apns = {} + local apn_store = nil + local apn_store_loaded = false + local apn_store_retry_delay_s = 1.0 + + -- Transitional APN bridge. GSM is still on its pre-modern service + -- architecture; keep this surface deliberately small until GSM is migrated. + -- UI calls GSM-owned APN RPCs, GSM persists custom APNs through the HAL + -- control-store, and cfg/gsm only points at the store rather than being edited + -- by UI. The later GSM migration should move this to config_watch, capability + -- dependency gating and request_owner/scoped_work handling. + + ---@type table + local modems = {} + + local parent_scope = fibers.current_scope() + + local function apn_store_opts_from_config(cfg) + local spec = cfg and cfg.apn_store or nil + if spec ~= nil and type(spec) ~= 'table' then return nil, 'invalid_apn_store_config' end + spec = spec or {} + local kind = spec.kind or 'control-store' + if kind ~= 'control-store' then return nil, 'unsupported_apn_store_kind:' .. tostring(kind) end + return { + id = spec.id or spec.store_id or apn_store_control.DEFAULT_ID, + key = spec.key or apn_store_control.DEFAULT_KEY, + }, nil + end + + local function publish_apn_state(extra) + extra = extra or {} + local store_desc = apn_store and apn_store:describe() or nil + local payload = { + schema = 'devicecode.gsm.apns.custom/1', + count = #custom_apns, + records = apn_model.redact_list(custom_apns), + store = store_desc, + prototype_admin_network_access = true, + at = svc:wall(), + } + svc:_retain(gsm_topics.custom_apns_state(), payload) + svc:_retain(gsm_topics.apns_status_state(), { + schema = 'devicecode.gsm.apns.status/1', + state = extra.state or 'ready', + ready = extra.ready ~= false, + reason = extra.reason, + store = store_desc, + count = #custom_apns, + at = svc:wall(), + }) + end + + local function publish_gsm_capability() + svc:_retain(gsm_topics.cap_status('main'), { + schema = 'devicecode.cap.status/1', + state = 'available', + available = true, + methods = gsm_topics.apn_methods(), + }) + svc:_retain(gsm_topics.cap_meta('main'), { + schema = 'devicecode.cap.meta/1', + class = 'gsm', + id = 'main', + methods = gsm_topics.apn_methods(), + }) + end + + local function open_apn_store_for_config(cfg) + apn_store_retry_delay_s = 1.0 + local store_opts, err = apn_store_opts_from_config(cfg) + if not store_opts then + apn_store = nil + apn_store_loaded = false + custom_apns = {} + svc.custom_apns = custom_apns + publish_apn_state({ state = 'unavailable', ready = false, reason = err }) + return nil, err + end + apn_store = apn_store_control.new(conn, store_opts) + local records, lerr = perform(apn_store:load_op()) + if not records then + apn_store_loaded = false + apn_store_retry_delay_s = 0 + publish_apn_state({ state = 'degraded', ready = false, reason = lerr or 'apn_store_load_failed' }) + return nil, lerr or 'apn_store_load_failed' + end + apn_store_loaded = true + custom_apns = records + svc.custom_apns = custom_apns + publish_apn_state({ state = 'ready', ready = true }) + return true, nil + end + + local function signal_all_modems() + for _, modem in pairs(modems) do + modem:_signal_config_change() + end + end + + local function apn_store_retry_op() + return fibers.run_scope_op(function () + if apn_store_retry_delay_s > 0 then + perform(sleep.sleep_op(apn_store_retry_delay_s)) + end + if not apn_store then return nil, 'apn_store_unavailable' end + return perform(apn_store:load_op()) + end):wrap(function (st, rep, records, lerr) + if st ~= 'ok' then return nil, tostring(records or lerr or rep or 'apn_store_load_failed') end + return records, lerr + end) + end + + local function handle_apn_store_load_result(records, lerr) + if records then + apn_store_loaded = true + custom_apns = records + svc.custom_apns = custom_apns + publish_apn_state({ state = 'ready', ready = true }) + svc:obs_log('info', { what = 'apn_store_loaded', count = #custom_apns }) + apn_store_retry_delay_s = 1.0 + signal_all_modems() + return true, nil + end + publish_apn_state({ state = 'degraded', ready = false, reason = lerr or 'apn_store_load_failed' }) + if apn_store_retry_delay_s <= 0 then + apn_store_retry_delay_s = 1.0 + else + apn_store_retry_delay_s = math.min(apn_store_retry_delay_s * 2, 10.0) + end + return nil, lerr or 'apn_store_load_failed' + end + + local function replace_custom_apns(records) + if not apn_store then return nil, 'apn_store_unavailable' end + if not apn_store_loaded then return nil, 'apn_store_not_ready' end + local list, lerr = apn_model.normalise_list(records) + if not list then return nil, lerr end + local saved, serr = perform(apn_store:save_op(list)) + if not saved then + publish_apn_state({ state = 'degraded', ready = false, reason = serr or 'apn_store_save_failed' }) + return nil, serr or 'apn_store_save_failed' + end + custom_apns = saved + svc.custom_apns = custom_apns + publish_apn_state({ state = 'ready', ready = true }) + signal_all_modems() + return copy(custom_apns), nil + end + + local function handle_apn_request(method, payload) + if method == 'list-custom-apns' then + if not apn_store_loaded then return false, 'apn_store_not_ready' end + return true, copy(custom_apns) + elseif method == 'replace-custom-apns' then + local records, perr = apn_model.list_from_payload(payload or {}) + if not records then return false, perr end + local saved, err = replace_custom_apns(records) + if not saved then return false, err end + return true, saved + elseif method == 'add-custom-apn' then + local rec, rerr = apn_model.normalise_record((payload and (payload.record or payload)) or {}) + if not rec then return false, rerr end + local next_list = copy(custom_apns) + next_list[#next_list + 1] = rec + local saved, err = replace_custom_apns(next_list) + if not saved then return false, err end + return true, saved + elseif method == 'delete-custom-apn' then + local idx = tonumber(payload and payload.index) + if not idx or idx < 1 or idx % 1 ~= 0 or idx > #custom_apns then return false, 'invalid_index' end + local next_list = copy(custom_apns) + table.remove(next_list, idx) + local saved, err = replace_custom_apns(next_list) + if not saved then return false, err end + return true, saved + end + return false, 'unsupported_apn_method:' .. tostring(method) + end + + local function bind_apn_endpoints() + local endpoints = {} + local function cleanup() + for _, ep in pairs(endpoints) do bus_cleanup.unbind(conn, ep) end + end + parent_scope:finally(cleanup) + for _, method in ipairs(gsm_topics.apn_methods()) do + local method_name = method + local ep, err = bus_cleanup.bind(conn, gsm_topics.rpc(method_name, 'main'), { queue_len = 16 }) + if not ep then return nil, err or ('bind_failed:' .. method_name) end + endpoints[method_name] = ep + local ok, spawn_err = parent_scope:spawn(function () + while true do + local req = perform(ep:recv_op()) + if req == nil then return end + local ok_req, value = handle_apn_request(method_name, req.payload) + if type(req.reply) == 'function' then + req:reply({ ok = ok_req == true, reason = value }) + end + end + end) + if not ok then return nil, spawn_err or ('apn_endpoint_spawn_failed:' .. method_name) end + end + return true, nil + end + + parent_scope:finally(function() + for _, modem in pairs(modems) do + modem:stop(nil, true, true) + end + local _, primary = fibers.current_scope():status() + svc:lifecycle('stopped', { ready = false, reason = tostring(primary or 'scope_exit') }) + svc:obs_log('debug', 'service stopped') + end) + + local function ensure_modem(cap) + local id = cap.id + local modem = modems[id] + if modem then + return modem + end + + modem = GsmModem.new(cap, svc) + modem.summary_fn = function(reason) log_gsm_summary(svc, modems, reason) end + modems[id] = modem + + local device, device_err = modem_get_field(cap, 'device', REQUEST_TIMEOUT) + if device_err ~= "" then + svc:obs_log('debug', { what = 'device_lookup_failed', modem = tostring(id), err = device_err }) + else + modem.device = tostring(device or "") + end + + local cfg, modem_name, _ = get_modem_config(current_cfg, id, modem.device) + modem:apply_config(cfg, modem_name) + log_modem_detected(modem, {}) + + local ok, err = modem:start(parent_scope) + if not ok then + svc:obs_log('error', { what = 'modem_start_failed', modem = tostring(id), err = err }) + end + + return modem + end + + local function remove_modem(id) + local modem = modems[id] + if not modem then + return + end + modem:stop('modem removed', true, true) + modems[id] = nil + log_gsm_summary(svc, modems, 'modem_removed') + end + + local cfg_sub = conn:subscribe(t_cfg(name)) + + while not config_ready do + local which, msg, err = perform(op.named_choice({ + cfg = cfg_sub:recv_op(), + timeout = sleep.sleep_op(REQUEST_TIMEOUT), + })) + + if which == 'timeout' then + svc:obs_log('warn', { what = 'waiting_for_config' }) + else + if not msg then + svc:obs_log('warn', { what = 'config_sub_closed', err = tostring(err) }) + return + end + local cfg_data = msg.payload and msg.payload.data + if not is_plain_table(cfg_data) then + svc:obs_log('warn', { what = 'invalid_config_payload' }) + else + local cfg, cfg_err = normalize_config(cfg_data) + if cfg_err ~= "" then + svc:obs_log('warn', { what = 'invalid_config', err = cfg_err }) + else + current_cfg = cfg + svc._gsm_expected_roles = derive_expected_roles(cfg) + config_ready = true + end + end + end + end + + local store_ok, store_err = open_apn_store_for_config(current_cfg) + if not store_ok then + svc:obs_log('warn', { what = 'apn_store_unavailable', err = tostring(store_err) }) + end + + local endpoints_ok, endpoints_err = bind_apn_endpoints() + if not endpoints_ok then + svc:obs_log('error', { what = 'apn_endpoint_bind_failed', err = tostring(endpoints_err) }) + error(endpoints_err or 'apn_endpoint_bind_failed', 0) + end + publish_gsm_capability() + + local modem_cap_sub = conn:subscribe({ 'cap', 'modem', '+', 'state' }) + + svc:obs_event('config_applied', {}) + svc:running() + svc:obs_log('debug', 'service running') + + while true do + local choices = { + cap = modem_cap_sub:recv_op(), + cfg = cfg_sub:recv_op(), + } + if apn_store and not apn_store_loaded then + choices.apn_store_retry = apn_store_retry_op() + end + + local modem_fault_ops = {} + for id, modem in pairs(modems) do + table.insert(modem_fault_ops, modem.scope:fault_op():wrap(function(_, pr) + return { id = id, primary = pr } + end)) + end + + if #modem_fault_ops > 0 then + choices.modem_fault = op.choice(unpack(modem_fault_ops)) + end + + local which, msg, err = perform(op.named_choice(choices)) + + if not msg and which ~= 'apn_store_retry' then + svc:obs_log('debug', { what = 'subscription_closed', err = tostring(err) }) + return + end + + if which == 'cap' then + local id = msg.topic and msg.topic[3] + if msg.payload == 'added' then + ensure_modem(cap_sdk.new_cap_ref(conn, 'modem', id)) + elseif msg.payload == 'removed' then + remove_modem(id) + else + svc:obs_log('debug', + { what = 'unknown_modem_state', modem = tostring(id), state = tostring(msg.payload) }) + end + elseif which == 'modem_fault' then + local modem = modems[msg.id] + if modem then + svc:obs_log('debug', + { what = 'modem_scope_faulted', modem = tostring(msg.id), err = tostring(msg.primary) }) + modem:stop() + end + elseif which == 'apn_store_retry' then + local _, retry_err = handle_apn_store_load_result(msg, err) + if retry_err then + svc:obs_log('debug', { what = 'apn_store_retry_failed', err = tostring(retry_err) }) + end + elseif which == 'cfg' then + local cfg_data = msg.payload and msg.payload.data + if not is_plain_table(cfg_data) then + svc:obs_log('debug', { what = 'invalid_config_payload' }) + else + local updated_cfg, cfg_err = normalize_config(cfg_data) + if cfg_err ~= "" then + svc:obs_log('debug', { what = 'invalid_config', err = cfg_err }) + else + current_cfg = updated_cfg + svc._gsm_expected_roles = derive_expected_roles(updated_cfg) + local apn_reload_ok, apn_reload_err = open_apn_store_for_config(current_cfg) + if not apn_reload_ok then + svc:obs_log('warn', { what = 'apn_store_reload_failed', err = tostring(apn_reload_err) }) + end + for id, modem in pairs(modems) do + local modem_cfg, modem_name, _ = get_modem_config(current_cfg, id, modem.device) + modem:apply_config(modem_cfg, modem_name) + modem:stop('config change') + modem:start(parent_scope) + end + end + end + end + end +end + +GsmService._test = { + build_sim_payload = build_sim_payload, + uplink_state_for_modem = uplink_state_for_modem, +} + +return GsmService diff --git a/src/services/gsm/apn.lua b/src/services/gsm/apn.lua index 6bdbd87f..c1d9b241 100644 --- a/src/services/gsm/apn.lua +++ b/src/services/gsm/apn.lua @@ -1,17 +1,30 @@ -local binser = require "binser" +local binser = require "shared.binser" +local tablex = require "shared.table" + +local copy = tablex.deep_copy --- ask rich what these fellas do local g_authtypes = {} g_authtypes["0"] = "none" g_authtypes["1"] = "pap" g_authtypes["2"] = "chap" g_authtypes["3"] = "pap|chap" --- deserialise the apn database and return the apn for the sim +local function normalise_mnc(mnc) + if mnc == nil then return nil end + mnc = tostring(mnc) + if #mnc == 1 then return "0" .. mnc end + return mnc +end + +-- deserialise the bundled APN database and return APNs for the SIM. local function get_apns(mcc, mnc) - local apndb = binser.r("etc/apns")[1] - local apns = apndb[mcc][mnc] - return apns + local ok, apndb = pcall(function () return binser.r("etc/apns")[1] end) + if not ok or type(apndb) ~= 'table' then return {} end + local by_mcc = apndb[tostring(mcc or '')] + if type(by_mcc) ~= 'table' then return {} end + local apns = by_mcc[normalise_mnc(mnc) or tostring(mnc or '')] + if type(apns) ~= 'table' then return {} end + return copy(apns) end local function build_connection_string(apn, roaming_allow) @@ -21,7 +34,7 @@ local function build_connection_string(apn, roaming_allow) if k == "apn" then table.insert(a, "apn="..v) elseif k == "user" then table.insert(a, "user="..v) elseif k == "password" then table.insert(a, "password="..v) - elseif k == "authtype" then table.insert(a, "allowed-auth="..g_authtypes[v]) + elseif k == "authtype" and g_authtypes[tostring(v)] then table.insert(a, "allowed-auth="..g_authtypes[tostring(v)]) end end if roaming_allow then table.insert(a, "allow-roaming=true") end @@ -29,45 +42,96 @@ local function build_connection_string(apn, roaming_allow) return conn_string, nil end --- the connect function takes a list of apns and applies -local function rank(apns, imsi, spn, gid1) - -- first comes MVNO matches, next general MNO APNs, then generic "apn=internet", finally non-match MVNO - local rankings = {} - for k, v in pairs(apns) do - -- print("k is: ", k) - if v.mvno_type then - -- print("v is: ", v.mvno_type) - if v.mvno_type == "spn" and spn and string.find(spn, v.mvno_match_data) then - table.insert(rankings, {name=k, rank=1}) - elseif v.mvno_type == "gid" and gid1 and string.find(gid1, v.mvno_match_data) then - table.insert(rankings, {name=k, rank=1}) - elseif v.mvno_type == "imsi" and string.find(imsi, v.mvno_match_data) then - table.insert(rankings, {name=k, rank=1}) - else - table.insert(rankings, {name=k, rank=4}) - end +local function mvno_rank(apn, imsi, spn, gid1, ranks) + ranks = ranks or { match = 1, plain = 2, mismatch = 4 } + if apn.mvno_type then + local match_data = tostring(apn.mvno_match_data or '') + if apn.mvno_type == "spn" and spn and string.find(tostring(spn), match_data, 1, true) then + return ranks.match + elseif apn.mvno_type == "gid" and gid1 and string.find(tostring(gid1), match_data, 1, true) then + return ranks.match + elseif apn.mvno_type == "imsi" and imsi and string.find(tostring(imsi), match_data, 1, true) then + return ranks.match else - table.insert(rankings, {name=k, rank=2}) + return ranks.mismatch + end + end + return ranks.plain +end + +local function rank(apns, imsi, spn, gid1, prefix, ranks) + local out_apns = {} + local rankings = {} + prefix = prefix or '' + for k, v in pairs(apns or {}) do + if type(v) == 'table' then + local name = prefix .. tostring(k) + out_apns[name] = copy(v) + table.insert(rankings, { name = name, rank = mvno_rank(v, imsi, spn, gid1, ranks) }) end end - table.insert(apns, {default={apn='internet'}}) - table.insert(rankings, {name='default', rank=3}) - table.sort(rankings, function (k1, k2) return k1.rank < k2.rank end ) - return apns, rankings + return out_apns, rankings end -local function get_ranked_apns(mcc, mnc, imsi, spn, gid1) +local function custom_matches(apn, mcc, mnc) + if type(apn) ~= 'table' then return false end + return tostring(apn.mcc or '') == tostring(mcc or '') + and normalise_mnc(apn.mnc) == normalise_mnc(mnc) +end + +local function custom_apns_for(custom_records, mcc, mnc) + local out = {} + if type(custom_records) ~= 'table' then return out end + for i, apn in ipairs(custom_records) do + if custom_matches(apn, mcc, mnc) then + out['custom-' .. tostring(i)] = apn + end + end + return out +end + +local function add_default(apns, rankings) + apns.default = { apn = 'internet' } + table.insert(rankings, { name = 'default', rank = 4 }) +end + +local function merge_into(dst_apns, dst_rankings, src_apns, src_rankings) + for name, apn in pairs(src_apns or {}) do dst_apns[name] = apn end + for _, r in ipairs(src_rankings or {}) do dst_rankings[#dst_rankings + 1] = r end +end + +local function get_ranked_apns(mcc, mnc, imsi, spn, gid1, custom_records) if mnc == nil then return {}, {} end - if #mnc == 1 then mnc = "0"..mnc end - -- get apns from the network service - local apns = get_apns(mcc, mnc) - -- or read directly - -- local apns = binser.r("etc/apns")[1] - local rankapns, rankings = rank(apns, imsi, spn, gid1) - return rankapns, rankings + mnc = normalise_mnc(mnc) + + local ranked_apns, rankings = {}, {} + + local custom_map = custom_apns_for(custom_records, mcc, mnc) + local custom_apns, custom_rankings = rank(custom_map, imsi, spn, gid1, '', { + match = 1, + plain = 1, + mismatch = 5, + }) + merge_into(ranked_apns, rankings, custom_apns, custom_rankings) + + local builtin = get_apns(mcc, mnc) + local builtin_apns, builtin_rankings = rank(builtin, imsi, spn, gid1, 'builtin-', { + match = 2, + plain = 3, + mismatch = 5, + }) + merge_into(ranked_apns, rankings, builtin_apns, builtin_rankings) + + add_default(ranked_apns, rankings) + table.sort(rankings, function (k1, k2) + if k1.rank == k2.rank then return tostring(k1.name) < tostring(k2.name) end + return k1.rank < k2.rank + end) + return ranked_apns, rankings end return { get_ranked_apns = get_ranked_apns, - build_connection_string = build_connection_string + build_connection_string = build_connection_string, + get_apns = get_apns, } diff --git a/src/services/gsm/apn_model.lua b/src/services/gsm/apn_model.lua new file mode 100644 index 00000000..393f2010 --- /dev/null +++ b/src/services/gsm/apn_model.lua @@ -0,0 +1,139 @@ +-- services/gsm/apn_model.lua +-- +-- Pure validation/normalisation for user-supplied APN records. + +local tablex = require 'shared.table' + +local M = {} + +local copy = tablex.deep_copy + +local ALLOWED_KEYS = { + carrier = true, + mcc = true, + mnc = true, + apn = true, + type = true, + protocol = true, + roaming_protocol = true, + user_visible = true, + authtype = true, + mmsc = true, + mmsproxy = true, + mmsport = true, + proxy = true, + port = true, + bearer_bitmask = true, + read_only = true, + user = true, + password = true, + mvno_type = true, + mvno_match_data = true, + mtu = true, + ppp_number = true, + vivoentry = true, + server = true, + localized_name = true, + visit_area = true, + bearer = true, + profile_id = true, + modem_cognitive = true, + max_conns = true, + max_conns_time = true, + skip_464xlat = true, + carrier_enabled = true, + mtusize = true, + auth = true, +} + +local function trim(s) + return tostring(s or ''):gsub('^%s+', ''):gsub('%s+$', '') +end + +local function is_digits(s, n1, n2) + if type(s) ~= 'string' then return false end + if not s:match('^%d+$') then return false end + return #s >= n1 and #s <= n2 +end + +local function normalise_string(v) + if v == nil then return nil end + if type(v) ~= 'string' and type(v) ~= 'number' and type(v) ~= 'boolean' then return nil, 'field must be scalar' end + local s = trim(v) + if s == '' then return nil end + return s +end + +function M.normalise_record(rec) + if type(rec) ~= 'table' then return nil, 'apn record must be a table' end + local out = {} + for k, v in pairs(rec) do + if type(k) == 'string' and ALLOWED_KEYS[k] then + local nv, err = normalise_string(v) + if err then return nil, k .. ': ' .. err end + if nv ~= nil then out[k] = nv end + end + end + if type(out.carrier) ~= 'string' or out.carrier == '' then return nil, 'carrier is required' end + if not is_digits(out.mcc, 3, 3) then return nil, 'mcc must be three digits' end + if not is_digits(out.mnc, 1, 3) then return nil, 'mnc must be one to three digits' end + if type(out.apn) ~= 'string' or out.apn == '' then return nil, 'apn is required' end + if out.mvno_type ~= nil then + local mt = out.mvno_type:lower() + if mt ~= 'spn' and mt ~= 'gid' and mt ~= 'imsi' then return nil, 'mvno_type must be spn, gid or imsi' end + out.mvno_type = mt + if type(out.mvno_match_data) ~= 'string' or out.mvno_match_data == '' then + return nil, 'mvno_match_data is required when mvno_type is set' + end + end + return out, nil +end + +function M.normalise_list(records) + if records == nil then return {}, nil end + if type(records) ~= 'table' then return nil, 'apns must be a list' end + local out = {} + local seen = {} + for i, rec in ipairs(records) do + local item, err = M.normalise_record(rec) + if not item then return nil, 'apns[' .. tostring(i) .. ']: ' .. tostring(err) end + local key = table.concat({ item.carrier, item.mcc, item.mnc, item.apn }, '\0') + if seen[key] then return nil, 'duplicate APN: ' .. item.carrier .. '/' .. item.mcc .. '/' .. item.mnc .. '/' .. item.apn end + seen[key] = true + out[#out + 1] = item + end + return out, nil +end + +function M.list_from_payload(payload) + if type(payload) ~= 'table' then return nil, 'payload must be a table' end + if payload.records ~= nil then return M.normalise_list(payload.records) end + return M.normalise_list(payload) +end + +local SECRET_KEYS = { + user = true, + password = true, + auth = true, +} + +function M.redact_record(record) + local out = copy(record or {}) + for key in pairs(SECRET_KEYS) do + if out[key] ~= nil then + out[key] = nil + out['has_' .. key] = true + end + end + return out +end + +function M.redact_list(records) + local out = {} + for i, record in ipairs(records or {}) do + out[i] = M.redact_record(record) + end + return out +end + +return M diff --git a/src/services/gsm/apn_store_control_store.lua b/src/services/gsm/apn_store_control_store.lua new file mode 100644 index 00000000..8de021f7 --- /dev/null +++ b/src/services/gsm/apn_store_control_store.lua @@ -0,0 +1,112 @@ +-- services/gsm/apn_store_control_store.lua +-- +-- Control-store-backed custom APN storage. GSM owns this adapter; UI routes call +-- GSM capabilities and never address the HAL control-store directly. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local cjson = require 'cjson.safe' + +local cap_args = require 'services.hal.types.capability_args' +local apn_model = require 'services.gsm.apn_model' + +local M = {} + +local Store = {} +Store.__index = Store + +local DEFAULT_ID = 'gsm' +local DEFAULT_KEY = 'custom-apns-v1' + +local function rpc_topic(id, method) + return { 'cap', 'control-store', id or DEFAULT_ID, 'rpc', method } +end + +local function unwrap(method) + return function(reply, err) + if reply == nil then return nil, err end + if type(reply) ~= 'table' or type(reply.ok) ~= 'boolean' then return nil, 'invalid_control_store_reply' end + if reply.ok then + if reply.reason ~= nil then return reply.reason, nil end + if method == 'put' or method == 'delete' then return true, nil end + return nil, nil + end + return nil, tostring(reply.reason or err or ('control_store_' .. method .. '_failed')) + end +end + +local function call_op(self, method, payload) + if type(self._conn) ~= 'table' or type(self._conn.call_op) ~= 'function' then + return op.always(nil, 'control_store_connection_required') + end + return self._conn:call_op(rpc_topic(self._id, method), payload or {}, self._call_opts) + :wrap(unwrap(method)) +end + +local function decode_records(body) + if body == nil then return {}, nil end + if type(body) ~= 'string' then return nil, 'custom_apns_body_not_string' end + local payload, derr = cjson.decode(body) + if type(payload) ~= 'table' then return nil, 'custom_apns_json_invalid:' .. tostring(derr) end + local records = payload.records or payload.apns or payload + return apn_model.normalise_list(records) +end + +function Store:load_op() + return fibers.run_scope_op(function () + local opts, oerr = cap_args.new.ControlStoreGetOpts(self._key) + if not opts then return nil, oerr or 'invalid_control_store_get_opts' end + local body, err = fibers.perform(call_op(self, 'get', opts)) + if body == nil then + if tostring(err or '') == 'not found' then return {}, nil end + return nil, err or 'control_store_get_failed' + end + return decode_records(body) + end):wrap(function(st, rep, records, err) + if st ~= 'ok' then return nil, tostring(records or err or rep) end + return records, err + end) +end + +function Store:save_op(records) + return fibers.run_scope_op(function () + local list, lerr = apn_model.normalise_list(records) + if not list then return nil, lerr end + local body, berr = cjson.encode({ + schema = 'devicecode.gsm.custom-apns/1', + records = list, + }) + if type(body) ~= 'string' then return nil, 'custom_apns_json_encode_failed:' .. tostring(berr) end + local opts, oerr = cap_args.new.ControlStorePutOpts(self._key, body) + if not opts then return nil, oerr or 'invalid_control_store_put_opts' end + local ok, err = fibers.perform(call_op(self, 'put', opts)) + if ok == nil then return nil, err or 'control_store_put_failed' end + return list, nil + end):wrap(function(st, rep, records, err) + if st ~= 'ok' then return nil, tostring(records or err or rep) end + return records, err + end) +end + +function Store:describe() + return { + kind = 'control-store', + id = self._id, + key = self._key, + } +end + +function M.new(conn, opts) + opts = opts or {} + return setmetatable({ + _conn = conn, + _id = opts.id or opts.store_id or DEFAULT_ID, + _key = opts.key or DEFAULT_KEY, + _call_opts = opts.call_opts, + }, Store) +end + +M.DEFAULT_ID = DEFAULT_ID +M.DEFAULT_KEY = DEFAULT_KEY +M.Store = Store +return M diff --git a/src/services/gsm/topics.lua b/src/services/gsm/topics.lua new file mode 100644 index 00000000..68c793c6 --- /dev/null +++ b/src/services/gsm/topics.lua @@ -0,0 +1,50 @@ +-- services/gsm/topics.lua +-- +-- Topic helpers for GSM-owned state and capability surfaces. + +local M = {} + +local function t(...) return { ... } end + +function M.config(name) + return t('cfg', name or 'gsm') +end + +function M.modem_state(name, field) + return t('state', 'gsm', 'modem', name, field) +end + +function M.uplink(name) + return t('state', 'gsm', 'uplink', name) +end + +function M.custom_apns_state() + return t('state', 'gsm', 'apns', 'custom') +end + +function M.apns_status_state() + return t('state', 'gsm', 'apns', 'status') +end + +function M.rpc(method, id) + return t('cap', 'gsm', id or 'main', 'rpc', method) +end + +function M.cap_status(id) + return t('cap', 'gsm', id or 'main', 'status') +end + +function M.cap_meta(id) + return t('cap', 'gsm', id or 'main', 'meta') +end + +function M.apn_methods() + return { + 'list-custom-apns', + 'replace-custom-apns', + 'add-custom-apn', + 'delete-custom-apn', + } +end + +return M diff --git a/src/services/hal.lua b/src/services/hal.lua index d00fb938..b61bcec4 100644 --- a/src/services/hal.lua +++ b/src/services/hal.lua @@ -1,374 +1,1563 @@ -local modem_manager = require "services.hal.managers.modemcard" -local ubus_manager = require "services.hal.managers.ubus" -local uci_manager = require "services.hal.managers.uci" -local wlan_managaer = require "services.hal.managers.wlan" -local fiber = require "fibers.fiber" -local queue = require "fibers.queue" -local op = require "fibers.op" -local service = require "service" -local new_msg = require("bus").new_msg -local log = require "services.log" -local unpack = table.unpack or unpack - ----@class hal_service -local hal_service = { - name = "hal", - capabilities = {}, - devices = {}, - capability_info_q = queue.new(50), - device_event_q = queue.new(10), - managers = {} +-- HAL modules +local types = require "services.hal.types.core" +local base = require "devicecode.service_base" +local Logger = require "services.hal.logger" + +-- Fibers modules +local fibers = require "fibers" +local op = require "fibers.op" +local channel = require "fibers.channel" +local sleep = require "fibers.sleep" + +local tablex = require 'shared.table' +local dependencies = require 'services.hal.dependencies' + +local perform = fibers.perform + +local SCHEMA_STANDARD = "devicecode.config/hal/1" + +local DEFAULT_Q_LEN = 10 +local DEFAULT_CONTROL_TIMEOUT_S = 30.0 +local DEFAULT_MANAGER_START_TIMEOUT_S = 10.0 +local DEFAULT_MANAGER_APPLY_TIMEOUT_S = 10.0 +local DEFAULT_MANAGER_APPLY_TIMEOUTS = { + wired = 20.0, } -hal_service.__index = hal_service - ----Adds a device, its capabilities and events to the service ----@param device table ----@param capabilities table ----@return table device ----@return string? error -function hal_service:_register_device(device, capabilities) - if not self.devices[device.type] then - self.devices[device.type] = {} - end - device.capabilities = {} - for cap_name, cap in pairs(capabilities) do - if not cap.id then - log.error(string.format( - "Device Event: capability '%s' for device '%s' with id '%s' does not have an id", - cap_name, device.type, device.id - )) - else - if not self.capabilities[cap_name] then - self.capabilities[cap_name] = {} - end - - if self.capabilities[cap_name][cap.id] then - log.debug(string.format( - "Device Event: capability '%s' with id '%s' already exists, overwriting", - cap_name, cap.id - )) - end - self.capabilities[cap_name][cap.id] = cap.control - device.capabilities[cap_name] = cap.id - end - end - if self.devices[device.type][device.id] then - log.debug(string.format( - "Device Event: device '%s' with id '%s' already exists, overwriting", - device.type, device.id - )) - end - self.devices[device.type][device.id] = device - - return device -end - ----Removes a device, its capabilities and events from the service ----@param device table ----@return table? device ----@return string? error -function hal_service:_unregister_device(device) - -- retrieve full device - device = self.devices[device.type] and self.devices[device.type][device.id] - if not device then return nil, "removed device does not exist" end - self.devices[device.type][device.id] = nil - - -- remove capabilities and event channels - for cap_name, cap_id in pairs(device.capabilities) do - self.capabilities[cap_name][cap_id] = nil - self.conn:publish(new_msg( - { 'hal', 'capability', cap_name, cap_id, '#' }, - nil, - { retained = true } - )) - end - - return device -end - ----Checks that all required fields are present in the connection event ----@param connection_event table ----@return boolean ----@return string? error -local function connection_event_valid(connection_event) - if not connection_event.type then - return false, "missing device type" - end - if connection_event.connected == nil then - return false, "missing device connected status" - end - if not connection_event.id_field then - return false, "missing device id field" - end - if not connection_event.data then - return false, "missing device data" - end - if connection_event.connected and not connection_event.capabilities then - return false, "missing device capabilities" - end - return true -end - ----Handles device connection and disconnection events ----@param connection_event table -function hal_service:_handle_device_connection_event(connection_event) - local valid, err = connection_event_valid(connection_event) - if not valid then - log.error("Device Event: " .. err) - return - end - - local device_capabilities = connection_event.capabilities - - -- create a basic device instance, this will be built - -- up further in the register and unregister functions - local device = { - type = connection_event.type, - connected = connection_event.connected, - id = connection_event.data[connection_event.id_field], - data = connection_event.data - } - - if device.connected then - local full_device, register_err = self:_register_device(device, device_capabilities) - if register_err then - log.error("Device Event: " .. register_err) - return - end - device = full_device - else - local full_device, unregister_err = self:_unregister_device(device) - if unregister_err then - log.error("Device Event: " .. unregister_err) - return - end - device = full_device - -- since we now have the original device instance, change to disconnected - device.connected = false - end - - -- broadcast the capabilities and device connections/removals on the bus - - for cap_name, cap_id in pairs(device.capabilities) do - self.conn:publish(new_msg( - { 'hal', 'capability', cap_name, cap_id }, - { - connected = device.connected, - type = cap_name, - index = cap_id, - device = { type = device.type, index = device.id } - }, - { retained = true } - )) - end - - self.conn:publish(new_msg( - { 'hal', 'device', device.type, device.id }, - { - connected = device.connected, - type = device.type, - index = device.id, - -- in this case device refers to the specific type of hardware - identity = device.data.device, - metadata = device.data - }, - { retained = true } - )) -end - ----Uses device driver to execute control commands ----@param request table -function hal_service:_handle_capability_control(request) - local capability, instance_id, method = request.topic[3], request.topic[4], request.topic[6] - - local cap = self.capabilities[capability] - if cap == nil then - if request.reply_to then - local msg = new_msg({ request.reply_to }, { result = nil, err = 'capability does not exist' }) - self.conn:publish(msg) - end - return - end - - local instance = cap[instance_id] - if instance == nil then - if request.reply_to then - local msg = new_msg({ request.reply_to }, { result = nil, err = 'capability instance does not exist' }) - self.conn:publish(msg) - end - return - end - - local func = instance[method] - if func == nil then - if request.reply_to then - local msg = new_msg({ request.reply_to }, { result = nil, err = 'endpoint does not exist' }) - self.conn:publish(msg) - end - return - end - - -- execute capability asynchronously - fiber.spawn(function() - -- unpack arguments to function - local ret = func(instance, request.payload) - if request.reply_to then - local msg = new_msg({ request.reply_to }, { - result = ret.result, - err = ret.err - }) - self.conn:publish(msg) - end - end) -end - -function hal_service:_handle_capbility_info(data) - if not data then return end - - if not data.type then - log.error(string.format( - '%s - %s: Capability info message does not have a type field', - self.ctx:value("service_name"), - self.ctx:value("fiber_name") - )) - return - end - - if not data.id then - log.error(string.format( - '%s - %s: Capability info message does not have an id field', - self.ctx:value("service_name"), - self.ctx:value("fiber_name") - )) - return - end - - if not data.endpoints then - log.error(string.format( - '%s - %s: Capability info message must define an endpoints field ("single" or "multiple")', - self.ctx:value("service_name"), - self.ctx:value("fiber_name") - )) - return - end - - local sub_topic = data.sub_topic or {} - local topic = { 'hal', 'capability', data.type, data.id, 'info', unpack(sub_topic) } - - if data.endpoints == 'single' then - self.conn:publish(new_msg( - topic, - data.info, - { retained = true } - )) - elseif data.endpoints == 'multiple' then - self.conn:publish_multiple( - topic, - data.info, - { retained = true } - ) - end -end - -function hal_service:_apply_config(msg) - log.trace(string.format( - '%s - %s: Recieved new HAL config', - self.ctx:value("service_name"), - self.ctx:value("fiber_name") - )) - if not msg.payload or type(msg.payload) ~= 'table' then - log.error(string.format( - '%s - %s: Config message does not have a valid payload', - self.ctx:value("service_name"), - self.ctx:value("fiber_name") - )) - return - end - - local config = msg.payload - if not config.managers or type(config.managers) ~= 'table' then - log.error(string.format( - '%s - %s: Config message does not have a valid managers field', - self.ctx:value("service_name"), - self.ctx:value("fiber_name") - )) - return - end - - local manager_configs = config.managers - -- Remove managers that are not in the new config - for manager_name, manager in pairs(self.managers) do - if not manager_configs[manager_name] then - manager.ctx:cancel('manager removed') - self.managers[manager_name] = nil - end - end - - for manager_name, manager_config in pairs(manager_configs) do - local manager = self.managers[manager_name] - if manager then - manager:apply_config(manager_config) - else - local manager_pkg = require('services.hal.managers.' .. manager_name) - self.managers[manager_name] = manager_pkg.new() - self.managers[manager_name]:spawn(self.ctx, self.conn, self.device_event_q, self.capability_info_q) - self.managers[manager_name]:apply_config(manager_config) - end - end -end - ----Handles capability and device control, and connection events -function hal_service:_control_main(ctx) - local cap_ctrl_sub, cap_sub_err = self.conn:subscribe({ 'hal', 'capability', '+', '+', 'control', '+' }) - if cap_sub_err ~= nil then - log.error(string.format( - '%s - %s: Failed to subscribe to capability control topic, reason: %s', - ctx:value("service_name"), - ctx:value("fiber_name"), - cap_sub_err - )) - return - end - - local config_sub, config_sub_err = self.conn:subscribe({ 'config', 'hal' }) - if config_sub_err ~= nil then - log.error(string.format( - '%s - %s: Failed to subscribe to config topic, reason: %s', - ctx:value("service_name"), - ctx:value("fiber_name"), - config_sub_err - )) - return - end - - -- All interactions with devices and capabilities run on the same fiber - while not ctx:err() do - op.choice( - config_sub:next_msg_op():wrap(function(msg) self:_apply_config(msg) end), - cap_ctrl_sub:next_msg_op():wrap(function(msg) self:_handle_capability_control(msg) end), - self.device_event_q:get_op():wrap(function(msg) self:_handle_device_connection_event(msg) end), - self.capability_info_q:get_op():wrap(function(msg) self:_handle_capbility_info(msg) end), - ctx:done_op() - ):perform() - end - cap_ctrl_sub:unsubscribe() -end - ----Spin off all HAL service fibers ----@param ctx Context ----@param conn Connection -function hal_service:start(ctx, conn) - log.trace(string.format( - "%s: Starting", - ctx:value("service_name") - )) - self.ctx = ctx - self.conn = conn - - -- start main control loop - service.spawn_fiber('Control', conn, ctx, function(control_ctx) - self:_control_main(control_ctx) - end) -end - -return hal_service +local DEFAULT_MANAGER_SHUTDOWN_TIMEOUT_S = 5.0 + +local unpack = rawget(table, 'unpack') or _G.unpack +local pack = rawget(table, 'pack') or function (...) + return { n = select('#', ...), ... } +end + +---------------------------------------------------------------------- +-- Topic namespace +---------------------------------------------------------------------- + +local T = {} + +function T.dev_meta(class, id) return { 'dev', class, id, 'meta' } end +function T.dev_state(class, id) return { 'dev', class, id, 'state' } end + +function T.cap_meta(class, id) return { 'cap', class, id, 'meta' } end +function T.cap_legacy_state(class,id) return { 'cap', class, id, 'state' } end +function T.cap_status(class, id) return { 'cap', class, id, 'status' } end +function T.cap_state(class, id, key) return { 'cap', class, id, 'state', key } end +function T.cap_event(class, id, name) return { 'cap', class, id, 'event', name } end +function T.cap_rpc(class, id, verb) return { 'cap', class, id, 'rpc', verb } end + +function T.raw_source_meta(src) return { 'raw', 'host', src, 'meta' } end +function T.raw_source_status(src) return { 'raw', 'host', src, 'status' } end + +local function is_wired_provider_raw(src, class) + return src == 'wired' and class == 'wired-provider' +end + +function T.raw_cap_meta(src, class, id) + if is_wired_provider_raw(src, class) then return { 'raw', 'host', 'wired', 'provider', id, 'meta' } end + return { 'raw', 'host', src, 'cap', class, id, 'meta' } +end + +function T.raw_cap_status(src, class, id) + if is_wired_provider_raw(src, class) then return { 'raw', 'host', 'wired', 'provider', id, 'status' } end + return { 'raw', 'host', src, 'cap', class, id, 'status' } +end + +function T.raw_cap_state(src, class, id, key) + if is_wired_provider_raw(src, class) then return { 'raw', 'host', 'wired', 'provider', id, 'state', key } end + return { 'raw', 'host', src, 'cap', class, id, 'state', key } +end + +function T.raw_cap_event(src, class, id, name) + if is_wired_provider_raw(src, class) then return { 'raw', 'host', 'wired', 'provider', id, 'event', name } end + return { 'raw', 'host', src, 'cap', class, id, 'event', name } +end + +function T.raw_cap_rpc(src, class, id, verb) + if is_wired_provider_raw(src, class) then return { 'raw', 'host', 'wired', 'provider', id, 'rpc', verb } end + return { 'raw', 'host', src, 'cap', class, id, 'rpc', verb } +end + +---------------------------------------------------------------------- +-- Generic helpers +---------------------------------------------------------------------- + +local function class_valid(class) + return type(class) == 'string' and class ~= '' +end + +local function id_valid(id) + return (type(id) == 'string' and id ~= '') or (type(id) == 'number' and id >= 0) +end + +local function validate_config(config) + if type(config) ~= 'table' then + return false, "config must be a table" + end + + if config.schema ~= SCHEMA_STANDARD then + return false, "config schema must be " .. SCHEMA_STANDARD + end + + for key, value in pairs(config) do + if key ~= 'schema' then + if type(key) ~= 'string' then + return false, "config keys must be strings" + end + if type(value) ~= 'table' then + return false, "config values must be tables" + end + end + end + + return true, "" +end + +local shallow_copy = tablex.shallow_copy + +local function path_token(x) + local s = tostring(x or '') + s = s:gsub('[^%w%-_%.]+', '_') + s = s:gsub('_+', '_') + s = s:gsub('^_+', '') + s = s:gsub('_+$', '') + return (s ~= '') and s or 'unknown' +end + +local function device_source_id(device) + local meta = device.meta or {} + local candidates = { + meta.source_id, + meta.source, + meta.devpath, + meta.path, + meta.name, + meta.serial, + meta.uid, + } + + for i = 1, #candidates do + local v = candidates[i] + if type(v) == 'string' and v ~= '' then return path_token(v) end + if type(v) == 'number' then return path_token(v) end + end + + return path_token(('%s_%s'):format(tostring(device.class), tostring(device.id))) +end + +---------------------------------------------------------------------- +-- Manager lifecycle compatibility seam +-- +-- Legacy managers: +-- start(logger, dev_ch, cap_ch) -> "" | err +-- apply_config(config) -> true|nil, err +-- stop() -> nil +-- +-- Strict managers: +-- api_mode = 'op_only' +-- start_op(...) +-- apply_config_op(...) +-- shutdown_op() +-- terminate(reason) -> immediate, non-yielding cleanup +-- fault_op() -> Op +-- +-- New strict managers must implement the strict form exactly. This file keeps +-- the old form alive only for legacy managers that still depend on the legacy +-- HAL boundary. +---------------------------------------------------------------------- + +local function manager_is_op_only(manager) + return type(manager) == 'table' + and manager.api_mode == 'op_only' +end + +local STRICT_MANAGER_METHODS = { + 'start_op', + 'apply_config_op', + 'shutdown_op', + 'terminate', + 'fault_op', +} + +local function validate_strict_manager(manager_name, manager) + if not manager_is_op_only(manager) then + return true, nil + end + + for i = 1, #STRICT_MANAGER_METHODS do + local method = STRICT_MANAGER_METHODS[i] + if type(manager[method]) ~= 'function' then + return false, ('strict manager %q missing %s'):format( + tostring(manager_name), + method + ) + end + end + + return true, nil +end + +local function manager_call_op(manager_name, manager, method, ...) + local args = pack(...) + local op_name = method .. "_op" + + if type(manager[op_name]) == 'function' then + local ev = manager[op_name](unpack(args, 1, args.n)) + if type(ev) ~= 'table' or getmetatable(ev) ~= op.Op then + error( + ('manager %q method %q must return an Op, got %s (%s)'):format( + tostring(manager_name), + op_name, + type(ev), + tostring(ev) + ), + 2 + ) + end + return ev + end + + if manager_is_op_only(manager) then + error( + ('manager %q must provide %q as an _op method; legacy fallback is not allowed'):format( + tostring(manager_name), + op_name + ), + 2 + ) + end + + if type(manager[method]) ~= 'function' then + return op.always(false, ('manager %q does not implement %q'):format( + tostring(manager_name), + tostring(method) + )) + end + + return op.guard(function () + local a, b = manager[method](unpack(args, 1, args.n)) + + if method == 'start' and type(a) == 'string' and b == nil then + return op.always(a == "", (a == "") and nil or a) + end + + if method == 'stop' and a == nil and b == nil then + return op.always(true, nil) + end + + return op.always(a, b) + end) +end + +local function manager_call_with_timeout_op(manager_name, manager, method, timeout_s, ...) + local has_op_method = type(manager[method .. "_op"]) == 'function' + local ev = manager_call_op(manager_name, manager, method, ...) + + -- Only real _op manager methods can be timed out cooperatively. + -- Legacy methods run synchronously inside manager_call_op's guard and are + -- treated as immediate compatibility code. + if not has_op_method or type(timeout_s) ~= 'number' or timeout_s < 0 then + return ev + end + + return op.named_choice({ + result = ev, + timeout = sleep.sleep_op(timeout_s), + }):wrap(function(which, a, b) + if which == 'timeout' then return false, 'timeout' end + return a, b + end) +end + + +local function manager_shutdown_op(manager_name, manager, timeout_s) + if manager_is_op_only(manager) then + return manager_call_with_timeout_op( + manager_name, + manager, + 'shutdown', + timeout_s, + timeout_s + ) + end + + return manager_call_with_timeout_op(manager_name, manager, 'stop', timeout_s) +end + +local function manager_terminate(manager_name, manager, reason) + if not manager_is_op_only(manager) then + return true, nil + end + + if type(manager) ~= 'table' or type(manager.terminate) ~= 'function' then + return nil, ('strict manager %q missing terminate'):format(tostring(manager_name)) + end + + local ok, a, b = pcall(function () + return manager.terminate(reason) + end) + + if not ok then + return nil, ('manager %q terminate raised: %s'):format( + tostring(manager_name), + tostring(a) + ) + end + + if a == false or (a == nil and b ~= nil) then + return nil, ('manager %q terminate failed: %s'):format( + tostring(manager_name), + tostring(b or 'termination failed') + ) + end + + return true, nil +end + +local function manager_fault_watch_op(manager_name, manager) + if type(manager.fault_op) == 'function' then + local ev = manager.fault_op() + if type(ev) ~= 'table' or getmetatable(ev) ~= op.Op then + error( + ('manager %q method fault_op must return an Op, got %s (%s)'):format( + tostring(manager_name), + type(ev), + tostring(ev) + ), + 2 + ) + end + return ev + end + + if manager_is_op_only(manager) then + error( + ('strict manager %q missing fault_op'):format(tostring(manager_name)), + 2 + ) + end + + if manager.scope and type(manager.scope.fault_op) == 'function' then + return manager.scope:fault_op() + end + + return op.never() +end + +---------------------------------------------------------------------- +-- Bus request/reply and control-dispatch helpers +---------------------------------------------------------------------- + +local function fallback_reply(reason) + return select(1, types.new.Reply(false, tostring(reason or 'control failed'))) +end + +local function fail_request(req, reason, fallback_to_reply) + local msg = tostring(reason or 'failed') + + if fallback_to_reply and req and req.reply then + local reply = fallback_reply(msg) + if reply and req:reply(reply) then return true end + end + + if req and req.fail then + return not not req:fail(msg) + end + + return false +end + +local function reply_control_failure(req, reason) + local reply = fallback_reply(reason) + or assert(select(1, types.new.Reply(false, 'control failed'))) + + if req and type(req.reply) == 'function' then + return req:reply(reply) + end + + return false +end + +local function remaining_sleep_op(deadline) + return op.guard(function() + local now = fibers.now() + if deadline <= now then return op.always() end + return sleep.sleep_op(deadline - now) + end) +end + +local function dispatch_cap_ctrl(cap_entry, verb, payload, timeout_s, bus_req) + timeout_s = (type(timeout_s) == 'number' and timeout_s >= 0) + and timeout_s + or DEFAULT_CONTROL_TIMEOUT_S + + if not cap_entry or cap_entry.alive == false then + return nil, 'capability unavailable' + end + + local reply_ch = channel.new() + local caller_cancel_op = nil + if type(bus_req) == 'table' and type(bus_req.done_op) == 'function' then + caller_cancel_op = bus_req:done_op():wrap(function (status, _value, err) + if status == 'abandoned' then return 'caller_abandoned' end + return false + end) + end + local control_req, ctrl_req_err = types.new.ControlRequest(verb, payload, reply_ch, caller_cancel_op) + if not control_req then + return nil, tostring(ctrl_req_err or 'invalid control request') + end + + local deadline = fibers.now() + timeout_s + + local send_choices = { + sent = cap_entry.inst.control_ch:put_op(control_req):wrap(function() + return true + end), + + timeout = remaining_sleep_op(deadline):wrap(function() + return false, 'timeout' + end), + } + if caller_cancel_op ~= nil then send_choices.cancel = caller_cancel_op end + + local which, a, b = perform(op.named_choice(send_choices)) + + if which == 'timeout' then + return nil, 'timeout' + end + if which == 'cancel' and a ~= nil and a ~= false then + return nil, tostring(a or 'caller_abandoned') + end + if which ~= 'sent' or a ~= true then + return nil, tostring(b or 'control channel closed') + end + + local reply_choices = { + reply = reply_ch:get_op(), + timeout = remaining_sleep_op(deadline):wrap(function() + return nil, 'timeout' + end), + } + if caller_cancel_op ~= nil then reply_choices.cancel = caller_cancel_op end + + local which2, reply_or_status, err_or_reason = perform(op.named_choice(reply_choices)) + + if which2 == 'timeout' then + return nil, 'timeout' + end + if which2 == 'cancel' and reply_or_status ~= nil and reply_or_status ~= false then + return nil, tostring(reply_or_status or 'caller_abandoned') + end + if not reply_or_status then + return nil, tostring(err_or_reason or 'reply channel closed') + end + + return reply_or_status, nil +end + +---------------------------------------------------------------------- +-- Bus projection payloads and capability routing +-- +-- Public projection: +-- cap///... +-- +-- Raw host provenance projection: +-- raw/host//cap///... +-- raw/host/wired/provider//... for local wired provider observations +-- +-- The public projection is the stable capability surface. The raw projection +-- records where the host discovered it. +---------------------------------------------------------------------- + +local function availability_state(event_type) + if event_type == 'added' then return 'available' end + if event_type == 'removed' then return 'removed' end + return tostring(event_type) +end + +local function availability_flag(event_type) + return event_type == 'added' +end + +local function availability_payload(event_type, extra) + local out = { + state = availability_state(event_type), + available = availability_flag(event_type), + } + if type(extra) == 'table' then + for k, v in pairs(extra) do out[k] = v end + end + return out +end + +local function raw_source_meta_payload(device, source) + local meta = shallow_copy(device.meta) + meta.class = device.class + meta.id = device.id + meta.source = source + return meta +end + +local function raw_source_status_payload(event_type, source, device) + return availability_payload(event_type, { + source = source, + class = device.class, + id = device.id, + }) +end + +local function cap_public_meta_payload(cap, entry) + local out = { offerings = cap.offerings } + for k, v in pairs(entry.meta_fields or {}) do out[k] = v end + return out +end + +local function cap_raw_meta_payload(cap, entry) + local out = cap_public_meta_payload(cap, entry) + out.source_kind = entry.source_kind + out.source = entry.source + return out +end + +local function cap_status_payload(event_type, entry) + return availability_payload(event_type, { + source_kind = entry.source_kind, + source = entry.source, + }) +end + +-- Some host-discovered capabilities are deliberately kept raw until a higher +-- level appliance composer curates them into a public capability. For wired +-- providers, HAL only publishes raw/host/wired/provider//... and raw RPC. +-- Device owns product assembly; Wired owns the public state/wired/... projection. +local function public_cap_projection_enabled(class) + return class ~= 'wired-provider' +end + +local function parse_cap_ctrl_topic(topic) + if topic[1] == 'cap' and topic[4] == 'rpc' then + return 'public', topic[2], topic[3], topic[5], nil + end + + if topic[1] == 'raw' + and topic[2] == 'host' + and topic[3] == 'wired' + and topic[4] == 'provider' + and topic[6] == 'rpc' + then + return 'raw-host', 'wired-provider', topic[5], topic[7], 'wired' + end + + if topic[1] == 'raw' + and topic[2] == 'host' + and topic[4] == 'cap' + and topic[7] == 'rpc' + then + return 'raw-host', topic[5], topic[6], topic[8], topic[3] + end + + return nil, nil, nil, nil, nil +end + +---------------------------------------------------------------------- +-- HAL service +---------------------------------------------------------------------- + +local HalService = {} + +function HalService.start(conn, opts) + opts = opts or {} + + local svc = base.new(conn, { name = opts.name or "hal", env = opts.env }) + HalService.name = svc.name + + local service_scope = fibers.current_scope() + + local heartbeat_s = (type(opts.heartbeat_s) == 'number') and opts.heartbeat_s or 30.0 + local control_timeout_s = (type(opts.control_timeout_s) == 'number') + and opts.control_timeout_s + or DEFAULT_CONTROL_TIMEOUT_S + local manager_start_timeout_s = (type(opts.manager_start_timeout_s) == 'number') + and opts.manager_start_timeout_s + or DEFAULT_MANAGER_START_TIMEOUT_S + local configured_manager_apply_timeout_s = (type(opts.manager_apply_timeout_s) == 'number') + and opts.manager_apply_timeout_s + or DEFAULT_MANAGER_APPLY_TIMEOUT_S + local function manager_apply_timeout_for(name) + if type(opts.manager_apply_timeouts) == 'table' and type(opts.manager_apply_timeouts[name]) == 'number' then + return opts.manager_apply_timeouts[name] + end + if type(DEFAULT_MANAGER_APPLY_TIMEOUTS[name]) == 'number' then + return DEFAULT_MANAGER_APPLY_TIMEOUTS[name] + end + return configured_manager_apply_timeout_s + end + local manager_shutdown_timeout_s = (type(opts.manager_shutdown_timeout_s) == 'number') + and opts.manager_shutdown_timeout_s + or DEFAULT_MANAGER_SHUTDOWN_TIMEOUT_S + + local cap_emit_ch = channel.new(DEFAULT_Q_LEN) + local dev_ev_ch = channel.new(DEFAULT_Q_LEN) + local config_result_ch = channel.new(DEFAULT_Q_LEN) + + local config_generation = 0 + local active_config_generation = nil + local pending_config = nil + + local managers = {} + local registry = { + devices = {}, + caps = {}, + } + + local function obs_emitter(level, payload) + svc:obs_log(level, payload) + end + + local function log(level, what, fields) + local payload = fields or {} + payload.what = what + svc:obs_log(level, payload) + end + + local function reject_rpc(req, what, reason, fields) + fields = fields or {} + fields.err = tostring(reason) + log('warn', what, fields) + fail_request(req, reason, true) + end + + local function spawn_service_worker(label, req, fn) + local ok, err = fibers.spawn(fn) + if ok then return true end + + local reason = tostring(err or 'service not accepting work') + log('warn', 'worker_spawn_rejected', { label = tostring(label), err = reason }) + if req then fail_request(req, reason, true) end + return false + end + + local function shutdown_manager_async(name, manager, reason) + return spawn_service_worker('manager_shutdown', nil, function() + local ok, shutdown_err = perform(manager_shutdown_op(name, manager, manager_shutdown_timeout_s)) + if ok ~= true then + log('warn', 'manager_shutdown_failed', { + manager = name, + reason = reason, + err = tostring(shutdown_err), + }) + + local terminated, terminate_err = manager_terminate(name, manager, shutdown_err or reason) + if terminated ~= true then + log('warn', 'manager_terminate_failed', { + manager = name, + reason = reason, + err = tostring(terminate_err), + }) + end + end + end) + end + + local function unbind_cap_endpoints(entry) + if not entry then return end + + for _, ep in pairs(entry.rpc or {}) do ep:unbind() end + for _, ep in pairs(entry.raw_rpc or {}) do ep:unbind() end + end + + function registry:get_device(class, id) + local class_devices = self.devices[class] + return class_devices and class_devices[id] or nil + end + + function registry:set_device(device) + self.devices[device.class] = self.devices[device.class] or {} + if self.devices[device.class][device.id] then + return nil, 'device already exists' + end + self.devices[device.class][device.id] = device + return true + end + + function registry:remove_device(class, id) + local class_devices = self.devices[class] + if not class_devices or not class_devices[id] then + return nil, 'device does not exist' + end + class_devices[id] = nil + return true + end + + function registry:get_cap(class, id) + local class_caps = self.caps[class] + return class_caps and class_caps[id] or nil + end + + function registry:set_cap(class, id, cap_inst, source_kind, source) + self.caps[class] = self.caps[class] or {} + if self.caps[class][id] then + return nil, 'capability already exists' + end + + local entry = { + inst = cap_inst, + rpc = {}, + raw_rpc = {}, + source_kind = source_kind, + source = source, + state_keys = {}, + meta_fields = {}, + alive = true, + } + + self.caps[class][id] = entry + + local expose_public = public_cap_projection_enabled(class) + for offering in pairs(cap_inst.offerings) do + if expose_public then + entry.rpc[offering] = conn:bind(T.cap_rpc(class, id, offering)) + end + if source_kind == 'host' and source then + entry.raw_rpc[offering] = conn:bind(T.raw_cap_rpc(source, class, id, offering)) + end + end + + return entry, nil + end + + function registry:remove_cap(class, id) + local class_caps = self.caps[class] + local entry = class_caps and class_caps[id] + if not entry then return nil, 'capability does not exist' end + + entry.alive = false + unbind_cap_endpoints(entry) + + for key in pairs(entry.state_keys) do + if public_cap_projection_enabled(class) then + conn:unretain(T.cap_state(class, id, key)) + end + if entry.source_kind == 'host' and entry.source then + conn:unretain(T.raw_cap_state(entry.source, class, id, key)) + end + end + + class_caps[id] = nil + return true + end + + function registry:rpc_ops() + local out = {} + for _, class_caps in pairs(self.caps) do + for _, entry in pairs(class_caps) do + for _, ep in pairs(entry.rpc) do out[#out + 1] = ep:recv_op() end + for _, ep in pairs(entry.raw_rpc) do out[#out + 1] = ep:recv_op() end + end + end + return out + end + + function registry:terminate_caps(reason) + -- reason is accepted for finaliser-shaped call sites; bus endpoints + -- expose immediate unbind without a reason parameter. + for _, class_caps in pairs(self.caps) do + for _, entry in pairs(class_caps) do + entry.alive = false + unbind_cap_endpoints(entry) + end + end + end + + local function retain_raw_source(event_type, device) + local source = device_source_id(device) + conn:retain(T.raw_source_status(source), raw_source_status_payload(event_type, source, device)) + + if event_type == 'added' then + conn:retain(T.raw_source_meta(source), raw_source_meta_payload(device, source)) + else + conn:unretain(T.raw_source_meta(source)) + end + end + + local function retain_cap_projection(event_type, class, id, entry) + if public_cap_projection_enabled(class) then + conn:retain(T.cap_legacy_state(class, id), event_type) + conn:retain(T.cap_status(class, id), cap_status_payload(event_type, entry)) + + if event_type == 'added' then + conn:retain(T.cap_meta(class, id), cap_public_meta_payload(entry.inst, entry)) + else + conn:unretain(T.cap_meta(class, id)) + end + else + -- Defensive cleanup in case an earlier build exposed this class publicly. + conn:unretain(T.cap_legacy_state(class, id)) + conn:unretain(T.cap_status(class, id)) + conn:unretain(T.cap_meta(class, id)) + end + + if entry.source_kind ~= 'host' or not entry.source then return end + + local source = entry.source + conn:retain(T.raw_cap_status(source, class, id), cap_status_payload(event_type, entry)) + + if event_type == 'added' then + conn:retain(T.raw_cap_meta(source, class, id), cap_raw_meta_payload(entry.inst, entry)) + else + conn:unretain(T.raw_cap_meta(source, class, id)) + end + end + + local function publish_cap_event(entry, class, id, name, payload) + if public_cap_projection_enabled(class) then + conn:publish(T.cap_event(class, id, name), payload) + end + + if entry.source_kind == 'host' and entry.source then + conn:publish(T.raw_cap_event(entry.source, class, id, name), payload) + end + end + + local function retain_cap_state(entry, class, id, key, payload) + entry.state_keys[key] = true + if public_cap_projection_enabled(class) then + conn:retain(T.cap_state(class, id, key), payload) + end + + if entry.source_kind == 'host' and entry.source then + conn:retain(T.raw_cap_state(entry.source, class, id, key), payload) + end + end + + local function retain_cap_meta(entry, class, id) + if public_cap_projection_enabled(class) then + conn:retain(T.cap_meta(class, id), cap_public_meta_payload(entry.inst, entry)) + end + + if entry.source_kind == 'host' and entry.source then + conn:retain(T.raw_cap_meta(entry.source, class, id), cap_raw_meta_payload(entry.inst, entry)) + end + end + + local function register_device(event_type, device) + local ok, set_err = registry:set_device(device) + if not ok then + log('warn', 'register_device_skipped', { + err = set_err, + class = device.class, + id = device.id, + }) + return + end + + local source_kind = 'host' + local source = device_source_id(device) + + retain_raw_source(event_type, device) + + for _, cap in ipairs(device.capabilities) do + local entry, cap_set_err = registry:set_cap(cap.class, cap.id, cap, source_kind, source) + if not entry then + log('warn', 'register_capability_skipped', { + err = cap_set_err, + class = cap.class, + id = cap.id, + }) + else + svc:obs_event('capability_registered', { + class = cap.class, + id = cap.id, + source = source, + }) + retain_cap_projection(event_type, cap.class, cap.id, entry) + end + end + + conn:retain(T.dev_meta(device.class, device.id), device.meta) + conn:retain(T.dev_state(device.class, device.id), event_type) + + svc:obs_event('device_registered', { + class = device.class, + id = device.id, + event_type = event_type, + source = source, + }) + end + + local function unregister_device(event_type, device) + local source = device_source_id(device) + + for _, cap in ipairs(device.capabilities) do + local entry = registry:get_cap(cap.class, cap.id) + if not entry then + log('warn', 'remove_capability_skipped', { + err = 'capability does not exist', + class = cap.class, + id = cap.id, + }) + else + svc:obs_event('capability_unregistered', { + class = cap.class, + id = cap.id, + source = source, + }) + + retain_cap_projection(event_type, cap.class, cap.id, entry) + + local ok, cap_remove_err = registry:remove_cap(cap.class, cap.id) + if not ok then + log('warn', 'remove_capability_skipped', { + err = cap_remove_err, + class = cap.class, + id = cap.id, + }) + end + end + end + + local ok, remove_err = registry:remove_device(device.class, device.id) + if not ok then + log('warn', 'remove_device_skipped', { + err = remove_err, + class = device.class, + id = device.id, + }) + return + end + + conn:unretain(T.dev_meta(device.class, device.id)) + conn:retain(T.dev_state(device.class, device.id), event_type) + retain_raw_source(event_type, device) + + svc:obs_event('device_unregistered', { + class = device.class, + id = device.id, + event_type = event_type, + source = source, + }) + end + + local function serve_cap_ctrl(req, class, id, verb, cap_entry) + if not cap_entry or cap_entry.alive == false then + fail_request(req, 'capability unavailable', true) + return + end + + local reply, reply_err = dispatch_cap_ctrl(cap_entry, verb, req.payload, control_timeout_s, req) + if not reply then + if tostring(reply_err) == 'caller_abandoned' then return end + log('warn', 'control_dispatch_failed', { + err = tostring(reply_err), + class = class, + id = id, + verb = verb, + }) + reply_control_failure(req, reply_err) + return + end + + if not req:reply(reply) then + log('warn', 'control_reply_deliver_failed', { + class = class, + id = id, + verb = verb, + }) + end + end + + local function on_cap_ctrl(req) + if not req or type(req) ~= 'table' or type(req.topic) ~= 'table' then + return reject_rpc(req, 'invalid_rpc_request', 'invalid rpc request') + end + + local route, class, id, verb, source = parse_cap_ctrl_topic(req.topic) + if not route then + return reject_rpc(req, 'invalid_cap_rpc_route', 'invalid capability rpc route', { topic = req.topic }) + end + if not class_valid(class) then + return reject_rpc(req, 'invalid_cap_class', 'invalid capability class', { class = tostring(class) }) + end + if not id_valid(id) then + return reject_rpc(req, 'invalid_cap_id', 'invalid capability id', { class = tostring(class), id = tostring(id) }) + end + if type(verb) ~= 'string' or verb == '' then + return reject_rpc(req, 'invalid_control_verb', 'invalid control verb', { + class = tostring(class), + id = tostring(id), + verb = tostring(verb), + }) + end + + local cap_entry = registry:get_cap(class, id) + if not cap_entry or cap_entry.alive == false then + return reject_rpc(req, 'control_capability_unavailable', 'capability unavailable', { + class = class, + id = id, + verb = verb, + route = route, + }) + end + + if route == 'raw-host' and (cap_entry.source_kind ~= 'host' or cap_entry.source ~= source) then + return reject_rpc(req, 'raw_capability_route_unavailable', 'raw capability route unavailable', { + class = class, + id = id, + verb = verb, + source = tostring(source), + }) + end + + if not cap_entry.inst.offerings[verb] then + return reject_rpc(req, 'control_verb_unavailable', 'control verb unavailable', { + class = class, + id = id, + verb = verb, + }) + end + + spawn_service_worker('cap_ctrl', req, function() + serve_cap_ctrl(req, class, id, verb, cap_entry) + end) + end + + local function on_cap_emit(emit) + if getmetatable(emit) ~= types.Emit then + log('warn', 'invalid_emit_message') + return + end + + local entry = registry:get_cap(emit.class, emit.id) + if not entry then + log('debug', 'cap_emit_missing_capability', { + class = tostring(emit.class), + id = tostring(emit.id), + mode = tostring(emit.mode), + key = tostring(emit.key), + }) + return + end + + if emit.mode == 'event' then + publish_cap_event(entry, emit.class, emit.id, emit.key, emit.data) + return + end + + if emit.mode == 'state' then + retain_cap_state(entry, emit.class, emit.id, emit.key, emit.data) + return + end + + if emit.mode == 'meta' then + entry.meta_fields[emit.key] = emit.data + retain_cap_meta(entry, emit.class, emit.id) + return + end + + log('warn', 'cap_emit_unhandled_mode', { + class = tostring(emit.class), + id = tostring(emit.id), + mode = tostring(emit.mode), + key = tostring(emit.key), + }) + end + + local function on_device_event(device_event) + if getmetatable(device_event) ~= types.DeviceEvent then + log('warn', 'invalid_device_event_message') + return + end + + if device_event.event_type == 'added' then + local dev_inst, dev_err = types.new.Device( + device_event.class, + device_event.id, + device_event.meta, + device_event.capabilities + ) + if not dev_inst then + log('warn', 'device_instance_invalid', { + err = tostring(dev_err), + class = device_event.class, + id = device_event.id, + }) + return + end + register_device(device_event.event_type, dev_inst) + if device_event.ready_cond then + device_event.ready_cond:signal() + end + return + end + + if device_event.event_type == 'removed' then + local dev_inst = registry:get_device(device_event.class, device_event.id) + if not dev_inst then + log('warn', 'device_missing', { class = device_event.class, id = device_event.id }) + return + end + unregister_device(device_event.event_type, dev_inst) + return + end + + log('warn', 'device_event_unhandled', { + class = device_event.class, + id = device_event.id, + event_type = device_event.event_type, + }) + end + + local function drain_lifecycle_internal_event(label, source, payload) + if source == 'cap_emit' then + on_cap_emit(payload) + return true + end + + if source == 'device_event' then + on_device_event(payload) + return true + end + + log('warn', 'hal_lifecycle_pump_unknown_source', { + label = tostring(label), + source = tostring(source), + }) + return false + end + + local function perform_manager_lifecycle_op_with_hal_pump(label, result_op) + local result_ch = channel.new(1) + local spawned, spawn_err = fibers.spawn(function() + local results = pack(perform(result_op)) + perform(result_ch:put_op(results)) + end) + if spawned ~= true then return nil, tostring(spawn_err or 'lifecycle worker spawn failed') end + + while true do + local source, a = perform(op.named_choice({ + result = result_ch:get_op(), + cap_emit = cap_emit_ch:get_op(), + device_event = dev_ev_ch:get_op(), + })) + + if source == 'result' then + return unpack(a, 1, a.n) + end + + drain_lifecycle_internal_event(label, source, a) + end + end + + + local dependency_resolver, dependency_resolver_err = dependencies.resolver(conn) + if not dependency_resolver then + error('HAL dependency resolver failed: ' .. tostring(dependency_resolver_err), 0) + end + + local function start_manager(name, manager) + local valid, valid_err = validate_strict_manager(name, manager) + if not valid then + error(valid_err, 0) + end + + local manager_logger = Logger.new(obs_emitter, { + service = svc.name, + component = 'manager', + manager = name, + }) + + return perform_manager_lifecycle_op_with_hal_pump( + 'manager_start:' .. tostring(name), + manager_call_with_timeout_op( + name, + manager, + 'start', + manager_start_timeout_s, + manager_logger, + dev_ev_ch, + cap_emit_ch, + dependencies.manager_options(name, dependency_resolver, { service_name = svc.name }) + ) + ) + end + + local function apply_manager_config_with_wait(name, manager, manager_config, opts) + opts = opts or {} + local wait = opts.wait + local function wait_for(label, wait_op) + if type(wait) == 'function' then return wait(label, wait_op) end + return perform(wait_op) + end + + local timeout_s = manager_apply_timeout_for(name) + local has_op_method = type(manager.apply_config_op) == 'function' + if not has_op_method or type(timeout_s) ~= 'number' or timeout_s < 0 then + local ok, err = wait_for( + 'manager_apply:' .. tostring(name), + manager_call_op(name, manager, 'apply_config', manager_config) + ) + if ok == true then return 'applied', nil end + return 'failed', err + end + + local result_ch = channel.new(1) + local spawned, spawn_err = fibers.spawn(function() + local ok, err = perform(manager_call_op(name, manager, 'apply_config', manager_config)) + perform(result_ch:put_op({ ok = ok == true, err = err })) + end) + if spawned ~= true then return 'failed', tostring(spawn_err or 'manager apply worker spawn failed') end + + local which, result = wait_for( + 'manager_apply_wait:' .. tostring(name), + op.named_choice({ + result = result_ch:get_op(), + timeout = sleep.sleep_op(timeout_s), + }) + ) + if which == 'result' then + if result and result.ok == true then return 'applied', nil end + return 'failed', result and result.err or 'manager apply failed' + end + + log('debug', 'manager_apply_pending_timeout', { summary = string.format('HAL manager %s still pending after %ss', tostring(name), tostring(timeout_s)), manager = name, admitted = true, timeout_s = timeout_s }) + fibers.spawn(function() + local late = perform(result_ch:get_op()) + if late and late.ok == true then + log('info', 'hal_manager_recovered', { summary = string.format('HAL manager %s recovered after timeout', tostring(name)), manager = name }) + else + log('error', 'hal_manager_failed', { summary = string.format('HAL manager %s failed after timeout: %s', tostring(name), tostring(late and late.err or 'manager apply failed')), manager = name, err = tostring(late and late.err or 'manager apply failed') }) + end + end) + return 'pending_after_timeout', 'timeout' + end + + local function apply_manager_config(name, manager, manager_config) + return apply_manager_config_with_wait(name, manager, manager_config, { + wait = perform_manager_lifecycle_op_with_hal_pump, + }) + end + + local function copy_list(list) + local out = {} + for i, v in ipairs(list or {}) do out[i] = v end + return out + end + + local function copy_map(map) + local out = {} + for k, v in pairs(map or {}) do out[k] = v end + return out + end + + local function build_config_generation_plan(config) + local plan = { entries = {}, removals = {}, failed_managers = {}, start_errors = {} } + + for name, manager_config in pairs(config) do + if name ~= 'schema' then + local entry = { + name = name, + manager_config = manager_config, + manager = managers[name], + } + if not entry.manager then + local ok, manager = pcall(require, "services.hal.managers." .. name) + if not ok then + local err = tostring(manager) + plan.failed_managers[#plan.failed_managers + 1] = name + plan.start_errors[name] = err + else + local valid, valid_err = validate_strict_manager(name, manager) + if not valid then error(valid_err, 0) end + local start_ok, start_err = start_manager(name, manager) + if start_ok ~= true then + plan.failed_managers[#plan.failed_managers + 1] = name + plan.start_errors[name] = tostring(start_err) + else + entry.manager = manager + managers[name] = manager + svc:obs_event('manager_started', { manager = name }) + end + end + else + local valid, valid_err = validate_strict_manager(name, entry.manager) + if not valid then error(valid_err, 0) end + end + if entry.manager then plan.entries[#plan.entries + 1] = entry end + end + end + + for name, manager in pairs(managers) do + if not config[name] then + plan.removals[#plan.removals + 1] = { name = name, manager = manager } + end + end + + return plan + end + + local function run_config_generation(generation, plan) + local result = { + generation = generation, + pending_managers = {}, + failed_managers = copy_list(plan.failed_managers), + start_errors = copy_map(plan.start_errors), + apply_errors = {}, + removals = plan.removals or {}, + } + + for _, entry in ipairs(plan.entries or {}) do + local name = entry.name + local manager = entry.manager + if manager then + local ok, status, apply_err = pcall(apply_manager_config_with_wait, name, manager, entry.manager_config) + if not ok then + result.failed_managers[#result.failed_managers + 1] = name + result.apply_errors[name] = tostring(status) + elseif status == 'pending_after_timeout' then + result.pending_managers[#result.pending_managers + 1] = name + elseif status ~= 'applied' then + result.failed_managers[#result.failed_managers + 1] = name + result.apply_errors[name] = tostring(apply_err) + end + end + end + + return result + end + + local start_config_generation + + local function finish_config_result(result) + if type(result) ~= 'table' then return end + local generation = result.generation + if generation ~= active_config_generation then + log('debug', 'hal_config_generation_stale', { generation = generation, active_generation = active_config_generation }) + return + end + + for name, err in pairs(result.start_errors or {}) do + log('error', 'manager_start_failed', { manager = name, err = tostring(err), generation = generation }) + end + for name, err in pairs(result.apply_errors or {}) do + log('error', 'manager_apply_failed', { manager = name, err = tostring(err), generation = generation }) + end + + for _, rec in ipairs(result.removals or {}) do + if managers[rec.name] == rec.manager then + managers[rec.name] = nil + svc:obs_event('manager_stopping', { manager = rec.name, reason = 'removed_from_config', generation = generation }) + shutdown_manager_async(rec.name, rec.manager, 'removed_from_config') + end + end + + local pending_managers = result.pending_managers or {} + local failed_managers = result.failed_managers or {} + local config_ok = #failed_managers == 0 + local config_degraded = (#pending_managers > 0 or #failed_managers > 0) or nil + svc:obs_event('config_end', { + generation = generation, + ok = config_ok, + degraded = config_degraded, + pending_managers = (#pending_managers > 0) and pending_managers or nil, + failed_managers = (#failed_managers > 0) and failed_managers or nil, + }) + local hal_what = config_ok and (config_degraded and 'hal_config_degraded' or 'hal_ready') or 'hal_config_failed' + local hal_summary + if not config_ok then + hal_summary = string.format('HAL config failed failed=%d', #failed_managers) + elseif config_degraded then + hal_summary = string.format('HAL config degraded: pending=%s failed=%d', table.concat(pending_managers, ','), #failed_managers) + else + hal_summary = 'HAL ready' + end + log(config_ok and (config_degraded and 'warn' or 'info') or 'error', hal_what, { + summary = hal_summary, + generation = generation, + ok = config_ok, + degraded = config_degraded, + pending_managers = (#pending_managers > 0) and pending_managers or nil, + failed_managers = (#failed_managers > 0) and failed_managers or nil, + }) + + active_config_generation = nil + if pending_config then + local next_config = pending_config + pending_config = nil + start_config_generation(next_config) + end + end + + function start_config_generation(config) + svc:obs_event('config_begin', {}) + local valid, valid_err = validate_config(config) + if not valid then + log('warn', 'config_invalid', { err = valid_err }) + svc:obs_event('config_end', { ok = false, err = valid_err }) + return + end + + config_generation = config_generation + 1 + local generation = config_generation + active_config_generation = generation + svc:obs_event('config_generation_started', { generation = generation }) + local plan = build_config_generation_plan(config) + local spawned, spawn_err = fibers.spawn(function() + local ok, result = pcall(run_config_generation, generation, plan) + if ok ~= true then + result = { + generation = generation, + pending_managers = {}, + failed_managers = { '_config_generation' }, + start_errors = {}, + apply_errors = { _config_generation = tostring(result) }, + removals = {}, + } + end + perform(config_result_ch:put_op(result)) + end) + if spawned ~= true then + active_config_generation = nil + log('error', 'hal_config_worker_spawn_failed', { generation = generation, err = tostring(spawn_err) }) + svc:obs_event('config_end', { generation = generation, ok = false, err = tostring(spawn_err) }) + end + end + + + local function bootstrap() + svc:obs_event('bootstrap_begin', {}) + + local fs_manager = require "services.hal.managers.filesystem" + local start_ok, start_err = start_manager('filesystem', fs_manager) + + if start_ok ~= true then + svc:status('failed', { reason = 'filesystem manager start failed', err = tostring(start_err) }) + log('error', 'bootstrap_failed', { err = tostring(start_err), phase = 'start_filesystem_manager' }) + error("HAL bootstrap failed: Failed to start filesystem manager: " .. tostring(start_err)) + end + + local cfg_status, cfg_err = apply_manager_config('filesystem', fs_manager, { + { + name = "config", + root = os.getenv("DEVICECODE_CONFIG_DIR"), + } + }) + + if cfg_status ~= 'applied' then + svc:status('failed', { reason = 'filesystem manager config failed', err = tostring(cfg_err) }) + log('error', 'bootstrap_failed', { err = tostring(cfg_err), phase = 'apply_filesystem_config', status = tostring(cfg_status) }) + error("HAL bootstrap failed: " .. tostring(cfg_err)) + end + + managers.filesystem = fs_manager + svc:obs_event('bootstrap_end', { ok = true }) + end + + local function manager_fault_ops() + local out = {} + for name, manager in pairs(managers) do + out[#out + 1] = manager_fault_watch_op(name, manager):wrap(function () + return name + end) + end + return out + end + + local function on_manager_fault(name) + local manager = managers[name] + if not manager then return end + + svc:status('degraded', { reason = 'manager_fault', manager = name }) + log('error', 'manager_fault', { manager = name }) + + managers[name] = nil + shutdown_manager_async(name, manager, 'manager_fault') + end + + local function on_config_message(msg) + local cfg_data = msg and msg.payload and msg.payload.data + if type(cfg_data) ~= 'table' then + log('warn', 'config_bad_shape', { payload = msg and msg.payload }) + return + end + if active_config_generation then + pending_config = cfg_data + log('debug', 'hal_config_deferred', { active_generation = active_config_generation }) + return + end + start_config_generation(cfg_data) + end + + local config_sub + + svc:obs_state('boot', { at = svc:wall(), ts = svc:now(), state = 'entered' }) + svc:obs_log('debug', 'service start() entered') + svc:status('starting') + svc:spawn_heartbeat(heartbeat_s, 'tick') + + service_scope:finally(function() + local st, primary = service_scope:status() + if st == 'failed' then + log('error', 'scope_failed', { err = tostring(primary), status = st }) + end + + registry:terminate_caps(primary or 'scope_exit') + + for name, manager in pairs(managers) do + local ok, err = manager_terminate(name, manager, primary or 'scope_exit') + if ok ~= true then + log('warn', 'manager_terminate_failed', { + manager = name, + reason = tostring(primary or 'scope_exit'), + err = tostring(err), + }) + end + managers[name] = nil + end + + if config_sub then + config_sub:unsubscribe() + config_sub = nil + end + + svc:status('stopped', { reason = tostring(primary or 'scope_exit') }) + svc:obs_log('debug', 'service stopped') + end) + + bootstrap() + svc:status('running') + svc:obs_log('debug', 'bootstrap successful') + + config_sub = conn:subscribe({ 'cfg', svc.name }) + svc:obs_log('debug', { what = 'subscribed', topic = 'cfg/' .. svc.name }) + + while true do + local fault_op = active_config_generation and op.never() or op.choice(manager_fault_ops()) + local source, a, b = perform(op.named_choice({ + rpc = op.choice(registry:rpc_ops()), + manager_fault = fault_op, + cap_emit = cap_emit_ch:get_op(), + device_event = dev_ev_ch:get_op(), + config = config_sub:recv_op(), + config_result = config_result_ch:get_op(), + })) + + if source == 'rpc' then + on_cap_ctrl(a) + elseif source == 'cap_emit' then + on_cap_emit(a) + elseif source == 'device_event' then + on_device_event(a) + elseif source == 'config' then + on_config_message(a) + elseif source == 'config_result' then + finish_config_result(a) + elseif source == 'manager_fault' then + on_manager_fault(a) + else + log('error', 'unknown_operation_source', { source = tostring(source) }) + end + end +end + +return HalService diff --git a/src/services/hal/backends/band/contract.lua b/src/services/hal/backends/band/contract.lua new file mode 100644 index 00000000..4decc87b --- /dev/null +++ b/src/services/hal/backends/band/contract.lua @@ -0,0 +1,28 @@ +---Band backend contract definition. +---All band backends must implement the functions listed in BAND_BACKEND_FUNCTIONS. + +local BAND_BACKEND_FUNCTIONS = { + 'clear', + 'apply', +} + +---Validate that a backend table implements all required functions. +---@param backend table +---@return boolean ok +---@return string err Empty string on success. +local function validate(backend) + if type(backend) ~= 'table' then + return false, "backend must be a table" + end + for _, fn_name in ipairs(BAND_BACKEND_FUNCTIONS) do + if type(backend[fn_name]) ~= 'function' then + return false, "backend missing required function: " .. fn_name + end + end + return true, "" +end + +return { + BAND_BACKEND_FUNCTIONS = BAND_BACKEND_FUNCTIONS, + validate = validate, +} diff --git a/src/services/hal/backends/band/provider.lua b/src/services/hal/backends/band/provider.lua new file mode 100644 index 00000000..e452f051 --- /dev/null +++ b/src/services/hal/backends/band/provider.lua @@ -0,0 +1,31 @@ +---Band backend provider. +---Iterates registered backends and returns the first supported one. + +local contract = require "services.hal.backends.band.contract" + +local BACKENDS = { + "services.hal.backends.band.providers.openwrt-dawn", +} + +---Instantiate a new backend from the first supported provider. +---@return table|nil backend instance +---@return string err "" on success +local function new() + for _, path in ipairs(BACKENDS) do + local ok, provider = pcall(require, path) + if ok and provider.is_supported and provider.is_supported() then + local backend = provider.backend.new() + local valid, verr = contract.validate(backend) + if valid then + return backend, "" + else + return nil, "backend " .. path .. " failed contract check: " .. verr + end + end + end + return nil, "no supported band backend found on this device" +end + +return { + new = new, +} diff --git a/src/services/hal/backends/band/providers/openwrt-dawn/impl.lua b/src/services/hal/backends/band/providers/openwrt-dawn/impl.lua new file mode 100644 index 00000000..0417bf8e --- /dev/null +++ b/src/services/hal/backends/band/providers/openwrt-dawn/impl.lua @@ -0,0 +1,232 @@ +---OpenWrt DAWN band steering backend implementation. +---Translates staged driver config to dawn UCI sections via the shared UCI reactor. + +local uci = require "services.hal.backends.common.uci" + +local BAND_SECTION = { + ['2G'] = '802_11g', + ['5G'] = '802_11a', +} + +local KICK_MODE_ID = { + none = 0, + compare = 1, + absolute = 2, + both = 3, +} + +local NETWORKING_METHOD_ID = { + broadcast = 0, + ['tcp+umdns'] = 2, + multicast = 2, + tcp = 3, +} + +local BAND_KICKING_KEYS = { + rssi_center = 'rssi_center', + rssi_reward_threshold = 'rssi_val', + rssi_reward = 'rssi', + rssi_penalty_threshold = 'low_rssi_val', + rssi_penalty = 'low_rssi', + rssi_weight = 'rssi_weight', + channel_util_reward_threshold = 'chan_util_val', + channel_util_reward = 'chan_util', + channel_util_penalty_threshold = 'max_chan_util_val', + channel_util_penalty = 'max_chan_util', +} + +-- Fixed sections required in a clean DAWN config +local REQUIRED_SECTIONS = { + { name = 'global', type = 'metric' }, + { name = '802_11g', type = 'metric' }, + { name = '802_11a', type = 'metric' }, + { name = 'gbltime', type = 'times' }, + { name = 'gblnet', type = 'network' }, + { name = 'localcfg', type = 'local' }, +} + +------------------------------------------------------------------------ +-- BandBackend class +------------------------------------------------------------------------ + +---@class BandBackend +local BandBackend = {} +BandBackend.__index = BandBackend + +---@return BandBackend +function BandBackend.new() + return setmetatable({}, BandBackend) +end + +---Reset DAWN's UCI config to a clean base state. +---Deletes all non-hostapd sections, re-creates required sections with empty defaults. +---@return boolean ok +---@return string err +function BandBackend:clear() + uci.ensure_started() + local session = uci.new_session() + + -- Delete each known managed section only if it already exists + for _, sec in ipairs(REQUIRED_SECTIONS) do + if uci.section_exists('dawn', sec.name) then + session:delete('dawn', sec.name) + end + end + + -- Re-create required sections with their type using named-section creation + for _, sec in ipairs(REQUIRED_SECTIONS) do + session:set('dawn', sec.name, sec.type) + end + + local ok, err = session:commit('dawn') + if not ok then + return false, "failed to clear DAWN config: " .. tostring(err) + end + + return true, "" +end + +---Apply the staged band config table to DAWN UCI and restart the daemon. +---@param staged table Band staged config as accumulated by the driver +function BandBackend:apply(staged) + uci.ensure_started() + local session = uci.new_session() + + -- log_level + if staged.log_level ~= nil then + session:set('dawn', 'localcfg', 'log_level', staged.log_level) + end + + -- kicking + if staged.kicking then + local k = staged.kicking + if k.mode ~= nil then + session:set('dawn', 'global', 'kicking', + tostring(KICK_MODE_ID[k.mode] or 0)) + end + if k.bandwidth_threshold ~= nil then + session:set('dawn', 'global', 'bandwidth_threshold', k.bandwidth_threshold) + end + if k.kicking_threshold ~= nil then + session:set('dawn', 'global', 'kicking_threshold', k.kicking_threshold) + end + if k.evals_before_kick ~= nil then + session:set('dawn', 'global', 'min_number_to_kick', k.evals_before_kick) + end + end + + -- station counting + if staged.station_counting then + local sc = staged.station_counting + if sc.use_station_count ~= nil then + session:set('dawn', 'global', 'use_station_count', + sc.use_station_count and '1' or '0') + end + if sc.max_station_diff ~= nil then + session:set('dawn', 'global', 'max_station_diff', sc.max_station_diff) + end + end + + -- rrm_mode + if staged.rrm_mode ~= nil then + session:set('dawn', 'global', 'rrm_mode', staged.rrm_mode) + end + + -- neighbour reports + if staged.neighbour_reports then + local nr = staged.neighbour_reports + if nr.dyn_report_num ~= nil then + session:set('dawn', 'global', 'set_hostapd_nr', nr.dyn_report_num) + end + if nr.disassoc_report_len ~= nil then + session:set('dawn', 'global', 'disassoc_nr_length', nr.disassoc_report_len) + end + end + + -- legacy options (flat key-value under global) + if staged.legacy then + for key, value in pairs(staged.legacy) do + session:set('dawn', 'global', key, value) + end + end + + -- band priorities and kicking params + for band_key, section_name in pairs(BAND_SECTION) do + local bp = staged.band_priorities and staged.band_priorities[band_key] + if bp then + if bp.initial_score ~= nil then + session:set('dawn', section_name, 'initial_score', bp.initial_score) + end + end + + local bk = staged.band_kicking and staged.band_kicking[band_key] + if bk then + for driver_key, uci_key in pairs(BAND_KICKING_KEYS) do + if bk[driver_key] ~= nil then + session:set('dawn', section_name, uci_key, bk[driver_key]) + end + end + end + + local sb = staged.support_bonus and staged.support_bonus[band_key] + if sb then + for _, support in ipairs({'ht', 'vht'}) do + if sb[support] ~= nil then + session:set('dawn', section_name, support .. '_support', sb[support]) + end + end + end + end + + -- update frequencies + if staged.update_freq then + for key, freq in pairs(staged.update_freq) do + session:set('dawn', 'gbltime', 'update_' .. key, freq) + end + end + + -- client inactive kickoff (con_timeout) + if staged.con_timeout ~= nil then + session:set('dawn', 'gbltime', 'con_timeout', staged.con_timeout) + end + + -- cleanup timeouts + if staged.cleanup then + for key, timeout in pairs(staged.cleanup) do + session:set('dawn', 'gbltime', 'remove_' .. key, timeout) + end + end + + -- networking + if staged.networking then + local net = staged.networking + if net.method ~= nil then + local method_id = NETWORKING_METHOD_ID[net.method] + if method_id ~= nil then + session:set('dawn', 'gblnet', 'method', tostring(method_id)) + end + end + if net.ip ~= nil then + session:set('dawn', 'gblnet', 'tcp_ip', net.ip) + end + if net.port ~= nil then + session:set('dawn', 'gblnet', 'tcp_port', net.port) + end + if net.broadcast_port ~= nil then + session:set('dawn', 'gblnet', 'broadcast_port', net.broadcast_port) + end + if net.enable_encryption ~= nil then + session:set('dawn', 'gblnet', 'use_symm_enc', + net.enable_encryption and '1' or '0') + end + end + + local ok, err = session:commit('dawn', { { 'service', 'dawn', 'restart' } }) + if not ok then + error('apply commit failed: ' .. tostring(err)) + end +end + +return { + new = BandBackend.new, +} diff --git a/src/services/hal/backends/band/providers/openwrt-dawn/init.lua b/src/services/hal/backends/band/providers/openwrt-dawn/init.lua new file mode 100644 index 00000000..853e8c2f --- /dev/null +++ b/src/services/hal/backends/band/providers/openwrt-dawn/init.lua @@ -0,0 +1,15 @@ +local impl = require "services.hal.backends.band.providers.openwrt-dawn.impl" +local file = require "fibers.io.file" + +---Check whether DAWN is installed and its UCI config is accessible. +---@return boolean +local function is_supported() + local f, _ = file.open('/etc/config/dawn', 'r') + if f then f:close() return true end + return false +end + +return { + is_supported = is_supported, + backend = impl, +} diff --git a/src/services/hal/backends/common/uci.lua b/src/services/hal/backends/common/uci.lua new file mode 100644 index 00000000..7dc17c20 --- /dev/null +++ b/src/services/hal/backends/common/uci.lua @@ -0,0 +1,57 @@ +-- services/hal/backends/common/uci.lua +-- +-- Compatibility UCI surface used by the current Wi-Fi backend. +-- +-- The previous implementation kept a module-level commit queue and spawned its +-- reactor into the root scope. This wrapper keeps the colleague-facing surface +-- stable, but delegates to the scoped OpenWrt UCI manager introduced for the +-- new HAL work. +-- +-- New strict code should use services.hal.backends.openwrt.uci_manager +-- directly. This module exists for non-Op compatibility callers. + +local compat = require 'services.hal.backends.openwrt.uci_singleton_compat' + +local M = {} + +local function with_default_local_manager(opts) + opts = opts or {} + if opts.allow_local_manager == nil then + opts.allow_local_manager = true + end + return opts +end + +function M.bind_manager(mgr) + return compat.bind_manager(mgr) +end + +function M.clear_bound_manager() + return compat.clear_bound_manager() +end + +function M.manager_owner() + return compat.manager_owner() +end + +function M.ensure_started(opts) + return compat.ensure_started(with_default_local_manager(opts)) +end + +function M.new_session(opts) + return compat.new_session(with_default_local_manager(opts)) +end + +function M.get_value(config, section, option) + return compat.get_value(config, section, option) +end + +function M.section_exists(config, section) + return compat.section_exists(config, section) +end + +function M.get_sections(config, stype) + return compat.get_sections(config, stype) +end + +return M diff --git a/src/services/hal/backends/modem/at.lua b/src/services/hal/backends/modem/at.lua new file mode 100644 index 00000000..c640d588 --- /dev/null +++ b/src/services/hal/backends/modem/at.lua @@ -0,0 +1,126 @@ +package.path = '/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua;' .. package.path + +local file = require 'fibers.io.file' +local scope = require 'fibers.scope' + +---@alias ATCommand string +---@alias SerialPortPath string +---@alias ATResponseLine string + +---@class ATTerminalPattern +---@field pattern string Lua pattern matched against trimmed lines +---@field is_error boolean When true the matched line is returned as an error string + +---@class ATSendOpts +---@field terminal_patterns ATTerminalPattern[]? + +-- Usage example: +-- +-- local op = require 'fibers.op' +-- local sleep = require 'fibers.sleep' +-- local fibers = require 'fibers' +-- +-- while true do +-- local src, lines, err = fibers.perform(op.named_choice({ +-- response = at.send_op(port, "AT+QGMR"), +-- timeout = sleep.sleep_op(5), +-- })) +-- if src == "response" then +-- -- lines: ATResponseLine[]?, err: string? +-- break +-- end +-- -- timeout arm: loop retries with a fresh op; port re-opened cleanly +-- end + +local function trim(input) + -- Pattern matches non-printable characters and spaces at the start and end of the string + -- %c matches control characters, %s matches all whitespace characters + -- %z matches the character with representation 0x00 (NUL byte) + return (input:gsub("^[%c%s%z]+", ""):gsub("[%c%s%z]+$", "")) +end + +---@class AT +local at = {} + +---Return an Op that sends an AT command and collects response lines until a +---terminal line is hit. +--- +---The Op yields `(lines, err)`: +--- - On success: `lines` is a list of non-empty response lines, `err` is nil. +--- - On AT error: `lines` is the accumulated lines so far, `err` is a string +--- (`'error'`, `'unknown error'`, or a numeric code string from +CME/+CMS). +--- - On cancellation (e.g. the op loses a race against a timeout): `lines` is +--- nil, `err` is `'cancelled'`. The port is closed as part of losing the race. +--- +---`opts.terminal_patterns` is an optional list of additional terminal patterns. +---Each entry is `{ pattern = string, is_error = bool }`. When a trimmed line +---matches, that line is appended to `lines` and the op completes. If `is_error` +---is true, the matched line is also returned as `err`. +--- +---@param port SerialPortPath +---@param command ATCommand +---@param opts ATSendOpts? +---@return Op -- yields (ATResponseLine[]?, string?) +function at.send_op(port, command, opts) + local terminal_patterns = (opts and opts.terminal_patterns) or {} + + return scope.run_op(function(s) + local reader, rd_err = file.open(port, "r") + if not reader then + return nil, "error opening AT read port: " .. rd_err + end + + -- Centralised cleanup: reader is closed however this scope exits + -- (success, AT error, unhandled error, or cancelled by losing a race). + s:finally(function() reader:close() end) + + local writer, wr_err = file.open(port, "w") + if not writer then + return nil, "error opening AT write port: " .. wr_err + end + + writer:write(command .. '\r') + writer:close() + + local res = {} + while true do + local line, read_err = s:perform(reader:read_line_op()) + + if not line then + return nil, read_err or "unknown error" + end + + line = trim(line) + + -- Built-in terminals + if line:find("^OK$") then + return res, nil + elseif line:find("^ERROR$") then + return res, "error" + else + local error_code = line:match("^%+CME ERROR: (%d+)$") + or line:match("^%+CMS ERROR: (%d+)$") + if error_code then + return res, error_code + end + end + + -- User-supplied terminal patterns + for _, tp in ipairs(terminal_patterns) do + if line:find(tp.pattern) then + table.insert(res, line) + return res, tp.is_error and line or nil + end + end + + if #line > 0 then table.insert(res, line) end + end + end):wrap(function(st, _report, a, b) + if st == 'ok' then return a, b end + if st == 'cancelled' then return nil, 'cancelled' end + -- 'failed': a is the primary error string + return nil, a or "AT command failed" + end) +end + +return at diff --git a/src/services/hal/backends/modem/contract.lua b/src/services/hal/backends/modem/contract.lua new file mode 100644 index 00000000..33c123a4 --- /dev/null +++ b/src/services/hal/backends/modem/contract.lua @@ -0,0 +1,96 @@ +local function list_to_map(list) + local map = {} + for _, item in ipairs(list) do + map[item] = true + end + return map +end + +local BACKEND_FUNCTIONS = list_to_map { + -- Grouped reads + "read_identity", + "read_ports", + "read_sim_info", + "read_network_info", + "read_signal", + "read_traffic", + + -- Private reads + "_read_firmware", + + -- State monitoring + "start_state_monitor", + "stop_state_monitor_op", + "stop_state_monitor", + "monitor_state_op", + + -- SIM monitoring + "start_sim_presence_monitor", + "stop_sim_presence_monitor_op", + "stop_sim_presence_monitor", + "wait_for_sim_present_op", + "wait_for_sim_present", + "is_sim_present", + "trigger_sim_presence_check", + + -- Control operations + "enable", + "disable", + "reset", + "connect", + "disconnect", + "inhibit", + "uninhibit", + "uninhibit_op", + "shutdown_op", + "shutdown", + "terminate", + "set_signal_update_interval" +} + +local MONITOR_FUNCTIONS = list_to_map { + "next_event_op", + "shutdown_op", + "terminate", +} + +--- Check that a modem monitor provides all required functions and no extras. +---@param monitor ModemMonitor +---@return string error +local function validate_monitor(monitor) + for func in pairs(MONITOR_FUNCTIONS) do + if type(monitor[func]) ~= "function" then + return "Missing required function: " .. func + end + end + for key, value in pairs(monitor) do + if type(value) == "function" and not MONITOR_FUNCTIONS[key] then + return "Monitor provides unsupported function: " .. key + end + end + return "" +end + +--- Check that a modem backend provides all required functions and no more +---@param backend ModemBackend +---@return string error +local function validate(backend) + for func, _ in pairs(BACKEND_FUNCTIONS) do + if type(backend[func]) ~= "function" then + return "Missing required function: " .. func + end + end + + for key, value in pairs(backend) do + if type(value) == "function" and not BACKEND_FUNCTIONS[key] then + return "Backend provides unsupported function: " .. key + end + end + + return "" +end + +return { + validate = validate, + validate_monitor = validate_monitor, +} diff --git a/src/services/hal/backends/modem/models/fibocom.lua b/src/services/hal/backends/modem/models/fibocom.lua new file mode 100644 index 00000000..1510e57f --- /dev/null +++ b/src/services/hal/backends/modem/models/fibocom.lua @@ -0,0 +1,6 @@ +local function add_model_funcs(_, _) +end + +return { + add_model_funcs = add_model_funcs +} diff --git a/src/services/hal/backends/modem/models/quectel.lua b/src/services/hal/backends/modem/models/quectel.lua new file mode 100644 index 00000000..578d7cd9 --- /dev/null +++ b/src/services/hal/backends/modem/models/quectel.lua @@ -0,0 +1,237 @@ +local exec = require "fibers.io.exec" +local fibers = require "fibers" +local sleep = require "fibers.sleep" +local scope = require "fibers.scope" +local op = require "fibers.op" +local at = require "services.hal.backends.modem.at" + +local SIM_POLL_INTERVAL = 5 + +local function firmware_version_string(fwversion) + if not fwversion then + return nil + end + local version_string = string.match(fwversion, "^%w+_(%w+%.%w+)") + return version_string or fwversion +end + +local function firmware_version_code(version_str) + local major, minor = string.match(version_str or "", "^(%w+)%.(%w+)") + local major_num = tonumber(major, 16) + local minor_num = tonumber(minor) + if major_num and minor_num then + return major_num * 1000 + minor_num + end + return nil, "Invalid firmware version format" +end + +---@param out string +---@return boolean present +---@return string error +local function parse_card_status(out) + if not out or out == "" then + return false, "Empty output" + end + local card_state = out:match("Card state:%s*'([^']+)'") + if not card_state then + return false, "Could not parse card state from --uim-get-card-status output" + end + return card_state == "present", "" +end + +--- Helper for checking if a modem runs "legacy" firmware +---@param model string +---@param firmware_fn function +---@return boolean +local function is_legacy_modem(model, firmware_fn) + --- all em06 devices are legacy + if model == 'em06' then + return true + end + --- only em06 and eg25 devices have capability to be legacy + if model ~= 'eg25' then + return false + end + + local firmware_version = firmware_fn() + if not firmware_version or firmware_version == "" then + return false + end + + local version_code = firmware_version_code(firmware_version_string(firmware_version)) + return version_code ~= nil and version_code <= firmware_version_code("01.002") +end + +local funcs = { + { + name = '_read_firmware', + conditionals = { + function(_, identity_info) + return identity_info.model == 'rm520n' + end + }, + func = function(identity) + local AT_TIMEOUT = 10 + local at_send_op = at.send_op(identity.at_port, "AT+QGMR", { + terminal_patterns = { + { pattern = "^%w+_%w+%.%w+%.%w+%.%w+$", is_error = false } + } + }) + local source, resp, err = fibers.perform(op.named_choice({ + at = at_send_op, + timeout = sleep.sleep_op(AT_TIMEOUT) + })) + + if err then + return nil, "Failed to get firmware version: " .. err + end + + if source == "timeout" then + return nil, "Timed out while getting firmware version" + elseif source == "at" and resp[#resp] then + local firmware_version = string.match(resp[#resp], "([%w]+_[%w]+%.[%w]+%.[%w]+%.[%w]+)") + if firmware_version then + return firmware_version, "" + else + return nil, "Firmware version not found in AT response" + end + end + end + }, + { + name = 'connect', + conditionals = { + function(backend, identity_info) + local model = identity_info and identity_info.model or nil + return model == 'rm520n' and backend.base == "linux_mm" + end + }, + func = function(backend, connection_string) + local st, _, result_or_err = fibers.run_scope(function() + local cmd_clear = exec.command { + "mmcli", "-m", backend.identity.address, "--3gpp-set-initial-eps-bearer-settings=apn=", + stdin = "null", + stdout = "pipe", + stderr = "stdout" + } + local _, status, code, _, err = fibers.perform(cmd_clear:combined_output_op()) + if status ~= "exited" or code ~= 0 then + error(err or "Failed to clear initial bearer settings") + end + local connect_cmd = exec.command { + "mmcli", "-m", backend.identity.address, "--simple-connect=" .. connection_string, + stdin = "null", + stdout = "pipe", + stderr = "stdout" + } + local out, conn_status, conn_code, _, conn_err = fibers.perform(connect_cmd:combined_output_op()) + if conn_status ~= "exited" or conn_code ~= 0 then + error(conn_err or "Failed to connect") + end + return out + end) + if st == "ok" then + return result_or_err, "" + end + return false, result_or_err or "Command failed" + end + }, + { + name = 'wait_for_sim_present_op', + -- NOTE: _read_firmware must be listed before this hook in funcs so that any + -- model-specific override is already installed when this conditional runs. + conditionals = { + function(backend, identity_info) + return is_legacy_modem( + identity_info.model, + function() + return backend._read_firmware(backend.identity) + end + ) + end + }, + func = function(backend) + ---@cast backend ModemBackend + return op.guard(function() + return scope.run_op(function(s) + s:perform(sleep.sleep_op(SIM_POLL_INTERVAL)) + local present, err = backend:is_sim_present() + if err ~= "" then + return false, "Failed to poll SIM presence: " .. err + end + + return present, "" + end) + :wrap(function(st, _, ...) + if st == 'ok' then + return ... + elseif st == 'cancelled' then + return false, "cancelled" + else + return false, (... or "SIM poll failed") + end + end) + end) + end + }, + { + name = 'is_sim_present', + conditionals = { + function(backend, identity_info) + ---@cast backend ModemBackend + ---@cast identity_info ModemIdentityInfo + local is_legacy = is_legacy_modem( + identity_info.model, + function() + local firmware = backend._read_firmware(backend.identity) + return firmware + end + ) and identity_info.mode == "qmi" + return is_legacy + end + }, + func = function(backend) + ---@cast backend ModemBackend + local st, _, present_or_err = fibers.run_scope(function() + local cmd = exec.command { + "qmicli", "-p", "-d", backend.identity.mode_port, "--uim-get-card-status", + stdin = "null", + stdout = "pipe", + stderr = "stdout" + } + local out, status, code, _, err = fibers.perform(cmd:combined_output_op()) + if status ~= "exited" or code ~= 0 then + error("Failed to execute qmicli --uim-get-card-status: " .. tostring(err)) + end + + local present, parse_err = parse_card_status(out) + if parse_err ~= "" then + error("Failed to parse qmicli output: " .. tostring(parse_err)) + end + return present + end) + + if st == "ok" then + return present_or_err, "" + end + return false, present_or_err or "Unknown error" + end + }, +} + +---@param backend ModemBackend +---@param identity_info ModemIdentityInfo +local function add_model_funcs(backend, identity_info) + for _, func_def in ipairs(funcs) do + for _, cond in ipairs(func_def.conditionals) do + if cond(backend, identity_info) then + backend[func_def.name] = func_def.func + break + end + end + end +end + +return { + add_model_funcs = add_model_funcs +} diff --git a/src/services/hal/backends/modem/modes/mbim.lua b/src/services/hal/backends/modem/modes/mbim.lua new file mode 100644 index 00000000..b783107f --- /dev/null +++ b/src/services/hal/backends/modem/modes/mbim.lua @@ -0,0 +1,6 @@ +local function add_mode_funcs(_) +end + +return { + add_mode_funcs = add_mode_funcs +} diff --git a/src/services/hal/backends/modem/modes/qmi.lua b/src/services/hal/backends/modem/modes/qmi.lua new file mode 100644 index 00000000..97db6d4f --- /dev/null +++ b/src/services/hal/backends/modem/modes/qmi.lua @@ -0,0 +1,382 @@ +local fibers = require "fibers" +local exec = require "fibers.io.exec" +local sleep = require "fibers.sleep" +local scope = require "fibers.scope" +local op = require "fibers.op" + +local modem_types = require "services.hal.types.modem" +local process_flags = require "services.hal.backends.modem.process_flags" + +-- ---@param status string? +-- ---@return string card_status +-- ---@return string error +-- local function parse_slot_status(status) +-- if not status or status == "" then +-- return "", "Command closed" +-- end + +-- local card_state = status:match("Card state:%s*'([^']+)'") +-- if not card_state then +-- return "", "could not parse card state" +-- end + +-- if card_state == "present" then +-- return "present", "" +-- end + +-- return "absent", "" +-- end + +--- Parses the output of a sim slot status entry +---@param status string? +---@return string card_status +---@return string error +local function parse_slot_status(status) + if not status or status == "" then + return "", "Command closed" + end + for card_status, slot_status in status:gmatch("Card status:%s*(%S+).-Slot status:%s*(%S+)") do + if slot_status == "active" then + return card_status, "" + end + end + + return "", 'could not parse (no active slot or invalid string format)' +end + +---@param identity ModemIdentity +---@return string? +---@return string? +---@return string error +local function read_home_network_info(identity) + local st, _, result_or_err = fibers.run_scope(function() + local cmd = exec.command { + "qmicli", "-p", "-d", identity.mode_port, "--nas-get-home-network", + stdin = "null", + stdout = "pipe", + stderr = "stdout" + } + local out, status, code, _, err = fibers.perform(cmd:combined_output_op()) + if status ~= "exited" or code ~= 0 then + error("Failed to execute qmicli command: --nas-get-home-network, reason: " .. tostring(err)) + end + + local mcc = out:match("MCC:%s+'(%d+)'") + local mnc = out:match("MNC:%s+'(%d+)'") + if not mcc or not mnc then + error("Failed to parse qmicli output: " .. tostring(out)) + end + + return { mcc = mcc, mnc = mnc } + end) + + if st == "ok" then + return result_or_err.mcc, result_or_err.mnc, "" + end + return nil, nil, result_or_err or "Unknown error" +end + +---@param identity ModemIdentity +---@return string? +---@return string error +local function read_gid1(identity) + local st, _, gid1_or_err = fibers.run_scope(function() + local cmd = exec.command { + "qmicli", "-p", "-d", identity.mode_port, "--uim-read-transparent=0x3F00,0x7FFF,0x6F3E", + stdin = "null", + stdout = "pipe", + stderr = "stdout" + } + local out, status, code, _, err = fibers.perform(cmd:combined_output_op()) + if status ~= "exited" or code ~= 0 then + error("Failed to execute qmicli command: --uim-read-transparent, reason: " .. tostring(err)) + end + + local gid1 = out:match("%s+(%S+)%s*$") + if not gid1 then + error("Failed to parse qmicli output: " .. tostring(out)) + end + + return gid1:gsub(":", "") + end) + + if st == "ok" then + return gid1_or_err, "" + end + return nil, gid1_or_err or "Unknown error" +end + +---@param identity ModemIdentity +---@return string? +---@return string error +local function read_rf_band_info(identity) + local st, _, band_or_err = fibers.run_scope(function() + local cmd = exec.command { + "qmicli", "-p", "-d", identity.mode_port, "--nas-get-rf-band-info", + stdin = "null", + stdout = "pipe", + stderr = "stdout" + } + local out, status, code, _, err = fibers.perform(cmd:combined_output_op()) + if status ~= "exited" or code ~= 0 then + error("Failed to execute qmicli command: --nas-get-rf-band-info, reason: " .. tostring(err)) + end + + local active_band_class = out:match("Active Band Class:%s*'([^']+)'") + if not active_band_class then + error("Failed to parse qmicli output: " .. tostring(out)) + end + + return active_band_class + end) + + if st == "ok" then + return band_or_err, "" + end + return nil, band_or_err or "Unknown error" +end + +---@param info ModemSimInfo +---@param gid1 string? +---@param gid_err string +---@return ModemSimInfo? +---@return string error +local function sim_info_with_gid1(info, gid1, gid_err) + if gid_err ~= "" then + gid1 = nil + end + + return modem_types.new.ModemSimInfo( + info.sim, + info.iccid, + info.imsi, + gid1, + info.sim_lock, + info.sim_lock_retries, + info.modem_state + ) +end + +local function add_mode_funcs(ModemBackend) + ---@cast ModemBackend ModemBackend + local base_read_network_info = ModemBackend.read_network_info + local base_read_sim_info = ModemBackend.read_sim_info + + ---@return boolean ok + ---@return string error + function ModemBackend:start_sim_presence_monitor() + if self.sim_present then + return false, "Already monitoring sim presence" + end + + local cmd = exec.command { + "qmicli", "-p", "-d", self.identity.mode_port, "--uim-monitor-slot-status", + stdin = "null", + stdout = "pipe", + stderr = "stdout", + flags = process_flags.owned_monitor_flags(), + } + + local stdout, err = cmd:stdout_stream() + if not stdout then + return false, "Failed to start QMI monitor: " .. tostring(err) + end + self.sim_present = { + cmd = cmd, + stdout = stdout + } + return true, "" + end + + ---Stop the long-running QMI SIM presence monitor. + ---@param timeout number? + ---@return Op + function ModemBackend:stop_sim_presence_monitor_op(timeout) + return op.guard(function() + local rec = self.sim_present + self.sim_present = nil + + if not rec then + return op.always(true, "") + end + + if rec.stdout then + pcall(function() rec.stdout:terminate("modem_sim_monitor_stop") end) + end + + if not rec.cmd then + return op.always(true, "") + end + + return rec.cmd:shutdown_op(timeout or 1.0):wrap(function(status, _code, _sig, err) + if status == "exited" or status == "signalled" then + return true, "" + end + return false, err or ("SIM monitor shutdown failed: " .. tostring(status)) + end) + end) + end + + ---@param timeout number? + ---@return boolean ok + ---@return string error + function ModemBackend:stop_sim_presence_monitor(timeout) + return fibers.perform(self:stop_sim_presence_monitor_op(timeout)) + end + + ---@return Op + function ModemBackend:wait_for_sim_present_op() + return op.guard(function() + return scope.run_op(function(s) + if not self.sim_present then + return false, "Sim presence monitor not started" + end + while true do + local chunk = s:perform(self.sim_present.stdout:read_line_op({ + terminator = "Slot status: active", + keep_terminator = true, + })) + + if not chunk then + return false, "Stream closed" + end + + local card_status, parse_err = parse_slot_status(chunk) + if parse_err == "" and card_status ~= "" then + local sim_present = card_status == "present" + return sim_present, "" + end + end + end) + :wrap(function(st, _, ...) + if st == 'ok' then + return ... + elseif st == 'cancelled' then + return false, "cancelled" + else + return false, (... or "QMI monitor failed") + end + end) + end) + end + + ---@return boolean sim_present + function ModemBackend:wait_for_sim_present() + return fibers.perform(self:wait_for_sim_present_op()) + end + + ---@return boolean sim_present + ---@return string error + function ModemBackend:is_sim_present() + local st, _, present_or_err = fibers.run_scope(function() + local cmd = exec.command { + "qmicli", "-p", "-d", self.identity.mode_port, "--uim-get-slot-status", + stdin = "null", + stdout = "pipe", + stderr = "stdout" + } + local out, status, code, _, err = fibers.perform(cmd:combined_output_op()) + if status ~= "exited" or code ~= 0 then + error("Failed to execute qmicli command: " .. tostring(err)) + end + + local state, parse_err = parse_slot_status(out) + if parse_err ~= "" then + error("Failed to parse qmicli output: " .. tostring(parse_err)) + end + return state == 'present' + end) + + if st == "ok" then + return present_or_err, "" + end + return false, present_or_err or "Unknown error" + end + + ---@param cooldown number? + ---@return boolean ok + ---@return string error + function ModemBackend:trigger_sim_presence_check(cooldown) + local st, _, err = fibers.run_scope(function() + local errors = {} + cooldown = cooldown or 1 + local cmd = exec.command { + "qmicli", "-p", "-d", self.identity.mode_port, "--uim-sim-power-off=1", + stdin = "null", + stdout = "pipe", + stderr = "stdout", + flags = process_flags.owned_monitor_flags(), + } + local _, status, code, _, power_err = fibers.perform(cmd:combined_output_op()) + if status ~= "exited" or code ~= 0 then + table.insert(errors, "Failed to execute qmicli power off command: " .. tostring(power_err)) + end + + sleep.sleep(cooldown) + + local cmd_on = exec.command { + "qmicli", "-p", "-d", self.identity.mode_port, "--uim-sim-power-on=1", + stdin = "null", + stdout = "pipe", + stderr = "stdout", + flags = process_flags.owned_monitor_flags(), + } + local _, status_on, code_on, _, err_on = fibers.perform(cmd_on:combined_output_op()) + if status_on ~= "exited" or code_on ~= 0 then + table.insert(errors, "Failed to execute qmicli power on command: " .. tostring(err_on)) + end + + if #errors > 0 then + error(table.concat(errors, ";\n")) + end + end) + + return st == "ok", err or "" + end + + ---@return ModemNetworkInfo? + ---@return string error + function ModemBackend:read_network_info() + local info, err = base_read_network_info(self) + if not info then + return nil, err + end + + local mcc, mnc, network_err = read_home_network_info(self.identity) + if network_err ~= "" then + return nil, network_err + end + + local active_band_class, band_err = read_rf_band_info(self.identity) + if band_err ~= "" then + return nil, band_err + end + + return modem_types.new.ModemNetworkInfo( + info.operator, + info.access_techs, + mcc, + mnc, + active_band_class + ) + end + + ---@return ModemSimInfo? + ---@return string error + function ModemBackend:read_sim_info() + local info, err = base_read_sim_info(self) + if not info then + return nil, err + end + + local gid1, gid_err = read_gid1(self.identity) + return sim_info_with_gid1(info, gid1, gid_err) + end +end + +return { + add_mode_funcs = add_mode_funcs, + _test = { + sim_info_with_gid1 = sim_info_with_gid1, + }, +} diff --git a/src/services/hal/backends/modem/process_flags.lua b/src/services/hal/backends/modem/process_flags.lua new file mode 100644 index 00000000..05d8da48 --- /dev/null +++ b/src/services/hal/backends/modem/process_flags.lua @@ -0,0 +1,22 @@ +local exec = require "fibers.io.exec" + +local M = {} + +---Flags for long-running modem helper commands owned by devicecode. +---process_group is always requested so shutdown reaches simple child trees. +---parent_death_signal is added when the selected lua-fibers backend advertises it. +---It may be native on pidfd/FFI backends or reaper-backed on OpenWrt plain Lua. +---@return table flags +function M.owned_monitor_flags() + local flags = { + process_group = true, + } + + if type(exec.supports) == "function" and exec.supports("parent_death_signal") then + flags.parent_death_signal = "TERM" + end + + return flags +end + +return M diff --git a/src/services/hal/backends/modem/provider.lua b/src/services/hal/backends/modem/provider.lua new file mode 100644 index 00000000..77554b42 --- /dev/null +++ b/src/services/hal/backends/modem/provider.lua @@ -0,0 +1,118 @@ +local contract = require "services.hal.backends.modem.contract" + +local MODEL_INFO = { + quectel = { + -- these are ordered, as eg25gl should match before eg25g + { mod_string = "UNKNOWN", rev_string = "eg25gl", model = "eg25", model_variant = "gl" }, + { mod_string = "UNKNOWN", rev_string = "eg25g", model = "eg25", model_variant = "g" }, + { mod_string = "UNKNOWN", rev_string = "ec25e", model = "ec25", model_variant = "e" }, + { mod_string = "em06-e", rev_string = "em06e", model = "em06", model_variant = "e" }, + { mod_string = "rm520n-gl", rev_string = "rm520ngl", model = "rm520n", model_variant = "gl" }, + { mod_string = "em12-g", rev_string = "em12g", model = "em12", model_variant = "g" }, + -- more quectel models here + }, + fibocom = {} +} + +local BACKENDS = { + "linux_mm" +} + +--- Utility function to check if a string starts with a given prefix (case-insensitive) +---@param str string +---@param start string +---@return boolean +local function starts_with(str, start) + if str == nil or start == nil then return false end + str, start = str:lower(), start:lower() + -- Use string.sub to get the prefix of mainString that is equal in length to startString + return string.sub(str, 1, string.len(start)) == start +end + +--- Select the first supported provider for the current environment. +---@return table provider +local function get_provider() + for _, backend_name in ipairs(BACKENDS) do + local ok, mod = pcall(require, "services.hal.backends.modem.providers." .. backend_name .. ".init") + if ok and type(mod) == "table" and mod.is_supported and mod.is_supported() then + return mod + end + end + error("No supported modem provider found") +end + +local function new(address) + local provider = get_provider() + local impl = provider.backend + local backend = impl.new(address) + ---@cast backend ModemBackend + local identity_info, identity_err = backend:read_identity() + if not identity_info then + error("Failed to read modem identity: " .. tostring(identity_err)) + end + + local mode = identity_info.mode + + if mode then + local ok, driver_mod = pcall(require, "services.hal.backends.modem.modes." .. mode) + if ok and type(driver_mod) == "table" and driver_mod.add_mode_funcs then + driver_mod.add_mode_funcs(backend) + end + end + + local plugin = identity_info.plugin or "" + local model = identity_info.model or "" + local revision = identity_info.revision or "" + + local model_funcs_loaded = false + for manufacturer, models in pairs(MODEL_INFO) do + if string.match(plugin:lower(), manufacturer) then + for _, details in ipairs(models) do + if details.mod_string == model:lower() + or starts_with(revision, details.rev_string) then + model = details.model + local model_variant = details.model_variant + identity_info.model = model + identity_info.model_variant = model_variant + local ok, model_mod = pcall(require, "services.hal.backends.modem.models." .. manufacturer) + if ok and type(model_mod) == "table" and model_mod.add_model_funcs then + model_mod.add_model_funcs(backend, identity_info) + model_funcs_loaded = true + end + break + end + end + end + if model_funcs_loaded then break end + end + + local iface_err = contract.validate(backend) + if iface_err ~= "" then + error("Modem backend does not implement required interface: " .. tostring(iface_err)) + end + + return backend +end + +--- Create a new ModemMonitor using the selected provider. +---@return ModemMonitor monitor +local function new_monitor() + local provider = get_provider() + if not provider.new_monitor then + error("Selected modem provider does not support modem monitoring") + end + local monitor, err = provider.new_monitor() + if not monitor then + error("Failed to create modem monitor: " .. tostring(err)) + end + local validate_err = contract.validate_monitor(monitor) + if validate_err ~= "" then + error("Modem monitor does not satisfy required interface: " .. validate_err) + end + return monitor +end + +return { + new = new, + new_monitor = new_monitor, +} diff --git a/src/services/hal/backends/modem/providers/linux_mm/impl.lua b/src/services/hal/backends/modem/providers/linux_mm/impl.lua new file mode 100644 index 00000000..d55dbddd --- /dev/null +++ b/src/services/hal/backends/modem/providers/linux_mm/impl.lua @@ -0,0 +1,816 @@ +-- service modules +local modem_types = require "services.hal.types.modem" + +-- Fiber modules +local fibers = require "fibers" +local exec = require "fibers.io.exec" +local op = require "fibers.op" +local scope = require "fibers.scope" + +-- Other modules +local json = require "cjson.safe" +local process_flags = require "services.hal.backends.modem.process_flags" + +local function terminate_command(cmd, sig) + if not cmd or type(cmd.kill) ~= 'function' then return true, nil end + local ok, a, b = pcall(function() return cmd:kill(sig or 15) end) + if not ok then return nil, tostring(a) end + if a == false or a == nil then return nil, tostring(b or 'command kill failed') end + return true, nil +end + +local function list_to_map(list) + local map = {} + for _, item in ipairs(list) do + map[item] = true + end + return map +end + +local MODEM_INFO_PATHS = { + imei = { "generic", "equipment-identifier" }, + device = { "generic", "device" }, + primary_port = { "generic", "primary-port" }, + ports = { "generic", "ports" }, + access_techs = { "generic", "access-technologies" }, + sim = { "generic", "sim" }, + drivers = { "generic", "drivers" }, + plugin = { "generic", "plugin" }, + model = { "generic", "model" }, + revision = { "generic", "revision" }, + operator = { "3gpp", "operator-name" }, + modem_state = { "generic", "state" }, + sim_lock = { "generic", "unlock-required" }, + sim_lock_retries = { "generic", "unlock-retries" }, +} + +local SIM_INFO_PATHS = { + iccid = { "properties", "iccid" }, + imsi = { "properties", "imsi" }, +} + +local SIGNAL_TECHNOLOGIES = list_to_map { + "5g", + "cdma1x", + "evdo", + "gsm", + "lte", + "umts" +} + +local SIGNAL_IGNORE_FIELDS = list_to_map { + "error-rate" +} + +---@param nested table +---@param key_paths table +---@return table +local function nested_to_flat(nested, key_paths) + local flat = {} + for key, path in pairs(key_paths) do + ---@type any + local value = nested + for _, p in ipairs(path) do + if type(value) ~= 'table' then + value = nil + break + end + value = value[p] + if value == nil then + break + end + end + if value ~= nil then + flat[key] = value + end + end + return flat +end + +---@param tbl table +---@return table +local function shallow_copy(tbl) + local copy = {} + for key, value in pairs(tbl) do + copy[key] = value + end + return copy +end + +---@param ports string[] +---@return table +local function format_ports(ports) + local formatted = { + at_ports = {}, + qmi_ports = {}, + gps_ports = {}, + net_ports = {}, + mbim_ports = {}, + } + for _, port in ipairs(ports) do + local name, port_type = port:match("^(.*) %((.*)%)$") + if name and port_type then + local key = port_type .. "_ports" + if formatted[key] then + table.insert(formatted[key], name) + end + end + end + return formatted +end + +---@param value any +---@return string[] +local function normalize_string_list(value) + if type(value) == 'table' then + local result = {} + for _, entry in ipairs(value) do + if type(entry) == 'string' and entry ~= '' then + table.insert(result, entry) + end + end + return result + end + if type(value) == 'string' and value ~= '' and value ~= "--" then + return { value } + end + return {} +end + +---@param value any +---@return string? +local function normalize_optional_string(value) + if type(value) ~= 'string' then return nil end + if value == '' or value == '--' then return nil end + return value +end + +---@param value any +---@return table? +local function normalize_unlock_retries(value) + if type(value) ~= 'table' then return nil end + local out = {} + for _, entry in ipairs(value) do + if type(entry) == 'string' then + local key, retries = entry:match("^%s*([^%(]-)%s*%((%d+)%)%s*$") + if key and retries then + key = key:match("^%s*(.-)%s*$") + if key ~= '' then + out[key] = tonumber(retries) + end + end + end + end + if next(out) == nil then return nil end + return out +end + +---@param output string +---@return string +local function parse_firmware_version(output) + for line in output:gmatch("[^\r\n]+") do + local version = line:match("version:%s*(%S+)") + if version and version ~= '' then + return version + end + end + return "" +end + +---@param drivers string[] +---@param ports table +---@return string? +local function derive_mode(drivers, ports) + local drivers_str = table.concat(drivers or {}, ",") + if drivers_str:match("cdc_mbim") or #(ports.mbim_ports or {}) > 0 then + return "mbim" + end + if drivers_str:match("qmi_wwan") or #(ports.qmi_ports or {}) > 0 then + return "qmi" + end + return nil +end + +---@param address string +---@param args string[] +---@return string? output +---@return string error +local function run_command(address, args) + local st, _, output_or_err = fibers.run_scope(function() + local cmd_args = { + stdin = "null", + stdout = "pipe", + stderr = "stdout" + } + for _, arg in ipairs(args) do + table.insert(cmd_args, arg) + end + local cmd = exec.command(cmd_args) + local output, status, code, _, err = fibers.perform(cmd:combined_output_op()) + if status ~= "exited" or code ~= 0 then + error(table.concat(args, " ") .. " failed for modem " .. tostring(address) .. ": " .. tostring(err) + .. ", output: " .. tostring(output)) + end + return output + end) + + if st == "ok" then + return output_or_err, "" + end + return nil, output_or_err or "unknown command error" +end + +---@param address string +---@return table? +---@return string error +local function parse_modem_info_json(output) + local data, json_err = json.decode(output) + if not data then + return nil, "Failed to decode mmcli info output as JSON: " + .. tostring(json_err) .. ", output: " .. tostring(output) + end + if type(data.modem) ~= 'table' then + return nil, "No modem info found in mmcli output" + end + + local flat = nested_to_flat(data.modem, MODEM_INFO_PATHS) + local ports = format_ports(flat.ports or {}) + flat.ports = nil + for key, value in pairs(ports) do + flat[key] = value + end + flat.drivers = normalize_string_list(flat.drivers) + flat.access_techs = normalize_string_list(flat.access_techs) + flat.modem_state = normalize_optional_string(flat.modem_state) + flat.sim_lock = normalize_optional_string(flat.sim_lock) + flat.sim_lock_retries = normalize_unlock_retries(flat.sim_lock_retries) + return flat, "" +end + +---@param address string +---@return table? +---@return string error +local function read_modem_info(address) + local output, err = run_command(address, { "mmcli", "-J", "-m", address }) + if not output then + return nil, err + end + + return parse_modem_info_json(output) +end + +---@param identity ModemIdentity +---@return string? +---@return string error +local function read_firmware_version(identity) + local output, err = run_command(identity.address, { "mmcli", "-m", identity.address, "--firmware-status" }) + if not output then + return nil, err + end + + local version = parse_firmware_version(output) + if version == "" then + return nil, "Failed to parse firmware version from mmcli --firmware-status" + end + return version, "" +end + +---@param identity ModemIdentity +---@return ModemSignalInfo? +---@return string error +local function read_signal_info(identity) + local output, err = run_command(identity.address, { "mmcli", "-J", "-m", identity.address, "--signal-get" }) + if not output then + return nil, err + end + + local data, json_err = json.decode(output) + if not data then + return nil, "Failed to decode mmcli output as JSON: " .. tostring(json_err) .. ", output: " .. tostring(output) + end + + local signal_techs = data.modem and data.modem.signal or nil + if type(signal_techs) ~= 'table' then + return nil, "No signal info found in mmcli output" + end + + local active_signal = nil + for tech, signals in pairs(signal_techs) do + if SIGNAL_TECHNOLOGIES[tech] and type(signals) == 'table' then + local filtered_fields = {} + for signal_name, signal_value in pairs(signals) do + if not SIGNAL_IGNORE_FIELDS[signal_name] and signal_value ~= "--" then + filtered_fields[signal_name] = signal_value + end + end + if next(filtered_fields) ~= nil then + active_signal = filtered_fields + break + end + end + end + + if not active_signal then + return nil, "No active signal found" + end + + return modem_types.new.ModemSignalInfo(shallow_copy(active_signal)) +end + +---@param sim_path string +---@return table? +---@return string error +local function read_sim_payload(sim_path) + local output, err = run_command(sim_path, { "mmcli", "-J", "-i", sim_path }) + if not output then + return nil, err + end + + local data, json_err = json.decode(output) + if not data then + return nil, + "Failed to decode mmcli SIM output as JSON: " .. tostring(json_err) .. ", output: " .. tostring(output) + end + if type(data.sim) ~= 'table' then + return nil, "No SIM info found in mmcli output" + end + + return nested_to_flat(data.sim, SIM_INFO_PATHS), "" +end + +---@param net_port string +---@param stat string +---@return integer +---@return string +local function read_net_stat(net_port, stat) + local st, _, value_or_err = fibers.run_scope(function() + local path = "/sys/class/net/" .. net_port .. "/statistics/" .. stat + local file = io.open(path, "r") + if not file then + error("Failed to open file: " .. tostring(path)) + end + local content = file:read("*a") + file:close() + if not content then + error("Failed to read file: " .. tostring(path)) + end + local value = tonumber(content) + if not value then + error("Failed to parse net stat: " .. tostring(content)) + end + return value + end) + + if st == "ok" then + return value_or_err, "" + end + return -1, value_or_err or "Unknown error" +end + +---@param address string +---@return ModemIdentity +local function get_identity(address) + local modem_info, err = read_modem_info(address) + if not modem_info then + error("Failed to fetch modem info: " .. tostring(err)) + end + + local qmi_port = modem_info.qmi_ports and modem_info.qmi_ports[1] or nil + local mbim_port = modem_info.mbim_ports and modem_info.mbim_ports[1] or nil + local selected_mode_port = mbim_port or qmi_port + if not selected_mode_port then + error("Failed to determine modem control port") + end + + local at_port = modem_info.at_ports and modem_info.at_ports[1] or nil + if not at_port then + error("Failed to determine modem AT port") + end + + local net_port = modem_info.net_ports and modem_info.net_ports[1] or nil + if not net_port then + error("Failed to determine modem network port") + end + + local identity, id_err = modem_types.new.ModemIdentity( + modem_info.imei, + address, + "/dev/" .. selected_mode_port, + "/dev/" .. at_port, + net_port, + modem_info.device + ) + if not identity then + error("Failed to get modem identity: " .. tostring(id_err)) + end + return identity +end + +---@param line string? +---@return ModemStateEvent? +---@return string error +local function parse_modem_state_line(line) + if not line or line == "" then + return nil, "Command closed" + end + + line = line:match("^%s*(.-)%s*$") + + local initial_state = line:match(": Initial state, '([^']+)'") + if initial_state then + return modem_types.new.ModemStateInitialEvent(initial_state, "initial") + end + + local old_state, new_state, reason = line:match(": State changed, '([^']+)' %-%-> '([^']+)' %(Reason: ([^)]+)%)") + if old_state and new_state then + return modem_types.new.ModemStateChangeEvent(old_state, new_state, reason) + end + + if line:match(": Removed") then + return modem_types.new.ModemStateRemovedEvent("removed") + end + + return nil, "Unknown modem state line format: " .. line +end + +local ModemBackend = {} +ModemBackend.__index = ModemBackend + +-- Default firmware fetch; model hooks may override this via self-dispatch +ModemBackend._read_firmware = read_firmware_version + +---@return ModemIdentityInfo? +---@return string error +function ModemBackend:read_identity() + local modem_info, err = read_modem_info(self.identity.address) + if not modem_info then + return nil, err + end + + local firmware = self._read_firmware(self.identity) + -- Non-fatal: firmware may not be available yet, or the model hook may not be installed yet. + -- The driver will re-call read_identity() after model hooks are applied. + firmware = firmware or nil + + return modem_types.new.ModemIdentityInfo( + modem_info.imei, + modem_info.drivers or {}, + modem_info.model, + modem_info.revision, + firmware, + modem_info.plugin, + derive_mode(modem_info.drivers or {}, { + qmi_ports = modem_info.qmi_ports or {}, + mbim_ports = modem_info.mbim_ports or {}, + }) + ) +end + +---@return ModemPortsInfo? +---@return string error +function ModemBackend:read_ports() + local modem_info, err = read_modem_info(self.identity.address) + if not modem_info then + return nil, err + end + + return modem_types.new.ModemPortsInfo( + modem_info.device, + modem_info.primary_port, + modem_info.at_ports, + modem_info.qmi_ports, + modem_info.gps_ports, + modem_info.net_ports + ) +end + +---@return ModemSimInfo? +---@return string error +function ModemBackend:read_sim_info() + local modem_info, err = read_modem_info(self.identity.address) + if not modem_info then + return nil, err + end + + local sim_info = nil + if modem_info.sim and modem_info.sim ~= "--" then + sim_info, err = read_sim_payload(modem_info.sim) + if err ~= "" then + return nil, err + end + end + + return modem_types.new.ModemSimInfo( + modem_info.sim, + sim_info and sim_info.iccid or nil, + sim_info and sim_info.imsi or nil, + nil, + modem_info.sim_lock, + modem_info.sim_lock_retries, + modem_info.modem_state + ) +end + +---@return ModemNetworkInfo? +---@return string error +function ModemBackend:read_network_info() + local modem_info, err = read_modem_info(self.identity.address) + if not modem_info then + return nil, err + end + + return modem_types.new.ModemNetworkInfo( + modem_info.operator, + modem_info.access_techs, + nil, + nil, + nil + ) +end + +---@return ModemSignalInfo? +---@return string error +function ModemBackend:read_signal() + return read_signal_info(self.identity) +end + +---@return ModemTrafficInfo? +---@return string error +function ModemBackend:read_traffic() + local rx_bytes, rx_err = read_net_stat(self.identity.net_port, "rx_bytes") + if rx_err ~= "" then + return nil, rx_err + end + + local tx_bytes, tx_err = read_net_stat(self.identity.net_port, "tx_bytes") + if tx_err ~= "" then + return nil, tx_err + end + + return modem_types.new.ModemTrafficInfo(rx_bytes, tx_bytes) +end + +---@return boolean ok +---@return string error +function ModemBackend:enable() + local _, err = run_command(self.identity.address, { "mmcli", "-m", self.identity.address, "-e" }) + return err == "", err +end + +---@return boolean ok +---@return string error +function ModemBackend:disable() + local _, err = run_command(self.identity.address, { "mmcli", "-m", self.identity.address, "-d" }) + return err == "", err +end + +---@return boolean ok +---@return string error +function ModemBackend:reset() + local _, err = run_command(self.identity.address, { "mmcli", "-m", self.identity.address, "--reset" }) + return err == "", err +end + +---@param conn_string string +---@return boolean ok +---@return string error +function ModemBackend:connect(conn_string) + local _, err = run_command(self.identity.address, { + "mmcli", "-m", self.identity.address, "--simple-connect=" .. conn_string + }) + return err == "", err +end + +---@return boolean ok +---@return string error +function ModemBackend:disconnect() + local _, err = run_command(self.identity.address, { "mmcli", "-m", self.identity.address, "--simple-disconnect" }) + return err == "", err +end + +---@return boolean ok +---@return string error +function ModemBackend:inhibit() + if self.inhibit_cmd then + return false, "Modem is already inhibited" + end + + local cmd = exec.command { + "mmcli", "-m", self.identity.address, "--inhibit", + stdin = "null", + stdout = "pipe", + stderr = "stdout", + flags = process_flags.owned_monitor_flags(), + } + + local stream, err = cmd:stdout_stream() + if not stream then + return false, "Failed to start inhibit command: --inhibit, reason: " .. tostring(err) + end + + self.inhibit_cmd = cmd + return true, "Modem inhibit started" +end + +---@param timeout number? +---@return Op +function ModemBackend:uninhibit_op(timeout) + return op.guard(function() + local cmd = self.inhibit_cmd + if not cmd then + return op.always(false, "Modem is not inhibited") + end + + self.inhibit_cmd = nil + return cmd:shutdown_op(timeout or 1.0):wrap(function(status, _code, _sig, err) + if status == "exited" or status == "signalled" then + return true, "Modem uninhibited" + end + return false, err or ("failed to stop inhibit command: " .. tostring(status)) + end) + end) +end + +---@param timeout number? +---@return boolean ok +---@return string error +function ModemBackend:uninhibit(timeout) + return fibers.perform(self:uninhibit_op(timeout)) +end + +---@return boolean ok +---@return string error +function ModemBackend:start_state_monitor() + if self.state_monitor then + return false, "Already monitoring modem state" + end + local cmd = exec.command { + "mmcli", "-m", self.identity.address, "-w", + stdin = "null", + stdout = "pipe", + stderr = "stdout", + flags = process_flags.owned_monitor_flags(), + } + local stream, err = cmd:stdout_stream() + if not stream then + return false, "Failed to start monitor state command: " .. tostring(err) + end + self.state_monitor = { cmd = cmd, stream = stream } + return true, "" +end + +---@param timeout number? +---@return Op +function ModemBackend:stop_state_monitor_op(timeout) + return op.guard(function() + local rec = self.state_monitor + self.state_monitor = nil + + if not rec then + return op.always(true, "") + end + + if rec.stream then + pcall(function() rec.stream:terminate("modem_state_monitor_stop") end) + end + + if not rec.cmd then + return op.always(true, "") + end + + return rec.cmd:shutdown_op(timeout or 1.0):wrap(function(status, _code, _sig, err) + if status == "exited" or status == "signalled" then + return true, "" + end + return false, err or ("state monitor shutdown failed: " .. tostring(status)) + end) + end) +end + +---@param timeout number? +---@return boolean ok +---@return string error +function ModemBackend:stop_state_monitor(timeout) + return fibers.perform(self:stop_state_monitor_op(timeout)) +end + +local function terminate_monitor_record(rec, stream_key, reason) + if not rec then + return + end + + local stream = rec[stream_key] + if stream then + pcall(function() stream:terminate(reason) end) + end + + if rec.cmd then + terminate_command(rec.cmd, 15) + end +end + +---Best-effort non-blocking teardown for scope finalisers. +---@param reason string? +function ModemBackend:terminate(reason) + reason = reason or "modem_backend_terminated" + + terminate_monitor_record(self.sim_present, "stdout", reason) + self.sim_present = nil + + terminate_monitor_record(self.state_monitor, "stream", reason) + self.state_monitor = nil + + if self.inhibit_cmd then + terminate_command(self.inhibit_cmd, 15) + self.inhibit_cmd = nil + end +end + +---@param timeout number? +---@return Op +function ModemBackend:shutdown_op(timeout) + timeout = timeout or 1.0 + + return scope.run_op(function(s) + local errors = {} + + if self.stop_sim_presence_monitor_op then + local ok, err = s:perform(self:stop_sim_presence_monitor_op(timeout)) + if not ok then errors[#errors + 1] = tostring(err) end + end + + if self.stop_state_monitor_op then + local ok, err = s:perform(self:stop_state_monitor_op(timeout)) + if not ok then errors[#errors + 1] = tostring(err) end + end + + if self.inhibit_cmd and self.uninhibit_op then + local ok, err = s:perform(self:uninhibit_op(timeout)) + if not ok then errors[#errors + 1] = tostring(err) end + end + + if #errors > 0 then + return false, table.concat(errors, "; ") + end + + return true, "" + end):wrap(function(st, _, ...) + if st == "ok" then + return ... + elseif st == "cancelled" then + return false, "cancelled" + end + return false, (... or "modem backend shutdown failed") + end) +end + +---@param timeout number? +---@return boolean ok +---@return string error +function ModemBackend:shutdown(timeout) + return fibers.perform(self:shutdown_op(timeout)) +end + +---@return Op +function ModemBackend:monitor_state_op() + return op.guard(function() + if not self.state_monitor then + return op.always(nil) + end + return self.state_monitor.stream:read_line_op():wrap(function(line) + local state_ev, err = parse_modem_state_line(line) + if state_ev then + self.last_state_event = state_ev + end + return state_ev, err + end) + end) +end + +---@param period number +---@return boolean ok +---@return string error +function ModemBackend:set_signal_update_interval(period) + local _, err = run_command(self.identity.address, { + "mmcli", "-m", self.identity.address, "--signal-setup=" .. tostring(period) + }) + return err == "", err +end + +---@return ModemBackend +local function new(address) + local self = { + identity = get_identity(address), + base = "linux_mm", + last_state_event = nil, + } + return setmetatable(self, ModemBackend) +end + +return { + new = new, + _test = { + normalize_unlock_retries = normalize_unlock_retries, + parse_modem_info_json = parse_modem_info_json, + }, +} diff --git a/src/services/hal/backends/modem/providers/linux_mm/init.lua b/src/services/hal/backends/modem/providers/linux_mm/init.lua new file mode 100644 index 00000000..c8f7b224 --- /dev/null +++ b/src/services/hal/backends/modem/providers/linux_mm/init.lua @@ -0,0 +1,57 @@ +local file = require "fibers.io.file" +local exec = require "fibers.io.exec" +local fibers = require "fibers" + +local backend = require "services.hal.backends.modem.providers.linux_mm.impl" +local monitor = require "services.hal.backends.modem.providers.linux_mm.monitor" + +local function is_linux() + local fh, open_err = file.open("/proc/version", "r") + if not fh or open_err then + return false + end + + local content, read_err = fh:read_all() + fh:close() + if not content or read_err then + return false + end + + return content:lower():find("linux") ~= nil +end + +--- Returns true if `mmcli` is runnable +---@return boolean ok +local function has_mmcli() + local cmd = exec.command{ + "mmcli", "--version", + stdin = "null", + stdout = "pipe", + stderr = "stdout" + } + local _, status, code, _, _ = fibers.perform(cmd:combined_output_op()) + if status == "exited" and code == 0 then + return true + end + return false +end + +--- Returns if linux with modem manager is supported +---@return boolean +local function is_supported() + local res = is_linux() and has_mmcli() + return res +end + +---@return ModemMonitor? monitor +---@return string error +local function new_monitor() + return monitor.new() +end + +return { + is_supported = is_supported, + backend = backend, + new_monitor = new_monitor, +} + diff --git a/src/services/hal/backends/modem/providers/linux_mm/monitor.lua b/src/services/hal/backends/modem/providers/linux_mm/monitor.lua new file mode 100644 index 00000000..65e08917 --- /dev/null +++ b/src/services/hal/backends/modem/providers/linux_mm/monitor.lua @@ -0,0 +1,117 @@ +local exec = require "fibers.io.exec" +local op = require "fibers.op" +local modem_types = require "services.hal.types.modem" +local process_flags = require "services.hal.backends.modem.process_flags" + +local function terminate_command(cmd, sig) + if not cmd or type(cmd.kill) ~= 'function' then return true, nil end + local ok, a, b = pcall(function() return cmd:kill(sig or 15) end) + if not ok then return nil, tostring(a) end + if a == false or a == nil then return nil, tostring(b or 'command kill failed') end + return true, nil +end + +---@class ModemMonitor +---@field cmd Command +---@field stream any +local ModemMonitor = {} +ModemMonitor.__index = ModemMonitor + +--- Parse one line from `mmcli -M` output into a ModemMonitorEvent. +--- Returns (nil, "Command closed") when the stream yields nil (end of stream). +--- Returns (nil, error) for lines that cannot be parsed. +---@param line string? +---@return ModemMonitorEvent? +---@return string error +local function parse_monitor_line(line) + if not line then + return nil, "Command closed" + end + + local status, address = line:match("^(.-)(/org%S+)") + if not address then + return nil, "line could not be parsed: " .. tostring(line) + end + + local is_added = not status:match("-") + local event, err = modem_types.new.ModemMonitorEvent(is_added, address) + if not event then + return nil, "failed to create monitor event: " .. tostring(err) + end + + return event, "" +end + +--- Returns an Op that when performed yields the next ModemMonitorEvent. +--- (nil, "Command closed") signals end of stream. +--- (nil, error) signals an unparseable line — the caller should continue looping. +---@return Op +function ModemMonitor:next_event_op() + return op.guard(function() + return self.stream:read_line_op():wrap(parse_monitor_line) + end) +end + +---Best-effort non-blocking teardown for scope finalisers. +---@param reason string? +function ModemMonitor:terminate(reason) + reason = reason or "modem_monitor_terminated" + + if self.stream then + pcall(function() self.stream:terminate(reason) end) + end + + if self.cmd then + terminate_command(self.cmd, 15) + end + + self.stream = nil + self.cmd = nil +end + +---@param timeout number? +---@return Op +function ModemMonitor:shutdown_op(timeout) + return op.guard(function() + if self.stream then + pcall(function() self.stream:terminate("modem_monitor_stop") end) + end + + local cmd = self.cmd + self.cmd = nil + self.stream = nil + + if not cmd then + return op.always(true, "") + end + + return cmd:shutdown_op(timeout or 1.0):wrap(function(status, _code, _sig, err) + if status == "exited" or status == "signalled" then + return true, "" + end + return false, err or ("modem monitor shutdown failed: " .. tostring(status)) + end) + end) +end + +--- Create and start a new ModemMonitor backed by `mmcli -M`. +---@return ModemMonitor? monitor +---@return string error +local function new() + local cmd = exec.command { + "mmcli", "-M", + stdin = "null", + stdout = "pipe", + stderr = "stdout", + flags = process_flags.owned_monitor_flags(), + } + local stream, err = cmd:stdout_stream() + if not stream then + return nil, "failed to start modem monitor: " .. tostring(err) + end + return setmetatable({ cmd = cmd, stream = stream }, ModemMonitor), "" +end + +return { + new = new, +} diff --git a/src/services/hal/backends/network/contract.lua b/src/services/hal/backends/network/contract.lua new file mode 100644 index 00000000..6d53f8a3 --- /dev/null +++ b/src/services/hal/backends/network/contract.lua @@ -0,0 +1,37 @@ +-- services/hal/backends/network/contract.lua +-- Semantic network backend contract used by the strict HAL network manager. +-- +-- Provider operations are Op-first. The manager is infrastructure and may +-- perform these Ops; service code must never call provider methods directly. + +local M = {} + +local REQUIRED_OPS = { + 'validate_op', + 'plan_op', + 'apply_op', + 'snapshot_op', + 'watch_op', + 'probe_link_op', + 'read_counters_op', + 'apply_live_weights_op', + 'apply_shaping_op', + 'speedtest_op', +} + +function M.validate_provider(provider) + if type(provider) ~= 'table' then + return nil, 'network provider must be a table' + end + for _, name in ipairs(REQUIRED_OPS) do + if type(provider[name]) ~= 'function' then + return nil, 'network provider missing ' .. name + end + end + if type(provider.terminate) ~= 'function' then + return nil, 'network provider missing terminate' + end + return true, nil +end + +return M diff --git a/src/services/hal/backends/network/provider.lua b/src/services/hal/backends/network/provider.lua new file mode 100644 index 00000000..c85a0cdc --- /dev/null +++ b/src/services/hal/backends/network/provider.lua @@ -0,0 +1,22 @@ +-- services/hal/backends/network/provider.lua +-- Network backend provider loader. + +local contract = require 'services.hal.backends.network.contract' + +local M = {} + +function M.new(config, opts) + config = config or {} + opts = opts or {} + local name = config.provider or config.backend or opts.provider or 'fake' + local modname = 'services.hal.backends.network.providers.' .. tostring(name) .. '.init' + local ok, mod_or_err = pcall(require, modname) + if not ok then return nil, mod_or_err end + local provider, err = mod_or_err.new(config, opts) + if not provider then return nil, err end + local valid, verr = contract.validate_provider(provider) + if valid ~= true then return nil, verr end + return provider, nil +end + +return M diff --git a/src/services/hal/backends/network/providers/fake/init.lua b/src/services/hal/backends/network/providers/fake/init.lua new file mode 100644 index 00000000..f4c96c14 --- /dev/null +++ b/src/services/hal/backends/network/providers/fake/init.lua @@ -0,0 +1,111 @@ +-- services/hal/backends/network/providers/fake/init.lua +-- In-memory semantic network provider for tests and initial service wiring. + +local op = require 'fibers.op' +local tablex = require 'shared.table' + +local M = {} +local Provider = {} +Provider.__index = Provider + +local function copy(v) return tablex.deep_copy(v) end + +function M.new(config, _opts) + return setmetatable({ + config = copy(config or {}), + last_intent = nil, + terminated = nil, + }, Provider), nil +end + +function Provider:validate_op(req) + return op.always({ ok = true, valid = true, checked = true, request = copy(req or {}) }) +end + +function Provider:plan_op(req) + return op.always({ + ok = true, + plan = { + backend = 'fake', + intent = copy(req and req.intent or req), + steps = {}, + }, + }) +end + +function Provider:apply_op(req) + local intent = req and (req.intent or req.desired or req) + self.last_intent = copy(intent) + return op.always({ + ok = true, + applied = true, + changed = true, + backend = 'fake', + intent_rev = intent and intent.rev or nil, + }) +end + +function Provider:snapshot_op(_req) + return op.always({ + ok = true, + backend = 'fake', + observed = copy(self.last_intent), + last_intent = copy(self.last_intent), + }) +end + +function Provider:watch_op(_req) + return op.always({ + ok = true, + backend = 'fake', + watching = true, + }) +end + +function Provider:probe_link_op(req) + return op.always({ + ok = true, + backend = 'fake', + link_id = req and req.link_id or nil, + state = 'unknown', + }) +end + +function Provider:read_counters_op(req) + return op.always({ + ok = true, + backend = 'fake', + links = {}, + request = copy(req or {}), + }) +end + + +function Provider:apply_live_weights_op(req) + self.last_live_weights = copy(req or {}) + return op.always({ ok = true, backend = 'fake', changed = true, applied = true, request = copy(req or {}) }) +end + +function Provider:apply_shaping_op(req) + self.last_shaping = copy(req or {}) + return op.always({ ok = true, backend = 'fake', changed = true, applied = true, request = copy(req or {}) }) +end + +function Provider:speedtest_op(req) + return op.always({ + ok = true, + backend = 'fake', + interface = req and req.interface, + device = req and req.device, + peak_mbps = req and req.fake_mbps or 10, + data_mib = 1, + duration_s = 0.1, + }) +end + +function Provider:terminate(reason) + self.terminated = reason or true + return true, nil +end + +return M diff --git a/src/services/hal/backends/network/providers/openwrt/hotplug_client.lua b/src/services/hal/backends/network/providers/openwrt/hotplug_client.lua new file mode 100644 index 00000000..883b12e4 --- /dev/null +++ b/src/services/hal/backends/network/providers/openwrt/hotplug_client.lua @@ -0,0 +1,90 @@ +-- services/hal/backends/network/providers/openwrt/hotplug_client.lua +-- +-- Tiny hotplug/MWAN3 event forwarder. It is intentionally dumb: it captures +-- environment supplied by procd/mwan3 and writes one JSON line to the provider's +-- UNIX socket. Policy and snapshotting stay in the OpenWrt provider. + +local socket = require 'fibers.io.socket' +local cjson = require 'cjson.safe' + +local M = {} + +local COMMON_ENV_KEYS = { + 'ACTION', 'INTERFACE', 'DEVICE', 'DEVICENAME', 'DEVNAME', 'DEVPATH', 'DEVTYPE', + 'SUBSYSTEM', 'SEQNUM', 'IFINDEX', 'IFUPDATE_ADDRESSES', 'IFUPDATE_ROUTES', + 'IFUPDATE_PREFIXES', 'IFUPDATE_DATA', 'PRODUCT', 'BUSNUM', 'DEVNUM', 'TYPE', +} + +local function copy_env(env) + local out = {} + for k, v in pairs(env or {}) do + if type(k) == 'string' and type(v) == 'string' then out[k] = v end + end + return out +end + +local function process_env() + local out = {} + for i = 1, #COMMON_ENV_KEYS do + local k = COMMON_ENV_KEYS[i] + local v = os.getenv(k) + if v ~= nil then out[k] = v end + end + return out +end + +local function argv_opts(argv) + local opts = {} + for i = 1, #(argv or {}) do + local a = tostring(argv[i] or '') + local k, v = a:match('^%-%-([^=]+)=(.*)$') + if k then opts[k] = v end + end + return opts +end + +function M.send(record, opts) + opts = opts or {} + local path = opts.socket_path or os.getenv('DEVICECODE_NET_HOTPLUG_SOCKET') or '/var/run/devicecode-net-observe.sock' + local line = cjson.encode(record or {}) + if type(line) ~= 'string' then return nil, 'encode failed' end + local st, err = socket.connect_unix(path) + if not st then return nil, err end + local ok, werr = st:write(line .. '\n') + if st.flush then st:flush() end + st:close() + if not ok then return nil, werr end + return true, nil +end + +function M.record_from_env(kind, directory, env) + return { + source = kind or 'hotplug', + kind = kind or 'hotplug', + directory = directory, + env = copy_env(env or process_env()), + } +end + +function M.main(argv, env) + argv = argv or arg or {} + env = env or process_env() + -- Lua cannot portably enumerate process environment. Shell wrappers should + -- pass the relevant variables in argv as --KEY=value, but tests may pass env. + local opts = argv_opts(argv) + local record = { + source = opts.source or opts.kind or 'hotplug', + kind = opts.kind or opts.source or 'hotplug', + directory = opts.directory or opts.dir, + env = {}, + } + for k, v in pairs(env) do record.env[k] = v end + for k, v in pairs(opts) do + if k ~= 'socket' and k ~= 'socket_path' and k ~= 'kind' and k ~= 'source' and k ~= 'directory' and k ~= 'dir' then + record.env[k] = v + end + end + return M.send(record, { socket_path = opts.socket or opts.socket_path }) +end + +return M diff --git a/src/services/hal/backends/network/providers/openwrt/hotplug_send.lua b/src/services/hal/backends/network/providers/openwrt/hotplug_send.lua new file mode 100644 index 00000000..54a74655 --- /dev/null +++ b/src/services/hal/backends/network/providers/openwrt/hotplug_send.lua @@ -0,0 +1,89 @@ +#!/usr/bin/env lua +-- Forward one OpenWrt hotplug/MWAN3 event to the devicecode network observer. +-- Intended for tiny scripts in /etc/hotplug.d/* and /etc/mwan3.user. + +local function dirname(path) + path = tostring(path or '') + local dir = path:match('^(.*)/[^/]*$') + if dir == nil or dir == '' then return '.' end + return dir +end + + +local function prepend_package_paths(paths, cpaths) + if #paths > 0 then package.path = table.concat(paths, ';') .. ';' .. package.path end + if #cpaths > 0 then package.cpath = table.concat(cpaths, ';') .. ';' .. package.cpath end +end + +local function split_sender_path(path) + path = tostring(path or '') + local prefix = path:match('^(.*)/src/services/hal/backends/network/providers/openwrt/hotplug_send%.lua$') + if prefix then return prefix, prefix .. '/src' end + prefix = path:match('^(.*)/services/hal/backends/network/providers/openwrt/hotplug_send%.lua$') + if prefix then return prefix, prefix end + return nil, nil +end + +local function bootstrap_package_path() + local src = arg and arg[0] or debug.getinfo(1, 'S').source or '' + if src:sub(1, 1) == '@' then src = src:sub(2) end + local project_root, lua_root = split_sender_path(src) + if not project_root then + local dir = dirname(src) + -- Fall back from .../openwrt to the package root if this file was copied + -- without its original tree. This remains harmless when paths do not exist. + lua_root = dir .. '/../../../../../../' + project_root = lua_root + end + + local paths = { + -- Installed/check-out deployments flatten third-party modules into lib/. + -- This is the layout on production targets, for example lib/fibers.lua + -- and lib/fibers/*.lua. Keep these before vendor paths so the hook + -- uses the same module set as the running checkout. + project_root .. '/lib/?.lua', + project_root .. '/lib/?/init.lua', + lua_root .. '/lib/?.lua', + lua_root .. '/lib/?/init.lua', + + lua_root .. '/?.lua', + lua_root .. '/?/init.lua', + project_root .. '/src/?.lua', + project_root .. '/src/?/init.lua', + + -- Development and test checkouts may still carry the original vendor tree. + project_root .. '/vendor/lua-fibers/src/?.lua', + project_root .. '/vendor/lua-fibers/src/?/init.lua', + project_root .. '/vendor/lua-bus/src/?.lua', + project_root .. '/vendor/lua-bus/src/?/init.lua', + project_root .. '/vendor/lua-trie/src/?.lua', + project_root .. '/vendor/lua-trie/src/?/init.lua', + project_root .. '/vendor/lua-trie/?.lua', + project_root .. '/vendor/lua-trie/?/init.lua', + } + local cpaths = { + project_root .. '/lib/?.so', + lua_root .. '/lib/?.so', + project_root .. '/vendor/?.so', + project_root .. '/vendor/?/?.so', + project_root .. '/vendor/?/lib/?.so', + } + prepend_package_paths(paths, cpaths) +end + +bootstrap_package_path() + +local fibers = require 'fibers' +local client = require 'services.hal.backends.network.providers.openwrt.hotplug_client' + +local ok, err +fibers.run(function () + ok, err = client.main(arg) +end) + +if ok ~= true then + -- Hotplug paths must not block or retry. Exit non-zero for diagnostics only. + io.stderr:write('devicecode hotplug forward failed: ', tostring(err), '\n') + os.exit(1) +end +os.exit(0) diff --git a/src/services/hal/backends/network/providers/openwrt/init.lua b/src/services/hal/backends/network/providers/openwrt/init.lua new file mode 100644 index 00000000..cc648213 --- /dev/null +++ b/src/services/hal/backends/network/providers/openwrt/init.lua @@ -0,0 +1,2463 @@ +-- services/hal/backends/network/providers/openwrt/init.lua +-- OpenWrt semantic network provider. +-- +-- This module is the HAL boundary where product-level NET intent is translated +-- into OpenWrt UCI packages. NET must not require this module directly. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local exec = require 'fibers.io.exec' +local file = require 'fibers.io.file' +local cjson = require 'cjson.safe' +local uci_manager = require 'services.hal.backends.openwrt.uci_manager' +local observer_mod = require 'services.hal.backends.network.providers.openwrt.observer' +local mwan3_mod = require 'services.hal.backends.network.providers.openwrt.mwan3' +local shaper_mod = require 'services.hal.backends.network.providers.openwrt.shaper' +local speedtest_mod = require 'services.hal.backends.network.providers.openwrt.speedtest' +local names_mod = require 'services.hal.backends.network.providers.openwrt.names' +local hal_types = require 'services.hal.types.core' + +local perform = fibers.perform +local M = {} +local Provider = {} +Provider.__index = Provider + +local HOST_NETMASK = '255.255.255.255' +local DEFAULT_OBSERVER_SOCKET = '/var/run/devicecode-net-observe.sock' +local DEFAULT_HOTPLUG_SENDER = '/usr/lib/lua/services/hal/backends/network/providers/openwrt/hotplug_send.lua' +local OBSERVER_HOTPLUG_NAME = '95-devicecode-net-observe' +local DEFAULT_IFACE_HOTPLUG_HOOK = '/etc/hotplug.d/iface/' .. OBSERVER_HOTPLUG_NAME +local DEFAULT_NET_HOTPLUG_HOOK = '/etc/hotplug.d/net/' .. OBSERVER_HOTPLUG_NAME +local OBSERVER_BLOCK_BEGIN = '# BEGIN devicecode-net-observer' +local OBSERVER_BLOCK_END = '# END devicecode-net-observer' +local OWNED_PACKAGES = { 'network', 'dhcp', 'firewall', 'mwan3' } +local COUNTER_STATS = { + 'rx_bytes', + 'rx_packets', + 'rx_dropped', + 'rx_errors', + 'tx_bytes', + 'tx_packets', + 'tx_dropped', + 'tx_errors', +} +local ACTIVATION_COMMANDS = { + network = { { kind = 'reload', target = 'network', wait = true } }, + dhcp = { { kind = 'restart', target = 'dnsmasq', wait = true } }, + firewall = { { kind = 'restart', target = 'firewall', wait = true } }, + -- Always include mwan3 so stale generated multi-WAN state is removed when + -- WAN/multi-WAN is disabled or a member disappears. Structural apply + -- restarts mwan3; live weight changes do not. + mwan3 = { { kind = 'restart', target = 'mwan3', wait = true } }, +} + +local function elapsed_ms(t0) + if not t0 then return nil end + return math.floor(((fibers.now() - t0) * 1000) + 0.5) +end + +local function log_provider(self, level, payload) + local logger = self and self.logger + if logger and type(logger[level]) == 'function' then + payload = payload or {} + payload.component = payload.component or 'provider' + payload.provider = payload.provider or 'openwrt' + logger[level](logger, payload) + end +end + +local function exec_spec(argv, opts) + opts = opts or {} + local spec = {} + for i = 1, #argv do + spec[i] = argv[i] + end + spec.stdin = opts.stdin ~= nil and opts.stdin or 'null' + spec.stdout = opts.stdout ~= nil and opts.stdout or 'null' + spec.stderr = opts.stderr ~= nil and opts.stderr or 'null' + if opts.flags ~= nil then spec.flags = opts.flags end + if opts.shutdown_grace ~= nil then spec.shutdown_grace = opts.shutdown_grace end + return spec +end + +local is_plain_table +local sorted_keys + +local function run_provider_command(self, argv) + local runner = self and self.shaper_run_cmd + if type(runner) == 'function' then + return runner(argv) + end + local cmd = exec.command(exec_spec(argv)) + local status, code = perform(cmd:run_op()) + if status == 'exited' and code == 0 then return true, nil end + return nil, table.concat(argv, ' ') .. ' exited with status=' .. tostring(status) .. ' code=' .. tostring(code) +end + +local function run_hook_command(self, argv) + local cfg = self and self.config or {} + local runner = cfg.hook_run_cmd or cfg.run_cmd + if type(runner) == 'function' then return runner(argv) end + local cmd = exec.command(exec_spec(argv)) + local status, code = perform(cmd:run_op()) + if status == 'exited' and code == 0 then return true, nil end + return nil, table.concat(argv or {}, ' ') .. ' exited with status=' .. tostring(status) .. ' code=' .. tostring(code) +end + +local function shell_quote(v) + v = tostring(v or '') + return "'" .. v:gsub("'", "'\\''") .. "'" +end + +local function observer_socket_path(config) + config = config or {} + return config.observer_socket_path or config.hotplug_socket_path or DEFAULT_OBSERVER_SOCKET +end + +local function path_dirname(path) + return tostring(path or ''):match('^(.*)/[^/]+$') or '.' +end + +local function absolute_path(path) + if type(path) ~= 'string' or path == '' then return nil end + if path:sub(1, 1) == '/' then return path end + path = path:gsub('^%./', '') + local cwd = os.getenv('PWD') + if type(cwd) == 'string' and cwd ~= '' then return cwd:gsub('/+$', '') .. '/' .. path end + return path +end + +local function source_tree_hotplug_sender_path() + local info = debug and debug.getinfo and debug.getinfo(1, 'S') or nil + local source = info and info.source or nil + if type(source) ~= 'string' then return nil end + if source:sub(1, 1) == '@' then source = source:sub(2) end + local dir = path_dirname(absolute_path(source) or source) + if dir and dir ~= '' then return dir .. '/hotplug_send.lua' end + return nil +end + +local function hotplug_sender_path(config) + config = config or {} + return config.hotplug_sender_path or source_tree_hotplug_sender_path() or DEFAULT_HOTPLUG_SENDER +end + +local function observer_hook_root(config) + config = config or {} + local root = config.observer_hook_root or config.observer_hooks_root + if type(root) == 'string' and root ~= '' then + return root:gsub('/+$', '') + end + return '' +end + +local function rooted_hook_path(config, path) + local root = observer_hook_root(config) + if root == '' then return path end + return root .. path +end + +local function write_text_file(path, content) + local f, err = file.open(path, 'w') + if not f then return nil, err end + local ok, werr = f:write(content or '') + if ok == false or ok == nil then + f:close() + return nil, werr + end + if f.flush then + local fok, ferr = f:flush() + if fok == false or fok == nil then + f:close() + return nil, ferr + end + end + local cok, cerr = f:close() + if cok == false or cok == nil then return nil, cerr end + return true, nil +end + +local function read_text_file(path) + local f = file.open(path, 'r') + if not f then return nil end + local data = f:read_all() + f:close() + return data or '' +end + +local function observer_forward_script(config, source, kind, directory, prelude) + local socket_path = observer_socket_path(config) + local sender = hotplug_sender_path(config) + local lines = { + '#!/bin/sh', + '# Installed by devicecode. Keep policy in Lua; this only wakes the observer.', + 'DEVICECODE_NET_HOTPLUG_SOCKET=${DEVICECODE_NET_HOTPLUG_SOCKET:-' .. shell_quote(socket_path) .. '}', + 'LUA_BIN=${DEVICECODE_LUA:-}', + 'if [ -z "$LUA_BIN" ]; then', + ' if command -v lua >/dev/null 2>&1; then LUA_BIN=lua; elif command -v luajit >/dev/null 2>&1; then LUA_BIN=luajit; fi', + 'fi', + } + if type(prelude) == 'string' and prelude ~= '' then + for line in tostring(prelude):gmatch('[^\n]+') do lines[#lines + 1] = line end + end + lines[#lines + 1] = 'DEVICECODE_NET_HOTPLUG_SENDER=' .. shell_quote(sender) + lines[#lines + 1] = 'if [ ! -S "$DEVICECODE_NET_HOTPLUG_SOCKET" ]; then' + lines[#lines + 1] = ' logger -t devicecode-net-observe "socket missing: $DEVICECODE_NET_HOTPLUG_SOCKET source=' .. tostring(source) .. ' action=${ACTION:-} interface=${INTERFACE:-}" 2>/dev/null || true' + lines[#lines + 1] = 'elif [ ! -r "$DEVICECODE_NET_HOTPLUG_SENDER" ]; then' + lines[#lines + 1] = ' logger -t devicecode-net-observe "sender missing: $DEVICECODE_NET_HOTPLUG_SENDER" 2>/dev/null || true' + lines[#lines + 1] = 'elif [ -n "$LUA_BIN" ]; then' + lines[#lines + 1] = ' "$LUA_BIN" "$DEVICECODE_NET_HOTPLUG_SENDER"' + .. ' --source=' .. shell_quote(source) + .. ' --kind=' .. shell_quote(kind) + .. ' --directory=' .. shell_quote(directory) + .. ' --socket="$DEVICECODE_NET_HOTPLUG_SOCKET"' + .. ' --ACTION="${ACTION:-}"' + .. ' --INTERFACE="${INTERFACE:-}"' + .. ' --DEVICE="${DEVICE:-}"' + .. ' --DEVICENAME="${DEVICENAME:-}"' + .. ' --DEVNAME="${DEVNAME:-}"' + .. ' --SUBSYSTEM="${SUBSYSTEM:-}" >/dev/null 2>&1 || {' + .. ' logger -t devicecode-net-observe "send failed source=' .. tostring(source) .. ' kind=' .. tostring(kind) .. ' directory=' .. tostring(directory) .. ' action=${ACTION:-} interface=${INTERFACE:-}" 2>/dev/null || true; }' + lines[#lines + 1] = 'else' + lines[#lines + 1] = ' logger -t devicecode-net-observe "lua interpreter missing" 2>/dev/null || true' + lines[#lines + 1] = 'fi' + lines[#lines + 1] = 'exit 0' + return table.concat(lines, '\n') .. '\n' +end + +local function mwan3_user_block(config) + local prelude = table.concat({ + 'case "$1" in', + ' ifup|ifdown|online|offline|connected|disconnected|disabled|enabled)', + ' [ -n "$ACTION" ] || export ACTION="$1"', + ' [ -n "$INTERFACE" ] || export INTERFACE="$2"', + ' ;;', + ' *)', + ' [ -n "$INTERFACE" ] || export INTERFACE="$1"', + ' [ -n "$ACTION" ] || export ACTION="$2"', + ' ;;', + 'esac', + }, '\n') + local body = observer_forward_script(config, 'mwan3', 'mwan3', 'mwan3.user', prelude) + body = body:gsub('^#!/bin/sh\n', '') + body = body:gsub('exit 0\n$', '') + return OBSERVER_BLOCK_BEGIN .. '\n' .. body .. OBSERVER_BLOCK_END .. '\n' +end + +local function strip_observer_block(content) + content = tostring(content or '') + local pattern = '\n?%# BEGIN devicecode%-net%-observer.-%# END devicecode%-net%-observer\n?' + local stripped = content:gsub(pattern, '\n') + stripped = stripped:gsub('\n+$', '\n') + return stripped +end + +local function install_observer_file(self, path, content) + local cfg = self and self.config or {} + local dir = path_dirname(path) + local ok, err = file.mkdir_p(dir) + if ok ~= true then return nil, 'failed to create ' .. dir .. ': ' .. tostring(err) end + ok, err = write_text_file(path, content) + if ok ~= true then return nil, 'failed to write ' .. path .. ': ' .. tostring(err) end + if cfg.observer_hook_chmod ~= false then + ok, err = run_hook_command(self, { 'chmod', '+x', path }) + if ok ~= true then return nil, 'failed to chmod ' .. path .. ': ' .. tostring(err) end + end + return true, nil +end + +local function install_mwan3_user_hook(self) + local cfg = self and self.config or {} + local path = rooted_hook_path(cfg, cfg.mwan3_user_path or '/etc/mwan3.user') + local old = read_text_file(path) + local base = strip_observer_block(old or '') + if base == '' then base = '#!/bin/sh\n' end + if not base:match('^#!') then base = '#!/bin/sh\n' .. base end + if not base:match('\n$') then base = base .. '\n' end + local shebang, rest = base:match('^(#![^\n]*\n?)(.*)$') + shebang = shebang or '#!/bin/sh\n' + rest = rest or base + local content = shebang .. '\n' .. mwan3_user_block(cfg) .. '\n' .. rest + return install_observer_file(self, path, content) +end + +local function default_hotplug_hook_path(directory) + directory = tostring(directory or '') + if directory == 'iface' then return DEFAULT_IFACE_HOTPLUG_HOOK end + if directory == 'net' then return DEFAULT_NET_HOTPLUG_HOOK end + return '/etc/hotplug.d/' .. directory .. '/' .. OBSERVER_HOTPLUG_NAME +end + +local function install_hotplug_hook(self, directory) + local cfg = self and self.config or {} + local path = rooted_hook_path(cfg, default_hotplug_hook_path(directory)) + local content = observer_forward_script(cfg, 'hotplug', 'hotplug', directory) + return install_observer_file(self, path, content) +end + +local function install_observer_hooks(self) + local cfg = self and self.config or {} + if cfg.install_observer_hooks == false then return true, nil end + if cfg.allow_fake_uci == true and cfg.install_observer_hooks ~= true and observer_hook_root(cfg) == '' then + return true, nil, { 'skipped:fake_uci' } + end + local installed = {} + local ok, err = install_hotplug_hook(self, 'iface') + if ok ~= true then return nil, err end + installed[#installed + 1] = 'hotplug:iface' + ok, err = install_hotplug_hook(self, 'net') + if ok ~= true then return nil, err end + installed[#installed + 1] = 'hotplug:net' + ok, err = install_mwan3_user_hook(self) + if ok ~= true then return nil, err end + installed[#installed + 1] = 'mwan3.user' + return true, nil, installed +end + +local function wait_for_linux_device(self, dev, opts) + if type(dev) ~= 'string' or dev == '' then return true, nil end + opts = opts or {} + local timeout_s = tonumber(opts.timeout_s) or 10 + local interval_s = tonumber(opts.interval_s) or 0.1 + local deadline = fibers.now() + timeout_s + local last_err = nil + repeat + local ok, err = run_provider_command(self, { 'ip', 'link', 'show', 'dev', dev }) + if ok == true then return true, nil end + last_err = err + fibers.perform(sleep.sleep_op(interval_s)) + until fibers.now() >= deadline + local ok, err = run_provider_command(self, { 'ip', 'link', 'show', 'dev', dev }) + if ok == true then return true, nil end + return nil, 'network activation did not produce shaping target ' .. tostring(dev) .. ': ' .. tostring(err or last_err or 'not found') +end + +local function wait_for_shaping_targets(self, shaping_request, opts) + local links = shaping_request and shaping_request.links or {} + for _, link_id in ipairs(sorted_keys(links)) do + local link = links[link_id] + if is_plain_table(link) and link.enabled ~= false then + local ok, err = wait_for_linux_device(self, link.iface or link.device, opts) + if ok ~= true then return nil, tostring(link_id) .. ': ' .. tostring(err) end + end + end + return true, nil +end + +local read_uci_packages +local snapshot_from_packages +local build_observed_snapshot + +function is_plain_table(v) + return type(v) == 'table' and getmetatable(v) == nil +end + +function sorted_keys(t) + local ks = {} + for k in pairs(t or {}) do ks[#ks + 1] = k end + table.sort(ks, function(a, b) return tostring(a) < tostring(b) end) + return ks +end + +local function count_keys(t) + local n = 0 + for _ in pairs(t or {}) do n = n + 1 end + return n +end + +local function validate_intent(intent) + if not is_plain_table(intent) then return { ok = false, err = 'intent must be a table', backend = 'openwrt' } end + if intent.schema ~= nil and intent.schema ~= 'devicecode.net.intent/1' then + return { ok = false, err = 'unsupported intent schema: ' .. tostring(intent.schema), backend = 'openwrt' } + end + if not is_plain_table(intent.segments) then return { ok = false, err = 'intent.segments must be a map', backend = 'openwrt' } end + if not is_plain_table(intent.interfaces) then return { ok = false, err = 'intent.interfaces must be a map', backend = 'openwrt' } end + return { ok = true, valid = true, backend = 'openwrt' } +end + + +local function bool_uci(v) + if v == nil then return nil end + return v and '1' or '0' +end + +local function parse_cidr(cidr) + if type(cidr) ~= 'string' then return nil, nil end + local addr, prefix = cidr:match('^([^/]+)/(%d+)$') + if not addr then return nil, nil end + prefix = tonumber(prefix) + if not prefix or prefix < 0 or prefix > 32 then return nil, nil end + return addr, prefix +end + +local function prefix_to_netmask(prefix) + prefix = tonumber(prefix) + if not prefix or prefix < 0 or prefix > 32 then return nil end + local rem = prefix + local octs = {} + for i = 1, 4 do + if rem >= 8 then + octs[i] = 255 + rem = rem - 8 + elseif rem > 0 then + octs[i] = 256 - (2 ^ (8 - rem)) + rem = 0 + else + octs[i] = 0 + end + end + return string.format('%d.%d.%d.%d', octs[1], octs[2], octs[3], octs[4]) +end + + +local function netmask_to_prefix(mask) + if type(mask) ~= 'string' then return nil end + local a, b, c, d = mask:match('^(%d+)%.(%d+)%.(%d+)%.(%d+)$') + if not a then return nil end + a, b, c, d = tonumber(a), tonumber(b), tonumber(c), tonumber(d) + if not (a and b and c and d) then return nil end + local bits = { [255] = 8, [254] = 7, [252] = 6, [248] = 5, [240] = 4, [224] = 3, [192] = 2, [128] = 1, [0] = 0 } + local pfx, partial = 0, false + for _, oct in ipairs({ a, b, c, d }) do + local n = bits[oct] + if n == nil then return nil end + if partial and n ~= 0 then return nil end + if n ~= 8 then partial = true end + pfx = pfx + n + end + return pfx +end + +local function cidr_from_addr_netmask(addr, netmask) + if type(addr) ~= 'string' or addr == '' then return nil end + local pfx = netmask_to_prefix(netmask) + if not pfx then return nil end + return addr .. '/' .. tostring(pfx) +end + +local function bool_from_uci(v) + if v == nil then return nil end + if v == true or v == '1' or v == 1 or v == 'true' or v == 'yes' or v == 'on' or v == 'enabled' then return true end + if v == false or v == '0' or v == 0 or v == 'false' or v == 'no' or v == 'off' or v == 'disabled' then return false end + return nil +end + +local function list_from_uci(v) + if v == nil then return {} end + if type(v) == 'table' then + local out = {} + for i = 1, #v do out[i] = v[i] end + return out + end + return { v } +end + +local function copy_plain(v) + if type(v) ~= 'table' then return v end + local out = {} + for k, val in pairs(v) do out[k] = copy_plain(val) end + return out +end + +local function ipv4_spec(primary, fallback) + local p = is_plain_table(primary) and primary or {} + local f = is_plain_table(fallback) and fallback or {} + local ip = p.ipv4 or p.v4 or p + if not is_plain_table(ip) then ip = {} end + local fip = f.ipv4 or f.v4 or f + if not is_plain_table(fip) then fip = {} end + + local out = {} + for k, v in pairs(fip) do out[k] = v end + for k, v in pairs(ip) do out[k] = v end + if out.mode == nil then out.mode = out.proto end + return out +end + +local function cidr_to_addr_prefix(spec) + if type(spec.cidr) == 'string' then + local addr, prefix = parse_cidr(spec.cidr) + if addr then return addr, prefix end + end + local addr = spec.address or spec.addr or spec.ip or spec.ipaddr + local prefix = spec.prefix or spec.prefix_len + if prefix == nil and type(spec.netmask) == 'string' then return addr, nil, spec.netmask end + return addr, prefix +end + +local function set_section(changes, config, section, stype) + changes[#changes + 1] = { op = 'set', config = config, section = section, option = stype } +end + +local function set_option(changes, config, section, option, value) + if value == nil then return end + changes[#changes + 1] = { op = 'set', config = config, section = section, option = option, value = value } +end + +local function add_section(changes, known, config, section, stype) + known[section] = true + set_section(changes, config, section, stype) + return section +end + +local function segment_zone_name(seg_id, seg) + local fw = is_plain_table(seg and seg.firewall) and seg.firewall or {} + if type(fw.zone) == 'string' and fw.zone ~= '' then return fw.zone end + return seg_id +end + +local function segment_vlan_id(seg) + local vlan = seg and seg.vlan + if type(vlan) == 'number' then return vlan end + if is_plain_table(vlan) then return vlan.id end + return nil +end + +local function platform_segment_trunk(provider_config) + provider_config = provider_config or {} + local platform = is_plain_table(provider_config.platform) and provider_config.platform or {} + local trunk = platform.segment_trunk or provider_config.segment_trunk + if not is_plain_table(trunk) then return nil end + local ifname = trunk.ifname or trunk.device or trunk.name + if type(ifname) ~= 'string' or ifname == '' then return nil end + return trunk, ifname +end + +local function segment_is_enabled(seg) + return is_plain_table(seg) and seg.enabled ~= false +end + + + +local function list_contains(list, value) + if type(list) ~= 'table' then return false end + for i = 1, #list do if list[i] == value then return true end end + return false +end + +local function ensure_array(v) + if v == nil then return {} end + if type(v) == 'table' then + local out = {} + if #v > 0 then + for i = 1, #v do out[#out + 1] = v[i] end + else + for _, k in ipairs(sorted_keys(v)) do out[#out + 1] = v[k] end + end + return out + end + return { v } +end + +local function dirname(path) + if type(path) ~= 'string' then return nil end + local d = path:match('^(.+)/[^/]+$') + return d +end + +local function add_unique_value(out, seen, v) + if type(v) ~= 'string' or v == '' or seen[v] then return end + seen[v] = true + out[#out + 1] = v +end + +local function ipv4_literal(s) + if type(s) ~= 'string' then return false end + local a, b, c, d = s:match('^(%d+)%.(%d+)%.(%d+)%.(%d+)$') + if not a then return false end + for _, part in ipairs({ a, b, c, d }) do + local n = tonumber(part) + if not n or n < 0 or n > 255 then return false end + end + return true +end + +local function ipv6_literal(s) + -- Conservative syntax check for dnsmasq address records. It is better to + -- omit an invalid static record than poison the whole per-segment dnsmasq + -- instance, which also owns DHCP for that segment. + if type(s) ~= 'string' or not s:find(':', 1, true) then return false end + return s:match('^[0-9A-Fa-f:%.]+$') ~= nil +end + +local function dns_address_literal(s) + return ipv4_literal(s) or ipv6_literal(s) +end + +local function dns_records_for_segment(dns, seg_id) + local out = {} + local records = is_plain_table(dns.records) and dns.records or {} + local function add_record(rid, rec) + if not is_plain_table(rec) then return end + local name = rec.name or rid + local addr = rec.address or rec.ip or rec.value + if type(name) ~= 'string' or name == '' or type(addr) ~= 'string' or addr == '' then return end + if rec.segment ~= nil and rec.segment ~= seg_id then return end + if rec.segments ~= nil and not list_contains(rec.segments, seg_id) then return end + if not dns_address_literal(addr) then return end + out[#out + 1] = '/' .. name .. '/' .. addr + end + if #records > 0 then + for i = 1, #records do add_record(tostring(i), records[i]) end + else + for _, rid in ipairs(sorted_keys(records)) do add_record(rid, records[rid]) end + end + table.sort(out) + return out +end + +local function resolve_host_file_sources(dns, seg) + local cfg = is_plain_table(dns.host_files) and dns.host_files or {} + local base_dir = cfg.base_dir or cfg.root or cfg.dir or '/data/devicecode/dns/hosts' + local sources = is_plain_table(cfg.sources) and cfg.sources or {} + local seg_dns = is_plain_table(seg.dns) and seg.dns or {} + local ids = seg_dns.host_files or seg_dns.host_sources or {} + local files, mounts, seen_files, seen_mounts = {}, {}, {}, {} + if cfg.addnmount ~= false then add_unique_value(mounts, seen_mounts, base_dir) end + for _, source_id in ipairs(ensure_array(ids)) do + local path + local sid = tostring(source_id) + local spec = is_plain_table(sources[sid]) and sources[sid] or nil + if spec then + path = spec.path or (spec.file and (base_dir .. '/' .. tostring(spec.file))) or (base_dir .. '/' .. sid .. '.hosts') + else + path = sid:sub(1, 1) == '/' and sid or (base_dir .. '/' .. sid .. '.hosts') + end + add_unique_value(files, seen_files, path) + add_unique_value(mounts, seen_mounts, dirname(path)) + end + table.sort(files) + table.sort(mounts) + return files, mounts +end + +local function segment_ipv4_cidr(seg) + local ipv4 = ipv4_spec(seg and seg.addressing or {}, {}) + if type(ipv4.cidr) == 'string' then return ipv4.cidr end + local addr, prefix, netmask = cidr_to_addr_prefix(ipv4) + if type(addr) == 'string' and prefix ~= nil then return addr .. '/' .. tostring(prefix) end + if type(addr) == 'string' and type(netmask) == 'string' then return cidr_from_addr_netmask(addr, netmask) end + return nil +end + +local function route_entries(routes) + local out = {} + if not is_plain_table(routes) then return out end + if #routes > 0 then + for i = 1, #routes do if is_plain_table(routes[i]) then out[#out + 1] = { id = tostring(i), rec = routes[i] } end end + else + for _, id in ipairs(sorted_keys(routes)) do if is_plain_table(routes[id]) then out[#out + 1] = { id = id, rec = routes[id] } end end + end + return out +end + + +local function seg_l2_mode(seg) + local l2 = is_plain_table(seg and seg.l2) and seg.l2 or {} + if type(l2.mode) == 'string' and l2.mode ~= '' then return l2.mode end + local kind = seg and seg.kind or 'lan' + if kind == 'wan' or kind == 'uplink' then return 'direct' end + return 'bridge' +end + +local function segment_data_device(provider_config, seg_id, seg, name_ctx) + local trunk, base_ifname = platform_segment_trunk(provider_config) + local vid = segment_vlan_id(seg) + if not (trunk and base_ifname and vid and name_ctx) then return nil end + if seg_l2_mode(seg) == 'bridge' and type(name_ctx.bridge) == 'function' then + return name_ctx:bridge(seg_id) + end + if type(name_ctx.vlan) == 'function' then + return name_ctx:vlan(seg_id) + end + return nil +end + +local function segment_shaping_device(provider_config, seg_id, seg, name_ctx) + local trunk, base_ifname = platform_segment_trunk(provider_config) + local vid = segment_vlan_id(seg) + if not (trunk and base_ifname and vid) then return nil end + return segment_data_device(provider_config, seg_id, seg, name_ctx) +end + +local DEFAULT_FQ_CODEL = { flows = 1024, limit = 10240, memory_limit = '16Mb', target = '5ms', interval = '100ms' } + +local function require_rate(v, field) + if type(v) ~= 'string' or v == '' then error('segment shaping ' .. field .. ' must be a non-empty rate string') end + return v +end + +local function compile_segment_shaping(seg_shape) + if not is_plain_table(seg_shape) or seg_shape.enabled == false then return nil end + local def = is_plain_table(seg_shape.host_default) and seg_shape.host_default or nil + if not def then return nil end + local fq = seg_shape.fq_codel or def.fq_codel or DEFAULT_FQ_CODEL + local mode = def.mode or 'budgeted_peak' + if mode ~= 'budgeted_peak' then error('segment shaping host_default.mode must be budgeted_peak') end + local function build_dir(name, match) + local d = is_plain_table(def[name]) and def[name] or nil + if not d then error('segment shaping host_default.' .. name .. ' is required') end + local aggregate_spec = is_plain_table(seg_shape[name]) and seg_shape[name] or nil + local aggregate = aggregate_spec and aggregate_spec.limit or nil + local out_dir = { + enabled = d.enabled ~= false, + match = match, + mode = mode, + host_rate = require_rate(d.sustained_rate, 'host_default.' .. name .. '.sustained_rate'), + host_ceil = require_rate(d.peak_rate, 'host_default.' .. name .. '.peak_rate'), + host_burst = require_rate(d.burst_budget, 'host_default.' .. name .. '.burst_budget'), + host_cburst = d.burst_budget, + all_hosts = def.all_hosts ~= false, + fq_codel = copy_plain(fq), + } + if aggregate then + out_dir.segment_aggregate = true + out_dir.pool_rate = require_rate(aggregate, name .. '.limit') + out_dir.pool_ceil = aggregate + end + return out_dir + end + return { + enabled = true, + egress = build_dir('download', 'dst'), + ingress = build_dir('upload', 'src'), + host_overrides = copy_plain(seg_shape.host_overrides or {}), + } +end + +local function interface_linux_device(intent, ifid, name_ctx) + if type(ifid) ~= 'string' or ifid == '' then return nil end + local iface = is_plain_table(intent.interfaces) and intent.interfaces[ifid] or nil + if is_plain_table(iface) then + local ep = is_plain_table(iface.endpoint) and iface.endpoint or {} + local dev = ep.ifname or ep.device or ep.name or iface.device + if type(dev) == 'string' and dev ~= '' then return dev end + end + return ifid +end + +local function provider_shaping_marks(provider_config) + local platform = is_plain_table(provider_config and provider_config.platform) and provider_config.platform or {} + local shaping = is_plain_table(platform.shaping) and platform.shaping or {} + return copy_plain(shaping.marks or {}) +end + +local function add_backhaul_shaping_link(intent, links, name_ctx, member_id, member, marks) + local b = is_plain_table(member) and member.shaping or nil + if not (is_plain_table(b) and b.enabled ~= false) then return end + local dev = interface_linux_device(intent, member.interface, name_ctx) + if type(dev) ~= 'string' or dev == '' then return end + local down = is_plain_table(b.download) and b.download or nil + local up = is_plain_table(b.upload) and b.upload or nil + local control = is_plain_table(b.control) and b.control or {} + local function limit(dir, fallback) + return dir and dir.limit or fallback + end + local link = { + kind = 'wan_mark', + iface = dev, + member = member_id, + marks = copy_plain(marks), + egress = { + enabled = up ~= nil and up.enabled ~= false, + client = { rate = limit(up, '1gbit'), ceil = limit(up, '1gbit') }, + control = { rate = control.rate or '1gbit', ceil = control.ceil or control.rate or '1gbit' }, + fq_codel = copy_plain(b.fq_codel or {}), + }, + ingress = { + enabled = down ~= nil and down.enabled ~= false, + client = { rate = limit(down, '1gbit'), ceil = limit(down, '1gbit') }, + control = { rate = control.rate or '1gbit', ceil = control.ceil or control.rate or '1gbit' }, + fq_codel = copy_plain(b.fq_codel or {}), + }, + } + links['backhaul_' .. tostring(member_id)] = link +end + +local function compile_backhaul_shaping_links(intent, links, name_ctx, marks) + local members = is_plain_table(intent.wan) and is_plain_table(intent.wan.members) and intent.wan.members or {} + for _, member_id in ipairs(sorted_keys(members)) do + local member = members[member_id] + add_backhaul_shaping_link(intent, links, name_ctx, member_id, member, marks) + end + return links +end + +local function has_segment_shaping_intent(seg_shape) + if not is_plain_table(seg_shape) then return false end + return seg_shape.download ~= nil or seg_shape.upload ~= nil or seg_shape.host_default ~= nil or seg_shape.host_overrides ~= nil +end + +local function build_shaping_request(intent, provider_config, name_ctx) + local marks = provider_shaping_marks(provider_config) + local links = {} + compile_backhaul_shaping_links(intent, links, name_ctx, marks) + for _, seg_id in ipairs(sorted_keys(intent.segments or {})) do + local seg = intent.segments[seg_id] + local seg_shape = is_plain_table(seg.shaping) and seg.shaping or {} + if seg_shape.enabled ~= false and has_segment_shaping_intent(seg_shape) then + local spec = compile_segment_shaping(seg_shape) + if is_plain_table(spec) and spec.enabled ~= false then + local iface = segment_shaping_device(provider_config, seg_id, seg, name_ctx) + local subnet = segment_ipv4_cidr(seg) + if iface and subnet then + local one = copy_plain(spec) + one.iface = iface + one.subnet = subnet + one.segment = seg_id + links[seg_id] = one + end + end + end + end + return { enabled = next(links) ~= nil, marks = marks, links = links } +end + + +local function build_segment_interface_proto(seg) + local ipv4 = ipv4_spec(seg and seg.addressing or {}, {}) + local proto = ipv4.mode or ipv4.proto + if proto == nil then + local addr = cidr_to_addr_prefix(ipv4) + proto = addr and 'static' or 'none' + end + if proto == 'manual' then proto = 'none' end + return proto, ipv4 +end + + +-- Devicecode owns the OpenWrt UCI packages completely; all OpenWrt-visible +-- names are allocated through names.lua before rendering. + + +local function add_static_or_dhcp_interface(changes, known, ifsec, devname, proto, ipv4, auto) + add_section(changes, known, 'network', ifsec, 'interface') + set_option(changes, 'network', ifsec, 'proto', proto) + set_option(changes, 'network', ifsec, 'auto', auto == false and '0' or '1') + set_option(changes, 'network', ifsec, 'disabled', auto == false and '1' or '0') + set_option(changes, 'network', ifsec, 'device', devname) + if proto == 'static' then + local addr, prefix, netmask = cidr_to_addr_prefix(ipv4) + set_option(changes, 'network', ifsec, 'ipaddr', addr) + set_option(changes, 'network', ifsec, 'netmask', netmask or prefix_to_netmask(prefix)) + set_option(changes, 'network', ifsec, 'gateway', ipv4.gateway or ipv4.gw) + set_option(changes, 'network', ifsec, 'dns', ipv4.dns) + else + set_option(changes, 'network', ifsec, 'peerdns', bool_uci(ipv4.peerdns)) + set_option(changes, 'network', ifsec, 'defaultroute', bool_uci(ipv4.defaultroute)) + if ipv4.metric then set_option(changes, 'network', ifsec, 'metric', ipv4.metric) end + end +end + + +local function mwan_members_by_interface(intent) + local out = {} + local members = intent and intent.wan and intent.wan.members or {} + local keys = sorted_keys(members) + for i, mid in ipairs(keys) do + local m = is_plain_table(members[mid]) and members[mid] or {} + if m.enabled ~= false and m.disabled ~= true then + local iface = m.interface or mid + if type(iface) == 'string' and iface ~= '' then + out[iface] = { route_metric = tonumber(m.route_metric) or (10 + i) } + end + end + end + return out +end + +local function apply_mwan_network_defaults(ipv4, rec) + if not rec then return ipv4 end + ipv4 = copy_plain(ipv4 or {}) or {} + ipv4.metric = ipv4.metric or rec.route_metric + -- mwan3 expects ordinary WAN interface routes in the main table. We may + -- still honour an explicit false override, but generated configs must not + -- suppress the default route by default. + return ipv4 +end + + +local function filter_unrealised_mwan_members(intent) + local wan = intent and intent.wan + local members = wan and wan.members + if not is_plain_table(members) then return intent end + local out = copy_plain(intent) or {} + out.wan = copy_plain(wan) or {} + out.wan.members = {} + for _, mid in ipairs(sorted_keys(members)) do + local m = is_plain_table(members[mid]) and members[mid] or {} + local iface = m.interface or mid + if (out.interfaces and out.interfaces[iface]) or (out.segments and out.segments[iface]) then + out.wan.members[mid] = copy_plain(m) + end + end + return out +end + +local function build_network_changes(intent, provider_config, name_ctx) + local changes = {} + local known = {} + local segment_to_ifaces = {} + local mwan_by_iface = mwan_members_by_interface(intent) + + add_section(changes, known, 'network', 'loopback', 'interface') + set_option(changes, 'network', 'loopback', 'device', 'lo') + set_option(changes, 'network', 'loopback', 'proto', 'static') + set_option(changes, 'network', 'loopback', 'ipaddr', '127.0.0.1') + set_option(changes, 'network', 'loopback', 'netmask', '255.0.0.0') + + add_section(changes, known, 'network', 'globals', 'globals') + set_option(changes, 'network', 'globals', 'ula_prefix', ((intent.addressing or {}).ipv6 or {}).ula_prefix or 'auto') + + local _trunk, base_ifname = platform_segment_trunk(provider_config) + local explicit_segment = {} + for _, ifid in ipairs(sorted_keys(intent.interfaces or {})) do + local iface = intent.interfaces[ifid] + local owns_segment = iface.role ~= 'wan' and iface.realises_segment ~= false + if owns_segment and type(iface.segment) == 'string' then explicit_segment[iface.segment] = true end + if owns_segment and type(iface.segments) == 'table' then + for i = 1, #iface.segments do explicit_segment[iface.segments[i]] = true end + end + end + + if base_ifname then + for _, seg_id in ipairs(sorted_keys(intent.segments or {})) do + local seg = intent.segments[seg_id] + if segment_is_enabled(seg) and not explicit_segment[seg_id] then + local vid = segment_vlan_id(seg) + if vid then + local vlan_name = name_ctx:vlan(seg_id) + local vlan_sec = name_ctx:section('dev_vlan', seg_id) + add_section(changes, known, 'network', vlan_sec, 'device') + set_option(changes, 'network', vlan_sec, 'type', '8021q') + set_option(changes, 'network', vlan_sec, 'ifname', base_ifname) + set_option(changes, 'network', vlan_sec, 'vid', vid) + set_option(changes, 'network', vlan_sec, 'name', vlan_name) + + local devname = segment_data_device(provider_config, seg_id, seg, name_ctx) or vlan_name + if seg_l2_mode(seg) == 'bridge' then + local br_name = name_ctx:bridge(seg_id) + local br_sec = name_ctx:section('dev_bridge', seg_id) + add_section(changes, known, 'network', br_sec, 'device') + set_option(changes, 'network', br_sec, 'name', br_name) + set_option(changes, 'network', br_sec, 'type', 'bridge') + set_option(changes, 'network', br_sec, 'ports', { vlan_name }) + set_option(changes, 'network', br_sec, 'bridge_empty', '1') + end + + local ifsec = name_ctx:iface(seg_id) + local proto, ipv4 = build_segment_interface_proto(seg) + ipv4 = apply_mwan_network_defaults(ipv4, mwan_by_iface[seg_id]) + add_static_or_dhcp_interface(changes, known, ifsec, devname, proto, ipv4, true) + segment_to_ifaces[seg_id] = segment_to_ifaces[seg_id] or {} + segment_to_ifaces[seg_id][#segment_to_ifaces[seg_id] + 1] = ifsec + end + end + end + end + + for _, ifid in ipairs(sorted_keys(intent.interfaces or {})) do + local iface = intent.interfaces[ifid] + local ifsec = name_ctx:iface(ifid) + local devname + if iface.kind == 'bridge' then + devname = name_ctx:bridge(ifid) + local devsec = name_ctx:section('dev_bridge', ifid) + add_section(changes, known, 'network', devsec, 'device') + set_option(changes, 'network', devsec, 'name', devname) + set_option(changes, 'network', devsec, 'type', 'bridge') + set_option(changes, 'network', devsec, 'ports', iface.members or {}) + set_option(changes, 'network', devsec, 'bridge_empty', '1') + else + local ep = is_plain_table(iface.endpoint) and iface.endpoint or {} + devname = ep.ifname or ep.device or ep.name or iface.device + end + local seg = iface.segment and (intent.segments or {})[iface.segment] or nil + local ipv4 = ipv4_spec(iface.addressing, seg and seg.addressing or {}) + local proto = ipv4.mode or ipv4.proto + if proto == nil then proto = iface.role == 'wan' and 'dhcp' or 'static' end + if proto == 'manual' then proto = 'none' end + ipv4 = apply_mwan_network_defaults(ipv4, mwan_by_iface[ifid]) + add_static_or_dhcp_interface(changes, known, ifsec, devname, proto, ipv4, iface.enabled ~= false) + if iface.mtu then set_option(changes, 'network', ifsec, 'mtu', iface.mtu) end + + local segs = {} + if type(iface.segment) == 'string' then segs[#segs + 1] = iface.segment end + if type(iface.segments) == 'table' then for i = 1, #iface.segments do segs[#segs + 1] = iface.segments[i] end end + for i = 1, #segs do + segment_to_ifaces[segs[i]] = segment_to_ifaces[segs[i]] or {} + segment_to_ifaces[segs[i]][#segment_to_ifaces[segs[i]] + 1] = ifsec + end + end + for _, list in pairs(segment_to_ifaces) do table.sort(list) end + + local routes = (is_plain_table(intent.routing) and intent.routing.routes) or {} + for _, item in ipairs(route_entries(routes)) do + local r = item.rec + local rsec = name_ctx:section('route', item.id) + add_section(changes, known, 'network', rsec, 'route') + set_option(changes, 'network', rsec, 'interface', r.interface and name_ctx:iface(r.interface) or nil) + set_option(changes, 'network', rsec, 'target', r.target) + set_option(changes, 'network', rsec, 'netmask', r.netmask or (r.kind == 'host' and HOST_NETMASK or nil)) + set_option(changes, 'network', rsec, 'gateway', r.gateway) + set_option(changes, 'network', rsec, 'metric', r.metric) + set_option(changes, 'network', rsec, 'table', r.table) + end + + return changes, known, segment_to_ifaces +end + +local function canonical_list(list) + local out = ensure_array(list) + table.sort(out, function(a, b) return tostring(a) < tostring(b) end) + return out +end + +local function list_key(list) + local out = {} + for i = 1, #(list or {}) do out[i] = tostring(list[i]) end + return table.concat(out, ',') +end + +local function dns_effective_for_segment(dns, dhcp, seg_id, seg) + local dh = is_plain_table(seg.dhcp) and seg.dhcp or {} + local seg_dns = is_plain_table(seg.dns) and seg.dns or {} + local host_ids = canonical_list(seg_dns.host_files or seg_dns.host_sources or {}) + local wants_dns = dns.enabled ~= false and seg_dns.enabled ~= false and (dh.enabled == true or seg_dns.local_server == true or seg_dns['local'] == true or #host_ids > 0) + if not wants_dns then return nil end + local addnhosts, addnmounts = resolve_host_file_sources(dns, seg) + local addresses = dns_records_for_segment(dns, seg_id) + local domain = seg_dns.domain or dns.domain + local upstreams = canonical_list(dns.upstreams or {}) + local label = #host_ids > 0 and table.concat(host_ids, '_') or 'standard' + local key = table.concat({ + tostring(domain or ''), + list_key(upstreams), + list_key(addnhosts), + list_key(addnmounts), + list_key(addresses), + tostring(dns.domainneeded ~= false), + tostring(dns.boguspriv ~= false), + tostring(dns.localservice ~= false), + }, '|') + return { + key = key, + label = label, + domain = domain, + upstreams = upstreams, + addnhosts = addnhosts, + addnmounts = addnmounts, + addresses = addresses, + cachesize = is_plain_table(dns.cache) and dns.cache.size or nil, + authoritative = (is_plain_table(dhcp.defaults) and dhcp.defaults.authoritative), + } +end + +local function build_dhcp_changes(intent, name_ctx, segment_to_ifaces) + local changes = {} + local known = {} + local dns = is_plain_table(intent.dns) and intent.dns or {} + local dhcp = is_plain_table(intent.dhcp) and intent.dhcp or {} + local defaults = is_plain_table(dhcp.defaults) and dhcp.defaults or {} + local loopback_segment = dns.loopback_segment or dns.local_resolver_segment + if loopback_segment == nil and is_plain_table(intent.segments) and intent.segments.int ~= nil then loopback_segment = 'int' end + local groups = {} + local seg_instance = {} + + for _, seg_id in ipairs(sorted_keys(intent.segments or {})) do + local seg = intent.segments[seg_id] + local eff = dns_effective_for_segment(dns, dhcp, seg_id, seg) + if eff then + groups[eff.key] = groups[eff.key] or eff + groups[eff.key].segments = groups[eff.key].segments or {} + groups[eff.key].interfaces = groups[eff.key].interfaces or {} + groups[eff.key].segments[#groups[eff.key].segments + 1] = seg_id + local ifaces = segment_to_ifaces[seg_id] or { name_ctx:iface(seg_id) } + for i = 1, #ifaces do groups[eff.key].interfaces[#groups[eff.key].interfaces + 1] = ifaces[i] end + seg_instance[seg_id] = eff.key + end + end + + for _, key in ipairs(sorted_keys(groups)) do + local g = groups[key] + table.sort(g.interfaces) + local dnssec = name_ctx:dns_instance(g.label .. '_' .. key) + add_section(changes, known, 'dhcp', dnssec, 'dnsmasq') + set_option(changes, 'dhcp', dnssec, 'domainneeded', dns.domainneeded ~= nil and bool_uci(dns.domainneeded) or '1') + set_option(changes, 'dhcp', dnssec, 'boguspriv', dns.boguspriv ~= nil and bool_uci(dns.boguspriv) or '1') + set_option(changes, 'dhcp', dnssec, 'localise_queries', '1') + set_option(changes, 'dhcp', dnssec, 'rebind_protection', '1') + set_option(changes, 'dhcp', dnssec, 'rebind_localhost', '1') + set_option(changes, 'dhcp', dnssec, 'expandhosts', '1') + set_option(changes, 'dhcp', dnssec, 'nonegcache', '0') + set_option(changes, 'dhcp', dnssec, 'readethers', '1') + set_option(changes, 'dhcp', dnssec, 'nonwildcard', '1') + -- dnsmasq implicitly listens on loopback even when constrained to a + -- specific interface. With one dnsmasq section per DNS policy, multiple + -- instances racing for 127.0.0.1:53 can make one or more instances fail to + -- start. Keep exactly one generated instance as the router-local resolver + -- and exclude lo from all the others. By default the private/int segment + -- owns loopback when present; configs can override this with + -- dns.loopback_segment / dns.local_resolver_segment. + if not (loopback_segment and list_contains(g.segments, loopback_segment)) then + set_option(changes, 'dhcp', dnssec, 'notinterface', { 'lo' }) + end + set_option(changes, 'dhcp', dnssec, 'localservice', dns.localservice ~= nil and bool_uci(dns.localservice) or '1') + set_option(changes, 'dhcp', dnssec, 'authoritative', g.authoritative ~= nil and bool_uci(g.authoritative) or '1') + set_option(changes, 'dhcp', dnssec, 'port', '53') + set_option(changes, 'dhcp', dnssec, 'noresolv', '1') + set_option(changes, 'dhcp', dnssec, 'interface', g.interfaces) + set_option(changes, 'dhcp', dnssec, 'leasefile', '/tmp/dhcp.leases.' .. dnssec) + set_option(changes, 'dhcp', dnssec, 'resolvfile', '/tmp/resolv.conf.d/resolv.conf.auto') + if g.cachesize ~= nil then set_option(changes, 'dhcp', dnssec, 'cachesize', g.cachesize) end + if #g.upstreams > 0 then set_option(changes, 'dhcp', dnssec, 'server', g.upstreams) end + if type(g.domain) == 'string' and g.domain ~= '' then + set_option(changes, 'dhcp', dnssec, 'domain', g.domain) + set_option(changes, 'dhcp', dnssec, 'local', '/' .. g.domain .. '/') + end + if #g.addnhosts > 0 then set_option(changes, 'dhcp', dnssec, 'addnhosts', g.addnhosts) end + if #g.addnmounts > 0 then set_option(changes, 'dhcp', dnssec, 'addnmount', g.addnmounts) end + if #g.addresses > 0 then set_option(changes, 'dhcp', dnssec, 'address', g.addresses) end + g.instance = dnssec + end + + for _, seg_id in ipairs(sorted_keys(intent.segments or {})) do + local seg = intent.segments[seg_id] + local dh = is_plain_table(seg.dhcp) and seg.dhcp or {} + local sec = name_ctx:section('dhcp', seg_id) + add_section(changes, known, 'dhcp', sec, 'dhcp') + set_option(changes, 'dhcp', sec, 'interface', (segment_to_ifaces[seg_id] and segment_to_ifaces[seg_id][1]) or name_ctx:iface(seg_id)) + local g = groups[seg_instance[seg_id]] + if g and g.instance then set_option(changes, 'dhcp', sec, 'instance', g.instance) end + if dh.enabled == true then + set_option(changes, 'dhcp', sec, 'start', dh.start or dh.range_start or defaults.start or 100) + set_option(changes, 'dhcp', sec, 'limit', dh.limit or dh.range_limit or defaults.limit or 150) + set_option(changes, 'dhcp', sec, 'leasetime', dh.leasetime or dh.lease_time or defaults.leasetime or defaults.lease_time or '12h') + local opts = dh.options or (is_plain_table(dhcp.options) and dhcp.options[seg_id]) + if type(opts) == 'table' then set_option(changes, 'dhcp', sec, 'dhcp_option', opts) end + else + set_option(changes, 'dhcp', sec, 'ignore', '1') + end + end + + + for _, ifid in ipairs(sorted_keys(intent.interfaces or {})) do + local iface = intent.interfaces[ifid] + local dh = is_plain_table(iface.dhcp) and iface.dhcp or {} + if iface.role == 'wan' or dh.enabled == false then + local sec = name_ctx:section('dhcp', ifid) + add_section(changes, known, 'dhcp', sec, 'dhcp') + set_option(changes, 'dhcp', sec, 'interface', name_ctx:iface(ifid)) + set_option(changes, 'dhcp', sec, 'ignore', '1') + end + end + + local reservations = is_plain_table(dhcp.reservations) and dhcp.reservations or {} + local function add_reservation(rid, rec) + if not is_plain_table(rec) then return end + local sec = name_ctx:section('host', rid) + add_section(changes, known, 'dhcp', sec, 'host') + set_option(changes, 'dhcp', sec, 'name', rec.name or rid) + set_option(changes, 'dhcp', sec, 'mac', rec.mac) + set_option(changes, 'dhcp', sec, 'ip', rec.ip or rec.address) + set_option(changes, 'dhcp', sec, 'leasetime', rec.leasetime or rec.lease_time) + set_option(changes, 'dhcp', sec, 'hostid', rec.hostid) + set_option(changes, 'dhcp', sec, 'duid', rec.duid) + end + if #reservations > 0 then for i = 1, #reservations do add_reservation(tostring(i), reservations[i]) end else for _, rid in ipairs(sorted_keys(reservations)) do add_reservation(rid, reservations[rid]) end end + + return changes, known +end + +local function build_firewall_changes(intent, segment_to_ifaces, name_ctx) + local changes = {} + local known = {} + local fw = is_plain_table(intent.firewall) and intent.firewall or {} + local defaults = is_plain_table(fw.defaults) and fw.defaults or {} + add_section(changes, known, 'firewall', 'defaults', 'defaults') + local wrote = {} + for _, key in ipairs({ 'input', 'output', 'forward' }) do + set_option(changes, 'firewall', 'defaults', key, defaults[key] or (key == 'output' and 'ACCEPT' or 'REJECT')) + wrote[key] = true + end + for _, key in ipairs(sorted_keys(defaults)) do if not wrote[key] then set_option(changes, 'firewall', 'defaults', key, defaults[key]) end end + + local zone_to_networks = {} + for _, seg_id in ipairs(sorted_keys(intent.segments or {})) do + local seg = intent.segments[seg_id] + local zname = segment_zone_name(seg_id, seg) + zone_to_networks[zname] = zone_to_networks[zname] or {} + local ifaces = segment_to_ifaces[seg_id] + if ifaces and #ifaces > 0 then for i = 1, #ifaces do zone_to_networks[zname][#zone_to_networks[zname] + 1] = ifaces[i] end end + end + for _, ifid in ipairs(sorted_keys(intent.interfaces or {})) do + local iface = intent.interfaces[ifid] + local zname = iface.firewall and iface.firewall.zone or (iface.role == 'wan' and 'wan' or nil) + if type(zname) == 'string' and zname ~= '' then + zone_to_networks[zname] = zone_to_networks[zname] or {} + zone_to_networks[zname][#zone_to_networks[zname] + 1] = name_ctx:iface(ifid) + end + end + local zone_specs = is_plain_table(fw.zones) and fw.zones or {} + for _, zname in ipairs(sorted_keys(zone_specs)) do zone_to_networks[zname] = zone_to_networks[zname] or {} end + for _, zname in ipairs(sorted_keys(zone_to_networks)) do + local zsec = name_ctx:section('zone', zname) + local zspec = is_plain_table(zone_specs[zname]) and zone_specs[zname] or {} + local nets = zone_to_networks[zname] + local dedup, seen_nets = {}, {} + for i = 1, #(nets or {}) do + local n = nets[i] + if type(n) == 'string' and n ~= '' and not seen_nets[n] then + seen_nets[n] = true + dedup[#dedup + 1] = n + end + end + nets = dedup + table.sort(nets) + local oz = name_ctx:zone(zname) + add_section(changes, known, 'firewall', zsec, 'zone') + set_option(changes, 'firewall', zsec, 'name', oz) + if #nets > 0 then set_option(changes, 'firewall', zsec, 'network', nets) end + set_option(changes, 'firewall', zsec, 'input', zspec.input or 'ACCEPT') + set_option(changes, 'firewall', zsec, 'output', zspec.output or 'ACCEPT') + set_option(changes, 'firewall', zsec, 'forward', zspec.forward or 'REJECT') + for _, key in ipairs(sorted_keys(zspec)) do + if key ~= 'input' and key ~= 'output' and key ~= 'forward' and key ~= 'network' then set_option(changes, 'firewall', zsec, key, zspec[key]) end + end + end + local policies = is_plain_table(fw.policies) and fw.policies or {} + local n = 0 + for _, pid in ipairs(sorted_keys(policies)) do + local p = policies[pid] + if is_plain_table(p) then + local src = p.src or p.from + local dest = p.dest or p.to + if type(src) == 'string' and type(dest) == 'string' then + n = n + 1 + local sec = name_ctx:section('fwd', pid .. '_' .. tostring(n)) + add_section(changes, known, 'firewall', sec, 'forwarding') + set_option(changes, 'firewall', sec, 'src', name_ctx:zone(src)) + set_option(changes, 'firewall', sec, 'dest', name_ctx:zone(dest)) + end + end + end + local rules = is_plain_table(fw.rules) and fw.rules or {} + local function add_rule(rid, r) + if not is_plain_table(r) then return end + local sec = name_ctx:section('rule', rid) + add_section(changes, known, 'firewall', sec, 'rule') + set_option(changes, 'firewall', sec, 'name', r.name or rid) + for _, key in ipairs({ 'enabled', 'family', 'proto', 'src_ip', 'dest_ip', 'src_port', 'dest_port', 'icmp_type', 'target', 'limit', 'limit_burst', 'extra', 'utc_time', 'weekdays', 'monthdays', 'start_time', 'stop_time', 'start_date', 'stop_date' }) do + set_option(changes, 'firewall', sec, key, r[key]) + end + if r.src then set_option(changes, 'firewall', sec, 'src', name_ctx:zone(r.src)) end + if r.dest then set_option(changes, 'firewall', sec, 'dest', name_ctx:zone(r.dest)) end + end + if #rules > 0 then for i = 1, #rules do add_rule(tostring(i), rules[i]) end else for _, rid in ipairs(sorted_keys(rules)) do add_rule(rid, rules[rid]) end end + return changes, known +end + +local function transaction_record(pkg, changes, restart_cmds) + return { + config = pkg, + replace_package = true, + changes = changes or {}, + restart_cmds = restart_cmds or {}, + } +end + +local function transaction_records(changes) + local records = {} + for i = 1, #OWNED_PACKAGES do + local pkg = OWNED_PACKAGES[i] + records[i] = transaction_record(pkg, changes[pkg], ACTIVATION_COMMANDS[pkg]) + end + return records +end + +local function build_uci_plan(intent, provider_config) + local name_ctx = names_mod.allocate(intent, provider_config) + local n_changes, n_known, segment_to_ifaces = build_network_changes(intent, provider_config, name_ctx) + local d_changes, d_known = build_dhcp_changes(intent, name_ctx, segment_to_ifaces) + local f_changes, f_known = build_firewall_changes(intent, segment_to_ifaces, name_ctx) + local m_changes, m_known, m_plan = mwan3_mod.build_changes(filter_unrealised_mwan_members(intent), name_ctx) + return { + name_ctx = name_ctx, + mwan_plan = m_plan, + changes = { network = n_changes, dhcp = d_changes, firewall = f_changes, mwan3 = m_changes }, + sections = { network = n_known, dhcp = d_known, firewall = f_known, mwan3 = m_known }, + counts = { + network = { changes = #n_changes, sections = count_keys(n_known) }, + dhcp = { changes = #d_changes, sections = count_keys(d_known) }, + firewall = { changes = #f_changes, sections = count_keys(f_known) }, + mwan3 = { changes = #m_changes, sections = count_keys(m_known) }, + }, + } +end + +local function trigger_post_apply_observation(self, trace) + local obs = self and self._observer or nil + if not obs or type(obs.ingest) ~= 'function' then return true, nil end + local ok, err = obs:ingest({ + source = 'apply', + kind = 'apply_done', + action = 'post_apply', + generation = trace and trace.generation or nil, + apply_id = trace and trace.apply_id or nil, + }) + log_provider(self, ok == true and 'debug' or 'warn', { + what = 'openwrt_apply_observation_queued', + ok = ok == true, + err = err, + generation = trace and trace.generation or nil, + apply_id = trace and trace.apply_id or nil, + }) + return ok, err +end + +function M.new(config, opts) + config = config or {} + opts = opts or {} + local cfg = {} + for k, v in pairs(config) do cfg[k] = v end + local self = setmetatable({ + config = cfg, + terminated = nil, + _scope = nil, + _uci_manager = config.uci_manager, + _external_uci_manager = config.uci_manager ~= nil, + _owner_scope = opts.owner_scope or config.owner_scope or fibers.current_scope(), + _observer = nil, + cap_emit_ch = opts.cap_emit_ch or config.cap_emit_ch, + logger = opts.logger or config.logger, + _runs = {}, + apply_mwan_live_weights = config.apply_mwan_live_weights, + mwan_run_cmd = config.mwan_run_cmd, + mwan_run_cmd_capture = config.mwan_run_cmd_capture, + mwan_run_restore = config.mwan_run_restore, + shaper_run_cmd = config.shaper_run_cmd or config.run_cmd, + shaper_run_restore = config.shaper_run_restore, + speedtest_run_cmd = config.speedtest_run_cmd, + counter_reader = config.counter_reader or config.stat_reader, + }, Provider) + log_provider(self, 'debug', { + what = 'openwrt_network_provider_instrumented_build', + marker = 'owned_activation_runner_v1', + activation_runner = 'owned', + confdir = cfg.confdir or cfg.uci_confdir or '/etc/config', + }) + return self, nil +end + +function Provider:_manager() + if self._uci_manager then + if self._uci_manager._closed then + if self._external_uci_manager then return nil, 'bound UCI manager is closed' end + self._uci_manager = nil + self._scope = nil + else + return self._uci_manager, nil + end + end + local mgr, err = uci_manager.new({ + confdir = self.config.confdir or self.config.uci_confdir or '/etc/config', + savedir = self.config.savedir or self.config.uci_savedir, + allow_fake = self.config.allow_fake_uci == true, + debounce_s = self.config.debounce_s or 0.02, + run_cmd = self.config.run_cmd, + logger = self.logger, + }) + if not mgr then return nil, err end + self._uci_manager = mgr + return mgr, nil +end + +function Provider:_owner() + if self._owner_scope then return self._owner_scope end + local scope = fibers.current_scope() + if scope then self._owner_scope = scope end + return self._owner_scope +end + +function Provider:_ensure_started() + local mgr, err = self:_manager() + if not mgr then return nil, err end + local scope = self:_owner() + if not scope then return nil, 'provider owner scope required' end + if self._scope == nil then + local ok, serr = mgr:start(scope) + if ok ~= true then return nil, serr end + self._scope = scope + end + return mgr, nil +end + + +function Provider:_emit_observed(ev) + if not self.cap_emit_ch then return true, nil end + local payload, err = hal_types.new.Emit('network-state', 'main', 'event', 'observed', ev) + if not payload then return nil, err end + self.cap_emit_ch:put(payload) + return true, nil +end + +function Provider:_snapshot_for_observer(subject, trigger) + local observed, packages_or_err = build_observed_snapshot(self, { live = true }, subject, trigger) + if not observed then + return { + ok = false, + backend = 'openwrt', + err = tostring(packages_or_err), + subject = subject, + trigger = trigger, + } + end + return { + ok = true, + backend = 'openwrt', + observed = observed, + subject = subject, + trigger = trigger, + packages = packages_or_err, + } +end + +function Provider:_ensure_observer() + if self._observer then return self._observer, nil end + local scope = self:_owner() + if not scope then return nil, 'provider owner scope required' end + local hooks_ok, hooks_err, hooks = install_observer_hooks(self) + log_provider(self, hooks_ok == true and 'info' or 'warn', { + what = 'openwrt_network_observer_hooks_installed', + summary = hooks_ok == true + and ('network observer hooks installed sender=' .. tostring(hotplug_sender_path(self.config)) .. ' socket=' .. tostring(observer_socket_path(self.config))) + or ('network observer hook installation failed: ' .. tostring(hooks_err)), + ok = hooks_ok == true, + err = hooks_err, + hooks = hooks, + sender = hotplug_sender_path(self.config), + socket = observer_socket_path(self.config), + }) + if hooks_ok ~= true then return nil, hooks_err end + local obs, err = observer_mod.new({ + logger = self.logger, + socket_path = observer_socket_path(self.config), + debounce_s = self.config.observer_debounce_s or self.config.debounce_observation_s or 0.15, + enable_socket = self.config.enable_hotplug_socket ~= false, + enable_ubus = self.config.enable_ubus_listener == true, + initial_snapshot = self.config.initial_observation_snapshot ~= false, + snapshot = function(subject, trigger) return self:_snapshot_for_observer(subject, trigger) end, + emit = function(ev) return self:_emit_observed(ev) end, + }) + if not obs then return nil, err end + local ok, serr = obs:start(scope) + if ok ~= true then return nil, serr end + self._observer = obs + return obs, nil +end + +function Provider:validate_op(req) + local intent = req and (req.intent or req.desired or req) + return op.always(validate_intent(intent)) +end + +function Provider:plan_op(req) + local intent = req and (req.intent or req.desired or req) + local valid = validate_intent(intent) + if not valid or valid.ok ~= true then return op.always(valid) end + + local built = build_uci_plan(intent, self.config) + local names = built.name_ctx:snapshot() + local shaping = build_shaping_request(intent, self.config, built.name_ctx) + local domains = { + vlan = { status = 'implemented' }, + shaping = { status = shaping.enabled and 'implemented' or 'not_configured' }, + multiwan = { status = (built.mwan_plan and built.mwan_plan.enabled) and 'implemented' or 'not_configured' }, + vpn = { status = next((intent.vpn and intent.vpn.tunnels) or {}) and 'unsupported' or 'not_configured' }, + } + return op.always({ + ok = true, + backend = 'openwrt', + plan = { + domains = domains, + packages = built.counts, + openwrt_names = names, + raw_changes = built.changes, + }, + openwrt_names = names, + }) +end + +function Provider:apply_op(req) + return fibers.run_scope_op(function() + local t0 = fibers.now() + local opts = is_plain_table(req and req.opts) and req.opts or {} + local trace = { + generation = opts.generation, + apply_id = opts.apply_id, + } + log_provider(self, 'debug', { + what = 'openwrt_apply_begin', + generation = trace.generation, + apply_id = trace.apply_id, + }) + + if self.terminated then + log_provider(self, 'warn', { + what = 'openwrt_apply_done', + ok = false, + err = 'provider terminated', + elapsed_ms = elapsed_ms(t0), + generation = trace.generation, + apply_id = trace.apply_id, + }) + return { ok = false, err = 'provider terminated', backend = 'openwrt' } + end + local intent = req and (req.intent or req.desired or req) + local valid = validate_intent(intent) + if not valid or valid.ok ~= true then + log_provider(self, 'warn', { + what = 'openwrt_apply_validate_failed', + err = valid and valid.err or 'invalid intent', + elapsed_ms = elapsed_ms(t0), + generation = trace.generation, + apply_id = trace.apply_id, + }) + return valid + end + trace.rev = intent.rev + + local phase = fibers.now() + local mgr, merr = self:_ensure_started() + log_provider(self, mgr and 'debug' or 'warn', { + what = 'openwrt_apply_manager_ready', + ok = mgr ~= nil, + err = merr, + elapsed_ms = elapsed_ms(phase), + apply_elapsed_ms = elapsed_ms(t0), + generation = trace.generation, + apply_id = trace.apply_id, + }) + if not mgr then return { ok = false, err = merr or 'uci manager unavailable', backend = 'openwrt' } end + + phase = fibers.now() + local built = build_uci_plan(intent, self.config) + log_provider(self, 'debug', { + what = 'openwrt_apply_built_uci', + elapsed_ms = elapsed_ms(phase), + apply_elapsed_ms = elapsed_ms(t0), + generation = trace.generation, + apply_id = trace.apply_id, + network_changes = built.counts.network.changes, + dhcp_changes = built.counts.dhcp.changes, + firewall_changes = built.counts.firewall.changes, + mwan3_changes = built.counts.mwan3.changes, + network_sections = built.counts.network.sections, + dhcp_sections = built.counts.dhcp.sections, + firewall_sections = built.counts.firewall.sections, + mwan3_sections = built.counts.mwan3.sections, + }) + + local records = transaction_records(built.changes) + local packages = OWNED_PACKAGES + phase = fibers.now() + log_provider(self, 'debug', { + what = 'openwrt_apply_uci_transaction_begin', + generation = trace.generation, + apply_id = trace.apply_id, + packages = packages, + apply_elapsed_ms = elapsed_ms(t0), + }) + local tx_result, admitted = fibers.perform(mgr:transaction_op({ + records = records, + packages = packages, + rollback = true, + }, { + trace = { + what = 'net_apply', + generation = trace.generation, + apply_id = trace.apply_id, + }, + })) + local tx_elapsed = elapsed_ms(phase) + log_provider(self, tx_result and tx_result.ok == true and 'debug' or 'warn', { + what = 'openwrt_apply_uci_transaction_done', + generation = trace.generation, + apply_id = trace.apply_id, + ok = tx_result and tx_result.ok == true, + admitted = admitted, + status = tx_result and tx_result.status or 'missing_result', + err = tx_result and tx_result.err or nil, + failed_step = tx_result and tx_result.failed_step or nil, + failed_config = tx_result and tx_result.failed_config or nil, + activation = tx_result and tx_result.activation or nil, + elapsed_ms = tx_elapsed, + apply_elapsed_ms = elapsed_ms(t0), + }) + if not tx_result or tx_result.ok ~= true then + return { + ok = false, + backend = 'openwrt', + admitted = admitted, + status = tx_result and tx_result.status or 'failed', + err = tx_result and tx_result.err or 'UCI transaction failed', + rollback = tx_result and tx_result.rollback or nil, + failed_step = tx_result and tx_result.failed_step or nil, + failed_config = tx_result and tx_result.failed_config or nil, + changed = tx_result and tx_result.status == 'failed_rollback_failed' or false, + } + end + + self._last_name_ctx = built.name_ctx + self._last_openwrt_names = built.name_ctx:snapshot() + + local shaping_result = nil + local shaping_request = build_shaping_request(intent, self.config, built.name_ctx) + if shaping_request and shaping_request.enabled == true then + phase = fibers.now() + log_provider(self, 'debug', { + what = 'openwrt_apply_shaping_wait_targets_begin', + generation = trace.generation, + apply_id = trace.apply_id, + apply_elapsed_ms = elapsed_ms(t0), + }) + local targets_ok, targets_err = wait_for_shaping_targets(self, shaping_request, { + timeout_s = tonumber(self.config.shaping_target_timeout_s) or 10, + interval_s = tonumber(self.config.shaping_target_poll_s) or 0.1, + }) + log_provider(self, targets_ok == true and 'debug' or 'warn', { + what = 'openwrt_apply_shaping_wait_targets_done', + generation = trace.generation, + apply_id = trace.apply_id, + ok = targets_ok == true, + err = targets_err, + elapsed_ms = elapsed_ms(phase), + apply_elapsed_ms = elapsed_ms(t0), + }) + if targets_ok ~= true then + return { ok = false, err = 'traffic shaping target unavailable: ' .. tostring(targets_err), backend = 'openwrt', partial = true } + end + + phase = fibers.now() + log_provider(self, 'debug', { + what = 'openwrt_apply_shaping_begin', + generation = trace.generation, + apply_id = trace.apply_id, + apply_elapsed_ms = elapsed_ms(t0), + }) + shaping_result = shaper_mod.apply(shaping_request, { run_cmd = self.shaper_run_cmd, run_restore = self.shaper_run_restore }) + log_provider(self, shaping_result and shaping_result.ok == true and 'debug' or 'warn', { + what = 'openwrt_apply_shaping_done', + generation = trace.generation, + apply_id = trace.apply_id, + ok = shaping_result and shaping_result.ok == true, + err = shaping_result and shaping_result.err or nil, + elapsed_ms = elapsed_ms(phase), + apply_elapsed_ms = elapsed_ms(t0), + }) + if not shaping_result or shaping_result.ok ~= true then + return { ok = false, err = 'traffic shaping apply failed: ' .. tostring(shaping_result and shaping_result.err or 'unknown'), backend = 'openwrt', partial = true } + end + end + + local result = { + ok = true, + applied = true, + changed = true, + backend = 'openwrt', + intent_rev = intent.rev, + packages = packages, + transaction = tx_result, + activation = tx_result and tx_result.activation or nil, + multiwan = built.mwan_plan, + openwrt_names = self._last_openwrt_names, + shaping = shaping_result, + } + log_provider(self, 'debug', { + what = 'openwrt_apply_done', + ok = true, + generation = trace.generation, + apply_id = trace.apply_id, + activation = result.activation, + elapsed_ms = elapsed_ms(t0), + }) + trigger_post_apply_observation(self, trace) + return result + end):wrap(function(status, _report, result) + if status ~= 'ok' then + log_provider(self, 'warn', { + what = 'openwrt_apply_done', + ok = false, + err = tostring(result or status), + }) + return { ok = false, err = tostring(result or status), backend = 'openwrt' } + end + return result + end) +end + +local function append_error(list, err) + if err == nil then return end + list[#list + 1] = tostring(err) +end + +local function command_capture(argv) + local cmd = exec.command(exec_spec(argv, { stdout = 'pipe', stderr = 'stdout' })) + local out, st, code, sig, err = perform(cmd:combined_output_op()) + local ok = (st == 'exited' and code == 0) + if ok then return true, out or '', nil end + local detail = err or out or ('status=' .. tostring(st)) + if st == 'exited' then + detail = tostring(detail) .. ' (exit ' .. tostring(code) .. ')' + elseif st == 'signalled' then + detail = tostring(detail) .. ' (signal ' .. tostring(sig) .. ')' + end + return nil, out or '', detail +end + +local function ubus_call(config, object, method, payload) + payload = payload or {} + local encoded = cjson.encode(payload) + if type(encoded) ~= 'string' then return nil, 'ubus payload encode failed' end + + local timeout_s = tonumber(config and config.ubus_timeout_s) or 2 + if timeout_s < 1 then timeout_s = 1 end + local argv = { 'ubus', '-t', tostring(timeout_s), 'call', tostring(object), tostring(method), encoded } + + local ok, out, err = command_capture(argv) + if not ok then return nil, err end + local decoded, derr = cjson.decode(out or '') + if type(decoded) ~= 'table' then return nil, derr or 'ubus JSON decode failed' end + return decoded, nil +end + +local function add_unique(list, value, seen) + if type(value) ~= 'string' or value == '' then return end + seen = seen or {} + if seen[value] then return end + seen[value] = true + list[#list + 1] = value +end + +local function infer_interface_ids(observed, req, subject, trigger) + local out, seen = {}, {} + if type(req) == 'table' and type(req.interfaces) == 'table' then + for i = 1, #req.interfaces do add_unique(out, req.interfaces[i], seen) end + end + local subj_if = type(subject) == 'string' and subject:match('^interface:(.+)$') or nil + add_unique(out, subj_if, seen) + local env = type(trigger) == 'table' and (trigger.env or trigger.payload or trigger) or nil + if type(env) == 'table' then add_unique(out, env.INTERFACE or env.interface, seen) end + for ifid in pairs((observed and observed.interfaces) or {}) do add_unique(out, ifid, seen) end + return out +end + +local function normalise_interface_status(ifid, st) + st = is_plain_table(st) and st or {} + local out = { + id = ifid, + up = st.up, + pending = st.pending, + available = st.available, + autostart = st.autostart, + uptime = st.uptime, + proto = st.proto, + device = st.device, + l3_device = st.l3_device, + ipv4 = {}, + ipv6 = {}, + routes = {}, + data = copy_plain(st.data or {}), + raw = copy_plain(st), + } + for i = 1, #(st.address or {}) do + local a = st.address[i] + if is_plain_table(a) then + out.ipv4[#out.ipv4 + 1] = { address = a.address, mask = a.mask, ptpaddress = a.ptpaddress } + end + end + for i = 1, #(st['ipv6-address'] or st.ipv6_address or {}) do + local a = (st['ipv6-address'] or st.ipv6_address)[i] + if is_plain_table(a) then + out.ipv6[#out.ipv6 + 1] = { address = a.address, mask = a.mask, preferred = a.preferred, valid = a.valid } + end + end + for i = 1, #(st.route or {}) do + local r = st.route[i] + if is_plain_table(r) then + out.routes[#out.routes + 1] = { + interface = ifid, + target = r.target, + mask = r.mask, + nexthop = r.nexthop, + source = r.source, + metric = r.metric, + table = r.table, + proto = r.proto, + } + end + end + return out +end + +local function normalise_device_status(name, st) + st = is_plain_table(st) and st or {} + return { + name = name, + type = st.type, + up = st.up, + link = st.link, + mtu = st.mtu, + macaddr = st.macaddr, + txqueuelen = st.txqueuelen, + statistics = copy_plain(st.statistics or {}), + raw = copy_plain(st), + } +end + +local function duration_seconds(v) + if type(v) == 'number' then return v end + if type(v) ~= 'string' then return nil end + local n = tonumber(v) + if n then return n end + local h, m, s = v:match('^(%d+)h:?%s*(%d+)m:?%s*(%d+)s$') + if h then return (tonumber(h) * 3600) + (tonumber(m) * 60) + tonumber(s) end + h, m, s = v:match('^(%d+):(%d+):(%d+)$') + if h then return (tonumber(h) * 3600) + (tonumber(m) * 60) + tonumber(s) end + m, s = v:match('^(%d+)m:?%s*(%d+)s$') + if m then return (tonumber(m) * 60) + tonumber(s) end + s = v:match('^(%d+)s$') + if s then return tonumber(s) end + return nil +end + +local function normalise_mwan3_status(st, name_ctx) + local out = { + schema = 'devicecode.hal.network.multiwan/1', + available = type(st) == 'table', + backend = 'openwrt', + source = 'mwan3', + observed_at = fibers.now(), + interfaces = {}, + interfaces_by_semantic = {}, + policies = {}, + connected = {}, + raw = copy_plain(st or {}), + } + if not is_plain_table(st) then return out end + local wall_now = os.time() + for ifid, rec in pairs(st.interfaces or {}) do + if is_plain_table(rec) then + local probes = {} + for i = 1, #(rec.track_ip or {}) do + local p = rec.track_ip[i] + if is_plain_table(p) then + probes[#probes + 1] = { + ip = p.ip, + status = p.status, + latency_ms = tonumber(p.latency), + packetloss_pct = tonumber(p.packetloss), + } + end + end + local state = rec.status + if rec.enabled == false then state = 'disabled' end + local online = state == 'online' or (state == nil and rec.online == true) + local semantic_interface = ifid + if name_ctx and type(name_ctx.semantic_for) == 'function' then + semantic_interface = name_ctx:semantic_for('mwan_iface', ifid) or ifid + end + local online_for = duration_seconds(rec.online) + local offline_for = duration_seconds(rec.offline) + local item = { + interface = ifid, + semantic_interface = semantic_interface, + state = state, + status = state, + backend = 'openwrt', + tool = 'mwan3', + mwan3_status = rec.status, -- compatibility/diagnostics only + enabled = rec.enabled, + running = rec.running, + tracking = rec.tracking, + up = rec.up, + usable = online, + age_s = tonumber(rec.age), + uptime_s = tonumber(rec.uptime), + age = tonumber(rec.age), -- compatibility + uptime = tonumber(rec.uptime), -- compatibility + online = online, + online_for = online_for, + online_since = online and online_for and (wall_now - online_for) or nil, + online_count = online_for, + offline_for = offline_for, + offline_since = (not online) and offline_for and (wall_now - offline_for) or nil, + offline = offline_for, + score = tonumber(rec.score), + lost = tonumber(rec.lost), + turn = tonumber(rec.turn), + probes = probes, + raw = copy_plain(rec), + } + out.interfaces[ifid] = item + if item.semantic_interface then out.interfaces_by_semantic[item.semantic_interface] = item end + end + end + out.policies = copy_plain(st.policies or {}) + out.connected = copy_plain(st.connected or {}) + return out +end + +local function augment_with_live_snapshot(config, observed, req, subject, trigger, name_ctx) + observed.live = observed.live or { interfaces = {}, devices = {}, routes = {}, errors = {} } + local live = observed.live + local ifaces = infer_interface_ids(observed, req, subject, trigger) + local devices, device_seen = {}, {} + for i = 1, #ifaces do + local ifid = ifaces[i] + local st, err = ubus_call(config, 'network.interface.' .. tostring(ifid), 'status', {}) + if st then + local norm = normalise_interface_status(ifid, st) + live.interfaces[ifid] = norm + if observed.interfaces[ifid] then observed.interfaces[ifid].live = norm end + add_unique(devices, norm.device, device_seen) + add_unique(devices, norm.l3_device, device_seen) + for j = 1, #norm.routes do live.routes[#live.routes + 1] = norm.routes[j] end + else + append_error(live.errors, 'network.interface.' .. tostring(ifid) .. ' status: ' .. tostring(err)) + end + end + for _, iface in pairs(observed.interfaces or {}) do + if iface.endpoint then add_unique(devices, iface.endpoint.ifname or iface.endpoint.device or iface.endpoint.name, device_seen) end + add_unique(devices, iface.device, device_seen) + end + for i = 1, #devices do + local dev = devices[i] + local st, err = ubus_call(config, 'network.device', 'status', { name = dev }) + if st then + live.devices[dev] = normalise_device_status(dev, st) + else + append_error(live.errors, 'network.device status ' .. tostring(dev) .. ': ' .. tostring(err)) + end + end + local mwan, merr = ubus_call(config, 'mwan3', 'status', {}) + if mwan then + observed.multiwan = normalise_mwan3_status(mwan, name_ctx) + else + observed.multiwan = observed.multiwan or { available = false, interfaces = {}, policies = {}, connected = {} } + observed.multiwan.available = false + observed.multiwan.err = tostring(merr) + append_error(live.errors, 'mwan3 status: ' .. tostring(merr)) + end + return observed +end + +function build_observed_snapshot(self, req, subject, trigger) + req = req or {} + local packages, err = read_uci_packages(self.config) + if not packages then return nil, err end + local observed = snapshot_from_packages(packages) + if req.live == true and self.config.enable_live_snapshot ~= false then + augment_with_live_snapshot(self.config, observed, req, subject, trigger, self._last_name_ctx) + end + return observed, packages +end + +function read_uci_packages(config) + local ok, uci_or_err = pcall(require, 'uci') + if not ok or not uci_or_err or type(uci_or_err.cursor) ~= 'function' then + return nil, 'uci module unavailable' + end + local c = uci_or_err.cursor(config.confdir or config.uci_confdir or '/etc/config', config.savedir or config.uci_savedir) + local out = {} + for _, pkg in ipairs(OWNED_PACKAGES) do + if type(c.load) == 'function' then pcall(function() c:load(pkg) end) end + out[pkg] = type(c.get_all) == 'function' and c:get_all(pkg) or {} + end + return out, nil +end + +function snapshot_from_packages(packages) + local network = packages.network or {} + local dhcp = packages.dhcp or {} + local firewall = packages.firewall or {} + local mwan3 = packages.mwan3 or {} + + local observed = { + schema = 'devicecode.net.observed/1', + segments = {}, + interfaces = {}, + dns = { enabled = true, upstreams = {}, records = {}, host_files = {} }, + dhcp = { pools = {} }, + firewall = { defaults = {}, zones = {}, policies = {}, rules = {} }, + routing = { routes = {} }, + multiwan = { config = { interfaces = {}, members = {}, policies = {}, rules = {} } }, + shaping = { applied = nil }, + } + + local bridge_by_name = {} + for _, secname in ipairs(sorted_keys(network)) do + local sec = network[secname] + if is_plain_table(sec) and sec['.type'] == 'device' and sec.type == 'bridge' and type(sec.name) == 'string' then + bridge_by_name[sec.name] = { + section = secname, + name = sec.name, + members = list_from_uci(sec.ports), + } + end + end + + for _, secname in ipairs(sorted_keys(network)) do + local sec = network[secname] + if is_plain_table(sec) and sec['.type'] == 'interface' then + local ifid = secname + local iface = { + id = ifid, + segment = ifid, + role = sec.proto == 'dhcp' and 'wan' or 'lan', + enabled = sec.disabled ~= '1' and sec.auto ~= '0', + kind = 'ethernet', + addressing = { ipv4 = { mode = sec.proto } }, + } + if type(sec.device) == 'string' and bridge_by_name[sec.device] then + iface.kind = 'bridge' + iface.members = bridge_by_name[sec.device].members + else + iface.endpoint = { ifname = sec.device } + end + if sec.proto == 'static' then + iface.addressing.ipv4.cidr = cidr_from_addr_netmask(sec.ipaddr, sec.netmask) + iface.addressing.ipv4.address = sec.ipaddr + iface.addressing.ipv4.netmask = sec.netmask + iface.addressing.ipv4.gateway = sec.gateway + elseif sec.proto == 'dhcp' then + iface.addressing.ipv4.peerdns = bool_from_uci(sec.peerdns) + iface.addressing.ipv4.metric = tonumber(sec.metric) + end + observed.interfaces[ifid] = iface + observed.segments[ifid] = observed.segments[ifid] or { id = ifid, interfaces = {} } + observed.segments[ifid].interfaces[#observed.segments[ifid].interfaces + 1] = ifid + end + end + + for _, secname in ipairs(sorted_keys(network)) do + local sec = network[secname] + if is_plain_table(sec) and sec['.type'] == 'route' then + observed.routing.routes[#observed.routing.routes + 1] = { + id = secname, + interface = sec.interface, + target = sec.target, + gateway = sec.gateway, + } + end + end + + for _, secname in ipairs(sorted_keys(dhcp)) do + local sec = dhcp[secname] + if is_plain_table(sec) and sec['.type'] == 'dnsmasq' then + local interfaces = list_from_uci(sec.interface) + local upstreams = list_from_uci(sec.server) + for i = 1, #upstreams do observed.dns.upstreams[#observed.dns.upstreams + 1] = upstreams[i] end + observed.dns.cache_size = tonumber(sec.cachesize) or sec.cachesize + observed.dns.host_files[secname] = { interfaces = interfaces, addnhosts = list_from_uci(sec.addnhosts), addnmount = list_from_uci(sec.addnmount) } + for _, addr in ipairs(list_from_uci(sec.address)) do + local name, ip = tostring(addr):match('^/([^/]+)/([^/]+)$') + if name and ip then observed.dns.records[name] = { name = name, address = ip, source = secname, interfaces = interfaces } end + end + elseif is_plain_table(sec) and sec['.type'] == 'dhcp' then + local seg_id = sec.interface or secname + local enabled = sec.ignore ~= '1' + observed.dhcp.pools[seg_id] = { + segment = seg_id, + enabled = enabled, + start = tonumber(sec.start) or sec.start, + limit = tonumber(sec.limit) or sec.limit, + leasetime = sec.leasetime, + } + observed.segments[seg_id] = observed.segments[seg_id] or { id = seg_id, interfaces = {} } + observed.segments[seg_id].dhcp = observed.dhcp.pools[seg_id] + end + end + + for _, secname in ipairs(sorted_keys(firewall)) do + local sec = firewall[secname] + if is_plain_table(sec) and sec['.type'] == 'defaults' then + observed.firewall.defaults = copy_plain(sec) + elseif is_plain_table(sec) and sec['.type'] == 'zone' then + local zname = sec.name or secname + local networks = list_from_uci(sec.network) + observed.firewall.zones[zname] = { + name = zname, + networks = networks, + input = sec.input, + output = sec.output, + forward = sec.forward, + masq = bool_from_uci(sec.masq), + mtu_fix = bool_from_uci(sec.mtu_fix), + } + for i = 1, #networks do + local seg_id = networks[i] + observed.segments[seg_id] = observed.segments[seg_id] or { id = seg_id, interfaces = {} } + observed.segments[seg_id].firewall = observed.segments[seg_id].firewall or {} + observed.segments[seg_id].firewall.zone = zname + end + elseif is_plain_table(sec) and sec['.type'] == 'forwarding' then + observed.firewall.policies[secname] = { + id = secname, + src = sec.src, + dest = sec.dest, + } + elseif is_plain_table(sec) and sec['.type'] == 'rule' then + observed.firewall.rules[secname] = copy_plain(sec) + end + end + + + for _, secname in ipairs(sorted_keys(mwan3)) do + local sec = mwan3[secname] + if is_plain_table(sec) and sec['.type'] == 'interface' then + observed.multiwan.config.interfaces[secname] = { + name = secname, + enabled = bool_from_uci(sec.enabled), + family = sec.family, + track_ip = list_from_uci(sec.track_ip), + } + elseif is_plain_table(sec) and sec['.type'] == 'member' then + observed.multiwan.config.members[secname] = { + name = secname, + interface = sec.interface, + metric = tonumber(sec.metric), + weight = tonumber(sec.weight), + } + elseif is_plain_table(sec) and sec['.type'] == 'policy' then + observed.multiwan.config.policies[secname] = { + name = secname, + use_member = list_from_uci(sec.use_member), + last_resort = sec.last_resort, + } + elseif is_plain_table(sec) and sec['.type'] == 'rule' then + observed.multiwan.config.rules[secname] = copy_plain(sec) + end + end + + for _, seg in pairs(observed.segments) do + table.sort(seg.interfaces) + end + + return observed +end + + +function Provider:watch_op(_req) + -- Starting a watch installs long-lived observer workers into the caller's + -- current scope. Do not create a private operation-scope boundary + -- here: it would be joined as soon as watch_op returned, cancelling the + -- socket server and ubus listener. + return op.guard(function() + if self.terminated then + return op.always({ ok = false, err = 'provider terminated', backend = 'openwrt' }) + end + local _, merr = self:_ensure_started() + if merr then + return op.always({ ok = false, err = merr, backend = 'openwrt' }) + end + local obs, oerr = self:_ensure_observer() + if not obs then + return op.always({ ok = false, err = oerr or 'observer unavailable', backend = 'openwrt' }) + end + return op.always({ + ok = true, + backend = 'openwrt', + watching = true, + socket_path = obs.socket_path, + }) + end) +end + +function Provider:ingest_observation(trigger) + local obs, err = self:_ensure_observer() + if not obs then return nil, err end + return obs:ingest(trigger) +end + +function Provider:snapshot_op(req) + return fibers.run_scope_op(function() + local observed, packages_or_err = build_observed_snapshot(self, req or {}, nil, { source = 'snapshot', action = 'manual' }) + if not observed then + return { ok = false, err = packages_or_err, backend = 'openwrt' } + end + return { + ok = true, + backend = 'openwrt', + observed = observed, + packages = copy_plain(packages_or_err), + } + end):wrap(function(status, _report, result) + if status ~= 'ok' then return { ok = false, err = tostring(result or status), backend = 'openwrt' } end + return result + end) +end + +function Provider:probe_link_op(_req) + return op.always({ ok = false, err = 'openwrt network provider probe_link not implemented', backend = 'openwrt' }) +end + +local function requested_counter_interfaces(req, observed) + local out = {} + local seen = {} + local function add(v) + if type(v) ~= 'string' or v == '' or seen[v] then return end + seen[v] = true + out[#out + 1] = v + end + if type(req) == 'table' then + add(req.interface) + local interfaces = req.interfaces or req.ids + if type(interfaces) == 'table' then + if #interfaces > 0 then + for i = 1, #interfaces do add(interfaces[i]) end + else + for _, id in ipairs(sorted_keys(interfaces)) do add(id) end + end + end + end + if #out == 0 then + for _, id in ipairs(sorted_keys(observed and observed.interfaces)) do add(id) end + end + return out +end + +local function counter_stats(req) + local stats = req and req.stats + if type(stats) ~= 'table' then return COUNTER_STATS end + local out, seen = {}, {} + local function add(stat) + if type(stat) ~= 'string' or stat == '' or seen[stat] then return end + seen[stat] = true + out[#out + 1] = stat + end + if #stats > 0 then + for i = 1, #stats do add(stats[i]) end + else + for _, stat in ipairs(sorted_keys(stats)) do add(stat) end + end + return #out > 0 and out or COUNTER_STATS +end + +local function translated_counter_interface(self, semantic) + local names = self._last_openwrt_names and self._last_openwrt_names.names or nil + local logical = names and names.logical_interface or nil + return (logical and logical[semantic]) or semantic +end + +local function req_device(req, semantic, generated) + local devices = req and req.devices + if type(devices) == 'table' then + local dev = devices[semantic] or devices[generated] + if type(dev) == 'string' and dev ~= '' then return dev end + end + if req and type(req.device) == 'string' and req.device ~= '' then return req.device end + return nil +end + +local function resolve_counter_device(observed, req, semantic, generated) + local explicit = req_device(req, semantic, generated) + if explicit then return explicit end + observed = observed or {} + local live = observed.live or {} + local iface = (observed.interfaces or {})[generated] or (observed.interfaces or {})[semantic] or {} + local live_iface = iface.live or (live.interfaces or {})[generated] or (live.interfaces or {})[semantic] or {} + local endpoint = iface.endpoint or {} + local dev = live_iface.l3_device or live_iface.device or endpoint.ifname or endpoint.device or endpoint.name or iface.device + if type(dev) == 'string' and dev ~= '' then return dev end + return nil +end + +local function read_counter_file(path) + local f, err = file.open(path, 'r') + if not f then return nil, err end + local data, rerr = f:read_all() + f:close() + if rerr ~= nil then return nil, rerr end + local n = tonumber((tostring(data or ''):gsub('%s+', ''))) + if not n then return nil, 'counter parse failed' end + return n, nil +end + +local function read_counter(self, device, stat) + local reader = self.counter_reader or self.config.counter_reader or self.config.stat_reader + if type(reader) == 'function' then return reader(device, stat) end + return read_counter_file('/sys/class/net/' .. tostring(device) .. '/statistics/' .. tostring(stat)) +end + +function Provider:read_counters_op(req) + return fibers.run_scope_op(function() + if self.terminated then return { ok = false, err = 'provider terminated', backend = 'openwrt' } end + req = req or {} + local observed = nil + if type(req.devices) ~= 'table' then + local snapshot, snap_err = build_observed_snapshot(self, { live = true }, nil, { source = 'diagnostics', action = 'read_counters' }) + if snapshot then observed = snapshot else observed = { interfaces = {}, live = {}, errors = { tostring(snap_err) } } end + end + + local result = { ok = true, backend = 'openwrt', counters = {}, errors = {} } + local stats = counter_stats(req) + for _, semantic in ipairs(requested_counter_interfaces(req, observed)) do + local generated = translated_counter_interface(self, semantic) + local dev = resolve_counter_device(observed, req, semantic, generated) + if not dev then + result.errors[semantic] = { device = 'unavailable' } + else + local rec = { interface = semantic, openwrt_interface = generated, device = dev, statistics = {} } + for _, stat in ipairs(stats) do + local value, err = read_counter(self, dev, stat) + if value ~= nil then + rec.statistics[stat] = tonumber(value) + else + result.errors[semantic] = result.errors[semantic] or { device = dev } + result.errors[semantic][stat] = tostring(err or 'counter unavailable') + end + end + if next(rec.statistics) ~= nil then result.counters[semantic] = rec end + end + end + if observed and observed.errors and next(observed.errors) ~= nil then result.snapshot_errors = copy_plain(observed.errors) end + return result + end):wrap(function(status, _report, result) + if status ~= 'ok' then return { ok = false, err = tostring(result or status), backend = 'openwrt' } end + return result + end) +end + + + +local function translate_mwan_policy_for_ctx(name_ctx, policy) + if not name_ctx or type(name_ctx.mwan_policy) ~= 'function' then return policy end + local p = policy or 'balanced' + if type(p) == 'string' and type(name_ctx.semantic_for) == 'function' and name_ctx:semantic_for('mwan_policy', p) then + return p + end + return name_ctx:mwan_policy(p) +end + +local function translate_mwan_iface_for_ctx(name_ctx, iface) + if not name_ctx or type(name_ctx.mwan_iface) ~= 'function' then return iface end + if type(iface) ~= 'string' or iface == '' then return iface end + if type(name_ctx.semantic_for) == 'function' and name_ctx:semantic_for('mwan_iface', iface) then + return iface + end + return name_ctx:mwan_iface(iface) +end + +local function translate_live_weights_req(req, name_ctx) + if not name_ctx then return req or {} end + local out = copy_plain(req or {}) or {} + out.policy = translate_mwan_policy_for_ctx(name_ctx, out.policy or 'balanced') + local members = {} + for i, m in ipairs((req and req.members) or {}) do + if is_plain_table(m) then + local mm = copy_plain(m) or {} + local semantic_iface = m.interface or m.id + local generated_iface = translate_mwan_iface_for_ctx(name_ctx, semantic_iface) + if type(generated_iface) == 'string' and generated_iface ~= '' then + mm.semantic_interface = semantic_iface + mm.interface = generated_iface + end + members[#members + 1] = mm + else + members[#members + 1] = m + end + end + out.members = members + return out +end + +local function translate_speedtest_req(req, name_ctx) + if not name_ctx then return req or {} end + local out = copy_plain(req or {}) or {} + local semantic_iface = out.interface + local generated_iface = translate_mwan_iface_for_ctx(name_ctx, semantic_iface) + if type(generated_iface) == 'string' and generated_iface ~= '' then + out.semantic_interface = semantic_iface + out.interface = generated_iface + end + local dev = out.device or out.linux_interface or out.ifname + if type(name_ctx.vlan) == 'function' + and type(semantic_iface) == 'string' + and semantic_iface ~= '' + and (dev == nil or dev == '' or dev == semantic_iface or dev == generated_iface) then + out.semantic_device = dev + out.device = name_ctx:vlan(semantic_iface) + end + return out +end + +function Provider:apply_live_weights_op(req) + return fibers.run_scope_op(function() + if self.terminated then return { ok = false, err = 'provider terminated', backend = 'openwrt' } end + local original_req = req or {} + local live_req = translate_live_weights_req(original_req, self._last_name_ctx) + local result = mwan3_mod.apply_live_weights(live_req, { + apply_mwan_live_weights = self.apply_mwan_live_weights, + run_cmd = self.mwan_run_cmd, + run_cmd_capture = self.mwan_run_cmd_capture, + run_restore = self.mwan_run_restore, + }) + local persist = req and req.persist ~= false + if persist then + local mgr, merr = self:_ensure_started() + if mgr then + local ok, err, admitted = fibers.perform(mwan3_mod.persist_weights_op(mgr, original_req, self._last_name_ctx)) + result.persisted = ok == true + result.persist_err = err + result.persist_admitted = admitted + else + result.persisted = false + result.persist_err = merr + end + end + return result + end):wrap(function(status, _report, result) + if status ~= 'ok' then return { ok = false, err = tostring(result or status), backend = 'openwrt' } end + return result + end) +end + +function Provider:apply_shaping_op(req) + return fibers.run_scope_op(function() + if self.terminated then return { ok = false, err = 'provider terminated', backend = 'openwrt' } end + local result = shaper_mod.apply(req or {}, { run_cmd = self.shaper_run_cmd, run_restore = self.shaper_run_restore }) + if not result then return { ok = false, err = 'traffic shaping apply failed', backend = 'openwrt' } end + return result + end):wrap(function(status, _report, result) + if status ~= 'ok' then return { ok = false, err = tostring(result or status), backend = 'openwrt' } end + return result + end) +end + +function Provider:speedtest_op(req) + return speedtest_mod.run_op(translate_speedtest_req(req or {}, self._last_name_ctx), { run_cmd = self.speedtest_run_cmd }) +end + +function Provider:terminate(reason) + if self._observer and type(self._observer.terminate) == 'function' then + self._observer:terminate(reason or 'provider terminated') + end + self._observer = nil + self.terminated = reason or true + if self._uci_manager and type(self._uci_manager.terminate) == 'function' then + self._uci_manager:terminate(reason or 'terminated') + end + return true, nil +end + +M._test = { + normalise_mwan3_status = normalise_mwan3_status, + observer_forward_script = observer_forward_script, + mwan3_user_block = mwan3_user_block, + strip_observer_block = strip_observer_block, + install_observer_hooks = install_observer_hooks, +} + +return M diff --git a/src/services/hal/backends/network/providers/openwrt/mwan3.lua b/src/services/hal/backends/network/providers/openwrt/mwan3.lua new file mode 100644 index 00000000..02445354 --- /dev/null +++ b/src/services/hal/backends/network/providers/openwrt/mwan3.lua @@ -0,0 +1,447 @@ +-- services/hal/backends/network/providers/openwrt/mwan3.lua +-- MWAN3 UCI and live-weight helpers for the OpenWrt network provider. + +local M = {} + +local function exec_spec(argv, opts) + opts = opts or {} + local spec = {} + for i = 1, #argv do spec[i] = argv[i] end + spec.stdin = opts.stdin ~= nil and opts.stdin or 'null' + spec.stdout = opts.stdout ~= nil and opts.stdout or 'null' + spec.stderr = opts.stderr ~= nil and opts.stderr or 'null' + if opts.shutdown_grace ~= nil then spec.shutdown_grace = opts.shutdown_grace end + return spec +end + +local function is_plain_table(v) return type(v) == 'table' and getmetatable(v) == nil end +local function sorted_keys(t) + local ks = {} + for k in pairs(t or {}) do ks[#ks + 1] = k end + table.sort(ks, function(a, b) return tostring(a) < tostring(b) end) + return ks +end +local function sid(v) + local s = tostring(v or ''):gsub('[^%w_]', '_') + if s == '' then s = '_' end + return s +end +local function set_section(changes, config, section, stype) + changes[#changes + 1] = { op = 'set', config = config, section = section, option = stype } +end +local function set_option(changes, config, section, option, value) + if value == nil then return end + changes[#changes + 1] = { op = 'set', config = config, section = section, option = option, value = value } +end +local function set_list_option(changes, config, section, option, value) + if value == nil then return end + local values = value + if type(values) ~= 'table' then values = { values } end + for i = 1, #values do + if values[i] ~= nil then + changes[#changes + 1] = { op = 'add_list', config = config, section = section, option = option, value = values[i] } + end + end +end + +local function bool_uci(v) if v == nil then return nil end return v and '1' or '0' end +local function sticky_uci(v) if v == nil then return nil end return v and '1' or '0' end + +local function policy_name_for(wan, fallback) + local lb = is_plain_table(wan and wan.load_balancing) and wan.load_balancing or {} + return lb.policy or fallback or 'balanced' +end + +local function member_iface(id, spec) + return spec.interface or id +end + +function M.build_changes(intent, name_ctx) + local wan = is_plain_table(intent and intent.wan) and intent.wan or {} + local members = is_plain_table(wan.members) and wan.members or {} + local changes, known = {}, {} + if wan.enabled == false or next(members) == nil then + return changes, known, { enabled = false } + end + if not name_ctx then + local ok_names, names_mod = pcall(require, 'services.hal.backends.network.providers.openwrt.names') + if ok_names and names_mod and type(names_mod.allocate) == 'function' then + name_ctx = names_mod.allocate(intent) + end + end + local function mw_iface(id) + if name_ctx and type(name_ctx.mwan_iface) == 'function' then return name_ctx:mwan_iface(id) end + return sid(id) + end + local function mw_member(id) + if name_ctx and type(name_ctx.mwan_member) == 'function' then return name_ctx:mwan_member(id) end + return sid(id) + end + local function mw_policy(id) + if name_ctx and type(name_ctx.mwan_policy) == 'function' then return name_ctx:mwan_policy(id) end + return sid(id) + end + local function mw_rule(id) + if name_ctx and type(name_ctx.mwan_rule) == 'function' then return name_ctx:mwan_rule(id) end + return sid(id) + end + + set_section(changes, 'mwan3', 'globals', 'globals') + known.globals = true + set_option(changes, 'mwan3', 'globals', 'mmx_mask', (wan.runtime and wan.runtime.mmx_mask) or '0x3F00') + set_option(changes, 'mwan3', 'globals', 'logging', bool_uci(wan.runtime and wan.runtime.logging)) + + local health = is_plain_table(wan.health) and wan.health or {} + local member_sections = {} + for _, mid in ipairs(sorted_keys(members)) do + local m = is_plain_table(members[mid]) and members[mid] or {} + local iface_sem = member_iface(mid, m) + local metric = math.floor(tonumber(m.metric or m.mwan_metric or 1) or 1) + local weight = math.max(1, math.floor(tonumber(m.weight or 1) or 1)) + local ifsec = mw_iface(iface_sem) + known[ifsec] = true + set_section(changes, 'mwan3', ifsec, 'interface') + set_option(changes, 'mwan3', ifsec, 'enabled', '1') + set_option(changes, 'mwan3', ifsec, 'family', m.family or health.family or 'ipv4') + set_list_option(changes, 'mwan3', ifsec, 'track_ip', m.track_ip or (m.health and m.health.track_ip) or health.track_ip) + set_option(changes, 'mwan3', ifsec, 'initial_state', m.initial_state or health.initial_state or 'offline') + set_option(changes, 'mwan3', ifsec, 'reliability', m.reliability or health.reliability) + set_option(changes, 'mwan3', ifsec, 'count', m.count or health.count) + set_option(changes, 'mwan3', ifsec, 'timeout', m.timeout or health.timeout) + set_option(changes, 'mwan3', ifsec, 'interval', m.interval or health.interval) + set_option(changes, 'mwan3', ifsec, 'up', m.up or health.up) + set_option(changes, 'mwan3', ifsec, 'down', m.down or health.down) + local member_sec = mw_member(mid) + known[member_sec] = true + set_section(changes, 'mwan3', member_sec, 'member') + set_option(changes, 'mwan3', member_sec, 'interface', ifsec) + set_option(changes, 'mwan3', member_sec, 'metric', metric) + set_option(changes, 'mwan3', member_sec, 'weight', weight) + member_sections[#member_sections + 1] = member_sec + end + table.sort(member_sections) + local policy_name = mw_policy(policy_name_for(wan, 'balanced')) + known[policy_name] = true + set_section(changes, 'mwan3', policy_name, 'policy') + set_list_option(changes, 'mwan3', policy_name, 'use_member', member_sections) + set_option(changes, 'mwan3', policy_name, 'last_resort', wan.last_resort or 'unreachable') + + local rule_sections = {} + for _, rid in ipairs(sorted_keys(wan.rules or {})) do + local r = is_plain_table(wan.rules[rid]) and wan.rules[rid] or {} + if r.enabled ~= false and r.disabled ~= true then + local rsec = mw_rule(rid) + known[rsec] = true + set_section(changes, 'mwan3', rsec, 'rule') + set_option(changes, 'mwan3', rsec, 'src_ip', r.src_ip) + set_option(changes, 'mwan3', rsec, 'dest_ip', r.dest_ip) + set_option(changes, 'mwan3', rsec, 'proto', r.proto) + set_option(changes, 'mwan3', rsec, 'src_port', r.src_port) + set_option(changes, 'mwan3', rsec, 'dest_port', r.dest_port) + set_option(changes, 'mwan3', rsec, 'family', r.family or 'ipv4') + set_option(changes, 'mwan3', rsec, 'sticky', sticky_uci(r.sticky)) + set_option(changes, 'mwan3', rsec, 'timeout', r.timeout) + set_option(changes, 'mwan3', rsec, 'use_policy', mw_policy(r.policy or policy_name_for(wan, 'balanced'))) + rule_sections[#rule_sections + 1] = rsec + end + end + + local rule_name = mw_rule('default_rule_v4') + known[rule_name] = true + set_section(changes, 'mwan3', rule_name, 'rule') + set_option(changes, 'mwan3', rule_name, 'dest_ip', '0.0.0.0/0') + set_option(changes, 'mwan3', rule_name, 'family', 'ipv4') + set_option(changes, 'mwan3', rule_name, 'use_policy', policy_name) + rule_sections[#rule_sections + 1] = rule_name + return changes, known, { enabled = true, policy = policy_name, members = member_sections, rules = rule_sections } +end + +function M.build_weight_only_changes(req, name_ctx) + local changes = {} + local members = req and req.members or {} + local function mw_member(id) + if name_ctx and type(name_ctx.mwan_member) == 'function' then return name_ctx:mwan_member(id) end + return sid(id) + end + for i = 1, #members do + local m = is_plain_table(members[i]) and members[i] or {} + local id = m.id or m.member or m.interface or ('member' .. tostring(i)) + local member_sec = mw_member(id) + local weight = tonumber(m.weight or m.live_weight) + if weight ~= nil then set_option(changes, 'mwan3', member_sec, 'weight', math.max(1, math.floor(weight))) end + local metric = tonumber(m.metric or m.mwan_metric) + if metric ~= nil then set_option(changes, 'mwan3', member_sec, 'metric', math.max(1, math.floor(metric))) end + end + return changes +end + +function M.persist_weights_op(mgr, req, name_ctx) + local changes = M.build_weight_only_changes(req, name_ctx) + return mgr:submit_op({ config = 'mwan3', changes = changes, restart_cmds = {} }) +end + +local function default_capture(argv) + local fibers = require 'fibers' + local exec = require 'fibers.io.exec' + local cmd = exec.command(exec_spec(argv, { stdout = 'pipe', stderr = 'stdout' })) + local out, st, code, sig, err = fibers.perform(cmd:combined_output_op()) + local ok = (st == 'exited' and code == 0) + if ok then return true, out or '', nil end + local detail = err or out or ('status=' .. tostring(st)) + if st == 'exited' then + detail = tostring(detail) .. ' (exit ' .. tostring(code) .. ')' + elseif st == 'signalled' then + detail = tostring(detail) .. ' (signal ' .. tostring(sig) .. ')' + end + return nil, out or '', detail +end + + +local function default_restore(content) + local fibers = require 'fibers' + local exec = require 'fibers.io.exec' + local cmd = exec.command(exec_spec({ 'iptables-restore', '--noflush' }, { + stdin = 'pipe', + stdout = 'pipe', + stderr = 'stdout', + shutdown_grace = 0.5, + })) + local stdin, serr = cmd:stdin_stream() + if not stdin then return nil, serr or 'failed to open iptables-restore stdin' end + local ok, werr = stdin:write(content or '') + if not ok then + pcall(function() stdin:terminate('iptables-restore write failed') end) + fibers.perform(cmd:shutdown_op(0.2)) + return nil, 'failed to write iptables-restore input: ' .. tostring(werr) + end + stdin:close() + local out, st, code, sig, err = fibers.perform(cmd:combined_output_op()) + if st == 'exited' and code == 0 then return true, nil, out or '' end + local detail = err or out or ('status=' .. tostring(st)) + if st == 'exited' then + detail = tostring(detail) .. ' (exit ' .. tostring(code) .. ')' + elseif st == 'signalled' then + detail = tostring(detail) .. ' (signal ' .. tostring(sig) .. ')' + end + return nil, detail, out or '' +end + + +local function normalise_policy(policy) + return sid(policy or 'balanced') +end + +local function interface_from_member(m, index) + if type(m) ~= 'table' then return nil end + local iface = m.interface or m.id + if type(iface) ~= 'string' or iface == '' then return nil, 'members[' .. tostring(index) .. '] missing interface' end + return sid(iface), nil +end + +local function explicit_mark(m) + if type(m) ~= 'table' then return nil end + local v = m.mark or m.fwmark or m.xmark + if type(v) == 'number' then return string.format('0x%x', v) end + if type(v) == 'string' and v ~= '' then + local mark = v:match('^(0x%x+)') or v:match('^(%d+)$') + return mark + end + return nil +end + +function M.parse_mangle_ruleset(text) + local parsed = { + mask = nil, + iface_marks = {}, + policy_rules = {}, + chains = {}, + } + for line in tostring(text or ''):gmatch('[^\r\n]+') do + local chain = line:match('^:([^%s]+)') + if chain then parsed.chains[chain] = true end + local achain, rest = line:match('^%-A%s+([^%s]+)%s+(.+)$') + if achain then + local mask = rest:match('%-%-set%-xmark%s+0x%x+/(0x%x+)') or rest:match('%-%-mark%s+0x%x+/(0x%x+)') + if mask and not parsed.mask then parsed.mask = mask end + if achain:match('^mwan3_iface_in_') then + local iface = rest:match('%-%-comment%s+"([^"]+)"%s+%-j%s+MARK') + or rest:match('%-%-comment%s+([^%s]+)%s+%-j%s+MARK') + local mark = rest:match('%-%-set%-xmark%s+(0x%x+)/0x%x+') + if iface and mark and iface ~= 'default' then parsed.iface_marks[sid(iface)] = mark end + elseif achain:match('^mwan3_policy_') then + parsed.policy_rules[achain] = parsed.policy_rules[achain] or {} + parsed.policy_rules[achain][#parsed.policy_rules[achain] + 1] = line + end + end + end + parsed.mask = parsed.mask or '0x3f00' + return parsed +end + +local function normalise_members(members, parsed) + local out, skipped, total, requested_positive = {}, {}, 0, 0 + for i = 1, #(members or {}) do + local m = members[i] + if type(m) == 'table' and m.enabled ~= false and m.disabled ~= true then + local weight = math.floor(tonumber(m.weight or m.live_weight or 0) or 0) + if weight > 0 then + requested_positive = requested_positive + 1 + local iface, ierr = interface_from_member(m, i) + if not iface then return nil, ierr end + local mark = explicit_mark(m) or (parsed and parsed.iface_marks and parsed.iface_marks[iface]) + if not mark then + skipped[#skipped + 1] = { + interface = iface, + semantic_interface = m.semantic_interface, + weight = weight, + metric = math.floor(tonumber(m.metric or 1) or 1), + reason = 'no_mwan3_mark', + } + else + total = total + weight + out[#out + 1] = { + interface = iface, + semantic_interface = m.semantic_interface, + weight = weight, + metric = math.floor(tonumber(m.metric or 1) or 1), + mark = mark, + } + end + end + end + end + if #out == 0 then + if requested_positive > 0 and #skipped > 0 then + return nil, 'no MWAN3 firewall mark found for any live member; no live MWAN3 members have firewall marks' + end + return nil, 'no enabled positive-weight MWAN3 members supplied' + end + return out, nil, total, skipped +end + +local function quote_restore_arg(s) + s = tostring(s or '') + if s:find('[\r\n]') then error('newline in iptables-restore argument') end + if s:find('%s') or s:find('"') then + return '"' .. s:gsub('(["\\])', '\\%1') .. '"' + end + return s +end + +function M.build_policy_rule_restore_line(chain, mask, member, remaining_weight, include_statistic) + local parts = { '-A', chain, '-m', 'mark', '--mark', '0x0/' .. mask } + if include_statistic then + local probability = member.weight / remaining_weight + if probability < 0 then probability = 0 end + if probability > 1 then probability = 1 end + parts[#parts + 1] = '-m'; parts[#parts + 1] = 'statistic' + parts[#parts + 1] = '--mode'; parts[#parts + 1] = 'random' + parts[#parts + 1] = '--probability'; parts[#parts + 1] = string.format('%.11f', probability) + end + parts[#parts + 1] = '-m'; parts[#parts + 1] = 'comment' + parts[#parts + 1] = '--comment'; parts[#parts + 1] = string.format('%s %d %d', member.interface, member.weight, remaining_weight) + parts[#parts + 1] = '-j'; parts[#parts + 1] = 'MARK' + parts[#parts + 1] = '--set-xmark'; parts[#parts + 1] = member.mark .. '/' .. mask + for i = 1, #parts do parts[i] = quote_restore_arg(parts[i]) end + return table.concat(parts, ' ') +end + +-- Kept for tests/diagnostics that prefer argv-like inspection. The live path +-- uses iptables-restore --noflush, not a sequence of iptables mutations. +function M.build_policy_rule_argv(chain, mask, member, remaining_weight, include_statistic) + local argv = { 'iptables', '-t', 'mangle', '-A', chain, '-m', 'mark', '--mark', '0x0/' .. mask } + if include_statistic then + local probability = member.weight / remaining_weight + if probability < 0 then probability = 0 end + if probability > 1 then probability = 1 end + argv[#argv + 1] = '-m'; argv[#argv + 1] = 'statistic' + argv[#argv + 1] = '--mode'; argv[#argv + 1] = 'random' + argv[#argv + 1] = '--probability'; argv[#argv + 1] = string.format('%.11f', probability) + end + argv[#argv + 1] = '-m'; argv[#argv + 1] = 'comment' + argv[#argv + 1] = '--comment'; argv[#argv + 1] = string.format('%s %d %d', member.interface, member.weight, remaining_weight) + argv[#argv + 1] = '-j'; argv[#argv + 1] = 'MARK' + argv[#argv + 1] = '--set-xmark'; argv[#argv + 1] = member.mark .. '/' .. mask + return argv +end + +function M.build_live_weight_restore(req, ruleset_text) + req = req or {} + local parsed = M.parse_mangle_ruleset(ruleset_text or '') + local policy = normalise_policy(req.policy or 'balanced') + local chain = 'mwan3_policy_' .. policy + if not parsed.chains[chain] and not parsed.policy_rules[chain] then + return nil, 'MWAN3 policy chain not found: ' .. chain + end + local members, merr, total, skipped = normalise_members(req.members, parsed) + if not members then return nil, merr end + local mask = tostring(req.mask or req.mmx_mask or parsed.mask or '0x3f00') + local lines = { '*mangle', '-F ' .. chain } + local remaining = total + for i = 1, #members do + local member = members[i] + lines[#lines + 1] = M.build_policy_rule_restore_line(chain, mask, member, remaining, i < #members) + remaining = remaining - member.weight + end + lines[#lines + 1] = 'COMMIT' + lines[#lines + 1] = '' + return table.concat(lines, '\n'), nil, { + policy = policy, + chain = chain, + mask = mask, + members = members, + total_weight = total, + single_member = (#members == 1), + skipped_members = skipped or {}, + skipped_count = #(skipped or {}), + } +end + +function M.build_live_weight_commands(req, ruleset_text) + local restore, err, detail = M.build_live_weight_restore(req, ruleset_text) + if not restore then return nil, err end + local commands = {} + for line in restore:gmatch('[^\n]+') do + if line:sub(1, 3) == '-F ' then + commands[#commands + 1] = { 'iptables', '-t', 'mangle', '-F', line:sub(4) } + elseif line:sub(1, 3) == '-A ' then + -- Diagnostics only; argument splitting is intentionally conservative. + commands[#commands + 1] = { 'iptables-restore-line', line } + end + end + return commands, nil, detail +end + +function M.apply_live_weights(req, opts) + req = req or {}; opts = opts or {} + if type(opts.apply_mwan_live_weights) == 'function' then + local ok, err, detail = opts.apply_mwan_live_weights(req) + return { ok = ok == true, changed = ok == true, applied = ok == true, err = err, detail = detail, backend = 'openwrt' } + end + + local capture = opts.run_cmd_capture or default_capture + local restore_runner = opts.run_restore or default_restore + local ok, ruleset, err = capture({ 'iptables-save', '-t', 'mangle' }) + if ok ~= true then + return { ok = false, err = 'failed to read MWAN3 mangle rules: ' .. tostring(err), backend = 'openwrt', code = 'mwan_live_weights_read_failed' } + end + local restore, cerr, detail = M.build_live_weight_restore(req, ruleset or '') + if not restore then + return { ok = false, err = cerr, backend = 'openwrt', code = 'mwan_live_weights_plan_failed' } + end + local rok, rerr, rout = restore_runner(restore) + if rok ~= true then + return { + ok = false, + err = 'MWAN3 live weight restore failed: ' .. tostring(rerr), + backend = 'openwrt', + code = 'mwan_live_weights_apply_failed', + restore = restore, + detail = detail, + output = rout, + } + end + return { ok = true, changed = true, applied = true, backend = 'openwrt', detail = detail, restore = restore, output = rout } +end + +return M diff --git a/src/services/hal/backends/network/providers/openwrt/names.lua b/src/services/hal/backends/network/providers/openwrt/names.lua new file mode 100644 index 00000000..e2d0764a --- /dev/null +++ b/src/services/hal/backends/network/providers/openwrt/names.lua @@ -0,0 +1,392 @@ +-- services/hal/backends/network/providers/openwrt/names.lua +-- Bounded OpenWrt name allocation. +-- +-- Product ids are semantic names. OpenWrt names are provider artefacts with +-- small hard limits; generate them centrally so long or hostile config ids can +-- never leak into netifd, Linux device, firewall, dnsmasq or mwan3 namespaces. + +local M = {} +local Ctx = {} +Ctx.__index = Ctx + +local MAX = { + logical_interface = 8, + linux_device = 14, + bridge_device = 14, + firewall_zone = 11, + mwan_name = 15, + dnsmasq_instance = 15, + uci_section = 32, +} + +local function is_plain_table(v) + return type(v) == 'table' and getmetatable(v) == nil +end + +local function sorted_keys(t) + local ks = {} + for k in pairs(t or {}) do ks[#ks + 1] = k end + table.sort(ks, function(a, b) return tostring(a) < tostring(b) end) + return ks +end + +local function clean(s) + s = tostring(s or ''):lower():gsub('[^a-z0-9]', '') + return s +end + +local function uci_clean(s) + s = tostring(s or ''):gsub('[^%w_]', '_') + if s == '' then s = 'x' end + if not s:match('^[A-Za-z_]') then s = 'x_' .. s end + return s +end + +local function safe_plain_name(s, max_len) + s = tostring(s or '') + if #s == 0 or #s > max_len then return nil end + if not s:match('^[A-Za-z][A-Za-z0-9_]*$') then return nil end + return s +end + +local function words(seed) + local out = {} + for w in tostring(seed or ''):lower():gmatch('[a-z0-9]+') do + out[#out + 1] = w + end + return out +end + +local function take(s, n) + s = clean(s) + if n <= 0 then return '' end + if #s >= n then return s:sub(1, n) end + return s +end + +local function pad_stem(s, fallback, n) + s = clean(s) + local f = clean(fallback) + if s == '' then s = f end + if s == '' then s = 'x' end + while #s < n do s = s .. (f ~= '' and f or 'x') end + if not s:sub(1, 1):match('%a') then s = 'x' .. s end + return s:sub(1, n) +end + +local function modem_stem(ws, n) + if ws[1] ~= 'modem' or n < 5 then return nil end + if #ws == 2 then + return pad_stem(take(ws[1], 2) .. take(ws[2], n - 2), table.concat(ws, ''), n) + end + if #ws == 3 then + return pad_stem(take(ws[1], 1) .. take(ws[2], 2) .. take(ws[3], n - 3), table.concat(ws, ''), n) + end + if #ws >= 4 then + return pad_stem(take(ws[1], 1) .. take(ws[2], 1) .. take(ws[3], 1) .. take(ws[#ws], n - 3), table.concat(ws, ''), n) + end + return nil +end + +local function readable_stem(seed, fallback, n) + local ws = words(seed) + local modem = modem_stem(ws, n) + if modem then return modem end + if #ws == 2 and n >= 4 then + local first = math.min(2, n - 2) + return pad_stem(take(ws[1], first) .. take(ws[2], n - first), table.concat(ws, ''), n) + end + local s = clean(seed) + if s == '' then s = clean(fallback) end + return pad_stem(s, fallback, n) +end + +local function uint32_to_hex(n) + -- Avoid string.format('%x', n). On some Lua 5.3/5.4 builds, values + -- that have passed through floating-point arithmetic are tagged as + -- numbers rather than integers and '%x' raises "integer expected". + -- This pure arithmetic formatter works on Lua 5.1/LuaJIT/5.3+. + local hex = '0123456789abcdef' + n = math.floor(tonumber(n) or 0) % 4294967296 + local out = {} + for i = 8, 1, -1 do + local d = n % 16 + out[i] = hex:sub(d + 1, d + 1) + n = (n - d) / 16 + end + return table.concat(out) +end + +local function hash_hex(seed, len) + -- Small deterministic FNV-1a-like hash using only double-safe arithmetic. + local h = 2166136261 + seed = tostring(seed or '') + for i = 1, #seed do + h = (h + seed:byte(i)) % 4294967296 + h = (h * 16777619) % 4294967296 + end + local s = uint32_to_hex(h) + while #s < len do s = s .. s end + return s:sub(1, len) +end + +local function make_name(seed, fallback, class, max_len, used) + local hash_len = max_len >= 8 and 3 or 2 + local prefix_len = math.max(1, max_len - hash_len) + local p = readable_stem(seed, fallback or class, prefix_len) + for attempt = 0, 64 do + local material = class .. ':' .. tostring(seed or '') .. ':' .. tostring(attempt) + local hash_len = math.max(1, max_len - #p) + local name = p .. hash_hex(material, hash_len) + if #name > max_len then name = name:sub(1, max_len) end + local prior = used[name] + if prior == nil or prior == material then + used[name] = material + return name + end + end + error('OpenWrt name collision for ' .. tostring(class) .. ':' .. tostring(seed), 2) +end + +local function make_uci_section(kind, seed, used) + local base = uci_clean(kind .. '_' .. tostring(seed or 'x')) + if #base > MAX.uci_section then + base = base:sub(1, 22) .. '_' .. hash_hex(kind .. ':' .. tostring(seed), 8) + end + local name = base + local i = 0 + while used[name] and used[name] ~= kind .. ':' .. tostring(seed) do + i = i + 1 + local suffix = '_' .. tostring(i) + name = base:sub(1, MAX.uci_section - #suffix) .. suffix + end + used[name] = kind .. ':' .. tostring(seed) + return name +end + +local function remember(self, class, key, name) + key = tostring(key) + self._cache[class] = self._cache[class] or {} + self._cache[class][key] = self._cache[class][key] or name + self._reverse[class] = self._reverse[class] or {} + self._reverse[class][name] = key + return name +end + +local function memo(self, class, key, max_len, fallback) + self._cache[class] = self._cache[class] or {} + if self._cache[class][key] then return self._cache[class][key] end + self._used[class] = self._used[class] or {} + local name = make_name(key, fallback, class, max_len, self._used[class]) + self._cache[class][key] = name + self._reverse[class] = self._reverse[class] or {} + self._reverse[class][name] = key + return name +end + +local function reserve(self, class, name, reason) + self._used[class] = self._used[class] or {} + if self._used[class][name] and self._used[class][name] ~= 'reserved:' .. tostring(reason) then + error('OpenWrt reserved-name collision for ' .. tostring(class) .. ':' .. tostring(name), 2) + end + self._used[class][name] = 'reserved:' .. tostring(reason) +end + +local function remember_if_available(self, class, key, name) + key = tostring(key) + local material = class .. ':' .. key + self._used[class] = self._used[class] or {} + local prior = self._used[class][name] + if prior ~= nil and prior ~= material then return nil end + self._used[class][name] = material + return remember(self, class, key, name) +end + +local function safe_or_memo(self, class, key, max_len, fallback) + local safe = safe_plain_name(key, max_len) + if safe then + local name = remember_if_available(self, class, key, safe) + if name then return name end + end + return memo(self, class, tostring(key), max_len, fallback) +end + +local function remember_prefixed_if_available(self, class, key, prefix, safe_stem) + if not safe_stem then return nil end + local name = prefix .. safe_stem + return remember_if_available(self, class, key, name) +end + +local function memo_prefixed(self, class, key, max_len, prefix, fallback) + key = tostring(key) + self._cache[class] = self._cache[class] or {} + if self._cache[class][key] then return self._cache[class][key] end + self._used[class] = self._used[class] or {} + local stem_len = max_len - #prefix + if stem_len < 1 then error('prefix too long for OpenWrt name class ' .. tostring(class), 2) end + local hash_len = stem_len >= 8 and 3 or math.min(3, math.max(1, stem_len - 1)) + local stem_prefix_len = math.max(1, stem_len - hash_len) + local stem_prefix = readable_stem(key, fallback or prefix, stem_prefix_len) + for attempt = 0, 64 do + local material = class .. ':' .. key .. ':' .. tostring(attempt) + local hash_len = math.max(1, stem_len - #stem_prefix) + local name = prefix .. stem_prefix .. hash_hex(material, hash_len) + if #name > max_len then name = name:sub(1, max_len) end + local prior = self._used[class][name] + if prior == nil or prior == material then + self._used[class][name] = material + self._cache[class][key] = name + self._reverse[class] = self._reverse[class] or {} + self._reverse[class][name] = key + return name + end + end + error('OpenWrt name collision for ' .. tostring(class) .. ':' .. tostring(key), 2) +end + +local function memo_mwan(self, role, id) + -- All MWAN3 UCI section names share one package namespace. Prefer the + -- semantic name when it is valid and unused, so conventional names such as + -- "balanced" and "default_rule_v4" remain readable. If another MWAN3 + -- section already owns that name, generate a bounded role-prefixed name. + local prefix = ({ iface = 'mi', member = 'mm', policy = 'mp', rule = 'mr' })[role] + local raw_id = tostring(id or (role == 'policy' and 'balanced' or '')) + local key = role .. ':' .. raw_id + self._cache.mwan = self._cache.mwan or {} + local name = self._cache.mwan[key] + if not name then + local safe = safe_plain_name(raw_id, MAX.mwan_name) + if safe then + name = remember_if_available(self, 'mwan', key, safe) + end + if not name then + name = memo_prefixed(self, 'mwan', key, MAX.mwan_name, prefix, role) + end + end + -- Preserve role-specific diagnostic maps in snapshots while the allocator + -- itself enforces one shared MWAN3 package namespace. + local cache_class = 'mwan_' .. role + self._cache[cache_class] = self._cache[cache_class] or {} + self._cache[cache_class][raw_id] = name + self._reverse[cache_class] = self._reverse[cache_class] or {} + self._reverse[cache_class][name] = raw_id + return name +end + +function Ctx:iface(id) + return safe_or_memo(self, 'logical_interface', tostring(id), MAX.logical_interface, 'if') +end + +function Ctx:bridge(id) + local safe = safe_plain_name(id, MAX.bridge_device - 3) + local name = remember_prefixed_if_available(self, 'bridge_device', id, 'br-', safe) + if name then return name end + return memo_prefixed(self, 'bridge_device', tostring(id), MAX.bridge_device, 'br', 'br') +end + +function Ctx:vlan(id) + local safe = safe_plain_name(id, MAX.linux_device - 3) + local name = remember_prefixed_if_available(self, 'linux_device', id, 'vl-', safe) + if name then return name end + return memo_prefixed(self, 'linux_device', tostring(id), MAX.linux_device, 'vl', 'vl') +end + +function Ctx:zone(id) + return safe_or_memo(self, 'firewall_zone', tostring(id), MAX.firewall_zone, 'zn') +end + +function Ctx:mwan_iface(id) + -- mwan3 interface section names are not independent identifiers: they must + -- exactly match the OpenWrt logical network interface names. Long semantic + -- product ids such as "modem_primary" may therefore be shortened by the + -- logical-interface allocator, and mwan3 must use that same generated name. + local raw_id = tostring(id or '') + local name = self:iface(raw_id) + local key = 'iface:' .. raw_id + self._cache.mwan = self._cache.mwan or {} + if not self._cache.mwan[key] then + local remembered = remember_if_available(self, 'mwan', key, name) + if not remembered then + error('OpenWrt logical interface name collides in mwan3 namespace for ' .. raw_id, 2) + end + end + local cache_class = 'mwan_iface' + self._cache[cache_class] = self._cache[cache_class] or {} + self._cache[cache_class][raw_id] = name + self._reverse[cache_class] = self._reverse[cache_class] or {} + self._reverse[cache_class][name] = raw_id + return name +end + +function Ctx:mwan_member(id) + return memo_mwan(self, 'member', id) +end + +function Ctx:mwan_policy(id) + return memo_mwan(self, 'policy', id or 'balanced') +end + +function Ctx:mwan_rule(id) + return memo_mwan(self, 'rule', id) +end + +function Ctx:dns_instance(id) + return safe_or_memo(self, 'dnsmasq_instance', tostring(id), MAX.dnsmasq_instance, 'dn') +end + +function Ctx:section(kind, id) + self._used.uci_section = self._used.uci_section or {} + return make_uci_section(kind, tostring(id), self._used.uci_section) +end + +function Ctx:semantic_for(class, generated) + return self._reverse[class] and self._reverse[class][generated] or nil +end + +function Ctx:snapshot() + local out = { max = {}, names = {} } + for k, v in pairs(MAX) do out.max[k] = v end + for class, map in pairs(self._cache) do + out.names[class] = {} + for semantic, generated in pairs(map) do out.names[class][semantic] = generated end + end + return out +end + +function Ctx:limits() + local out = {} + for k, v in pairs(MAX) do out[k] = v end + return out +end + +function M.allocate(intent, _provider_config) + local ctx = setmetatable({ _cache = {}, _used = {}, _reverse = {} }, Ctx) + -- Reserve baseline UCI names produced by the provider. Product ids that + -- would otherwise map directly to these names must be generated instead. + reserve(ctx, 'logical_interface', 'loopback', 'network.loopback') + reserve(ctx, 'logical_interface', 'globals', 'network.globals') + reserve(ctx, 'firewall_zone', 'defaults', 'firewall.defaults') + reserve(ctx, 'mwan', 'globals', 'mwan3.globals') + -- Pre-allocate common names to catch deterministic collisions before apply. + for _, seg_id in ipairs(sorted_keys((intent or {}).segments)) do + ctx:iface(seg_id); ctx:bridge(seg_id); ctx:vlan(seg_id) + local seg = intent.segments[seg_id] + local fw = is_plain_table(seg and seg.firewall) and seg.firewall or {} + ctx:zone(fw.zone or seg_id) + end + for _, if_id in ipairs(sorted_keys((intent or {}).interfaces)) do + ctx:iface(if_id); ctx:bridge(if_id); ctx:vlan(if_id) + end + local fw = is_plain_table((intent or {}).firewall) and intent.firewall or {} + for _, z in ipairs(sorted_keys(fw.zones or {})) do ctx:zone(z) end + local wan = is_plain_table((intent or {}).wan) and intent.wan or {} + local members = is_plain_table(wan.members) and wan.members or {} + for _, mid in ipairs(sorted_keys(members)) do + ctx:mwan_iface((members[mid] and (members[mid].interface or members[mid].iface)) or mid) + ctx:mwan_member(mid) + end + return ctx, nil +end + +M.MAX = MAX +return M diff --git a/src/services/hal/backends/network/providers/openwrt/observer.lua b/src/services/hal/backends/network/providers/openwrt/observer.lua new file mode 100644 index 00000000..a26379e2 --- /dev/null +++ b/src/services/hal/backends/network/providers/openwrt/observer.lua @@ -0,0 +1,472 @@ +-- services/hal/backends/network/providers/openwrt/observer.lua +-- +-- Event-led observation infrastructure for the OpenWrt network provider. +-- +-- Low-level OpenWrt event sources are treated as wake-up signals only. This +-- module coalesces them by semantic subject, takes a provider-owned snapshot, +-- and emits a curated observed-state event to the HAL capability event surface. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' +local socket = require 'fibers.io.socket' +local exec = require 'fibers.io.exec' +local file = require 'fibers.io.file' +local cjson = require 'cjson.safe' +local safe = require 'coxpcall' + +local perform = fibers.perform + +local M = {} +local Observer = {} +Observer.__index = Observer + +local function now() + return fibers.now() +end + +local function copy_plain(v, seen) + if type(v) ~= 'table' then return v end + if getmetatable(v) ~= nil then return v end + seen = seen or {} + if seen[v] then return seen[v] end + local out = {} + seen[v] = out + for k, val in pairs(v) do out[copy_plain(k, seen)] = copy_plain(val, seen) end + return out +end + +local function sorted_keys(t) + local ks = {} + for k in pairs(t or {}) do ks[#ks + 1] = k end + table.sort(ks, function(a, b) return tostring(a) < tostring(b) end) + return ks +end + +local function log(self, level, payload) + local logger = self.logger + if type(logger) == 'function' then return logger(level, payload) end + if logger and type(logger[level]) == 'function' then return logger[level](logger, payload) end +end + +local function subject_from_trigger(trigger) + trigger = trigger or {} + local source = trigger.source + local env = trigger.env or trigger.payload or trigger + local action = env.ACTION or env.action + local iface = env.INTERFACE or env.interface + local dev = env.DEVICE or env.DEVICENAME or env.DEVNAME or env.device or env.name + + if trigger.subject then return tostring(trigger.subject) end + if source == 'mwan3' or trigger.kind == 'mwan3' then + return 'mwan:' .. tostring(iface or dev or 'unknown') + end + if source == 'ubus' or trigger.kind == 'ubus' then + if iface then return 'interface:' .. tostring(iface) end + if dev then return 'device:' .. tostring(dev) end + return 'ubus:network' + end + if source == 'hotplug' or trigger.kind == 'hotplug' then + local subsystem = env.SUBSYSTEM or trigger.subsystem + if trigger.directory == 'iface' or subsystem == 'iface' or iface then + return 'interface:' .. tostring(iface or dev or 'unknown') + end + if trigger.directory == 'net' or subsystem == 'net' then + return 'device:' .. tostring(dev or iface or 'unknown') + end + if trigger.directory == 'dhcp' or subsystem == 'dhcp' then return 'dhcp' end + if trigger.directory == 'firewall' or subsystem == 'firewall' then return 'firewall' end + if trigger.directory == 'tty' or trigger.directory == 'usb' or subsystem == 'tty' or subsystem == 'usb' then + return 'modem-device:' .. tostring(dev or iface or 'unknown') + end + end + if action == 'ifup' or action == 'ifdown' or action == 'ifupdate' or action == 'iflink' then + return 'interface:' .. tostring(iface or dev or 'unknown') + end + return 'network' +end + +local function classify_event(subject, trigger, observed) + trigger = trigger or {} + local source = trigger.source + local env = trigger.env or trigger.payload or trigger + local action = env.ACTION or env.action + if subject:match('^interface:') then return 'interface_changed' end + if subject:match('^device:') then return 'device_changed' end + if subject:match('^mwan:') then return 'mwan_member_changed' end + if subject == 'dhcp' then return 'dhcp_changed' end + if subject == 'firewall' then return 'firewall_changed' end + if source == 'snapshot' or action == 'snapshot' or observed then return 'snapshot_done' end + return 'network_changed' +end + +local function build_event(self, subject, trigger, snapshot) + trigger = copy_plain(trigger or {}) + local observed = snapshot and snapshot.observed or nil + local event_kind = classify_event(subject, trigger, observed) + + return { + schema = 'devicecode.net.observation/1', + kind = event_kind, + event_id = tostring(self.next_event_id), + source = trigger.source or 'snapshot', + subject = subject, + seq = self.next_event_id, + monotonic = now(), + trigger = trigger, + observed = observed, + snapshot = snapshot and snapshot.ok == true, + backend = snapshot and snapshot.backend or 'openwrt', + err = snapshot and snapshot.err or nil, + } +end + +local function decode_ubus_line(line) + local obj = cjson.decode(line or '') + if type(obj) ~= 'table' then return nil end + local payload = obj['network.interface'] or obj.network_interface or obj.network + if type(payload) ~= 'table' then + for k, v in pairs(obj) do + if tostring(k):match('^network%.') and type(v) == 'table' then + payload = v + break + end + end + end + if type(payload) ~= 'table' then payload = obj end + return { + source = 'ubus', + kind = 'ubus', + payload = payload, + action = payload.action, + interface = payload.interface, + device = payload.device, + } +end + +local TRANSIENT_MWAN_ACTIONS = { + connecting = true, + disconnecting = true, +} + +local function trigger_env(trigger) + if type(trigger) ~= 'table' then return {} end + local env = trigger.env or trigger.payload or trigger + if type(env) ~= 'table' then return {} end + return env +end + +local function trigger_action(trigger) + local env = trigger_env(trigger) + local action = env.ACTION or env.action + if action == nil and type(trigger) == 'table' then action = trigger.action end + if action == nil then return nil end + return tostring(action):lower() +end + +local function is_transient_mwan_action(trigger) + return TRANSIENT_MWAN_ACTIONS[trigger_action(trigger) or ''] == true +end + +local function normalise_hotplug_record(rec) + if type(rec) ~= 'table' then return nil end + local env = rec.env or rec + return { + source = rec.source or 'hotplug', + kind = rec.kind or 'hotplug', + directory = rec.directory or rec.subsystem or env.SUBSYSTEM, + env = copy_plain(env), + } +end + +function M.new(opts) + opts = opts or {} + local tx, rx = mailbox.new(opts.queue_len or 64, { full = opts.full or 'drop_oldest' }) + return setmetatable({ + tx = tx, + rx = rx, + closed = false, + scope = nil, + listener = nil, + logger = opts.logger, + emit = opts.emit, + snapshot = opts.snapshot, + socket_path = opts.socket_path or '/var/run/devicecode-net-observe.sock', + debounce_s = opts.debounce_s or 0.15, + enable_socket = opts.enable_socket ~= false, + enable_ubus = opts.enable_ubus == true, + initial_snapshot = opts.initial_snapshot ~= false, + next_event_id = 1, + active_streams = {}, + ubus_cmd = nil, + ubus_stream = nil, + }, Observer), nil +end + +function Observer:ingest(trigger) + if self.closed then return false, 'observer closed' end + -- mwan3.user emits transitional phases such as connecting/disconnecting. + -- Treat these as diagnostics only: they are not stable data-path states and + -- must not become retained NET backhaul status. Terminal events such as + -- connected/disconnected still wake the observer and take a fresh snapshot. + if is_transient_mwan_action(trigger) then + log(self, 'debug', { what = 'network_observer_transient_trigger_ignored', summary = 'transient mwan event ignored action=' .. tostring(trigger_action(trigger) or '?'), trigger = trigger }) + return true, 'ignored_transient_mwan_action' + end + return self.tx:send(trigger) +end + +function Observer:emit_event(ev) + if type(self.emit) == 'function' then + local ok, err = safe.pcall(function() return self.emit(ev) end) + if not ok then return nil, tostring(err) end + if err ~= nil and err ~= true then return nil, tostring(err) end + end + return true, nil +end + +function Observer:take_snapshot(subject, trigger) + if type(self.snapshot) ~= 'function' then + return { ok = false, backend = 'openwrt', err = 'snapshot callback unavailable' } + end + local ok, result = safe.pcall(function() return self.snapshot(subject, trigger) end) + if not ok then return { ok = false, backend = 'openwrt', err = tostring(result) } end + if type(result) == 'table' then return result end + return { ok = result == true, backend = 'openwrt', result = result } +end + +function Observer:flush_subject(subject, trigger) + local snapshot = self:take_snapshot(subject, trigger) + local ev = build_event(self, subject, trigger, snapshot) + self.next_event_id = self.next_event_id + 1 + local ok, err = self:emit_event(ev) + if ok ~= true then + log(self, 'warn', { what = 'network_observer_emit_failed', err = tostring(err), subject = subject }) + return nil, err + end + return true, nil +end + +function Observer:coalescer() + local pending = {} + if self.initial_snapshot then + pending.network = { source = 'snapshot', kind = 'snapshot', action = 'initial' } + end + + while true do + local wait_s = nil + local due_subject = nil + local due_record = nil + local t = now() + + for _, subject in ipairs(sorted_keys(pending)) do + local rec = pending[subject] + local due = rec.due or t + local dt = due - t + if dt <= 0 then + due_subject = subject + due_record = rec + break + end + if wait_s == nil or dt < wait_s then wait_s = dt end + end + + if due_subject then + pending[due_subject] = nil + self:flush_subject(due_subject, due_record.trigger) + else + local trigger + if wait_s == nil then + trigger = perform(self.rx:recv_op()) + else + local which, val = perform(fibers.named_choice { + event = self.rx:recv_op(), + timer = sleep.sleep_op(wait_s), + }) + if which == 'event' then trigger = val else trigger = false end + end + + if trigger == nil then return end + if trigger ~= false then + local subject = subject_from_trigger(trigger) + pending[subject] = { + trigger = trigger, + due = now() + self.debounce_s, + } + end + end + end +end + +local function close_stream(st) + if not st then return true, nil end + if st.close then return st:close() end + return true, nil +end + +local function terminate_stream(st, reason) + if not st then return true, nil end + if st.terminate then + st:terminate(reason or 'terminated') + return true, nil + end + if st.close then + -- Fallback for non-Stream-like objects. Avoid ordinary pcall here: close() + -- may be an Op-performing wrapper under PUC Lua. + return st:close() + end + return true, nil +end + +local function owned_process_flags() + local flags = { process_group = true } + if type(exec.supports) == 'function' and exec.supports('parent_death_signal') then + flags.parent_death_signal = 'TERM' + end + return flags +end + +local function terminate_command(cmd, sig) + if not cmd or type(cmd.kill) ~= 'function' then return true, nil end + local ok, a, b = pcall(function () return cmd:kill(sig or 15) end) + if not ok then return nil, tostring(a) end + if a == false or a == nil then return nil, tostring(b or 'command kill failed') end + return true, nil +end + +function Observer:_track_stream(st) + if st then self.active_streams[st] = true end + return st +end + +function Observer:_untrack_stream(st) + if st then self.active_streams[st] = nil end +end + +function Observer:_stop_ubus_listener() + local stream = self.ubus_stream + self.ubus_stream = nil + terminate_stream(stream, 'ubus listener stopped') + + local cmd = self.ubus_cmd + self.ubus_cmd = nil + terminate_command(cmd, 15) +end + +function Observer:ubus_listener() + while not self.closed do + local cmd = exec.command { + 'ubus', 'listen', 'network.interface', + stdin = 'null', + stdout = 'pipe', + stderr = 'stdout', + shutdown_grace = 0.2, + flags = owned_process_flags(), + } + self.ubus_cmd = cmd + local stream, err = cmd:stdout_stream() + if not stream then + self.ubus_cmd = nil + log(self, 'warn', { what = 'ubus_listen_start_failed', err = tostring(err) }) + perform(sleep.sleep_op(5.0)) + else + self.ubus_stream = stream + while not self.closed do + local line, rerr = perform(stream:read_line_op()) + if line == nil then + if rerr and not self.closed then log(self, 'warn', { what = 'ubus_listen_read_failed', err = tostring(rerr) }) end + break + end + local trig = decode_ubus_line(line) + if trig then self:ingest(trig) end + end + if self.ubus_stream == stream then self.ubus_stream = nil end + if self.ubus_cmd == cmd then self.ubus_cmd = nil end + terminate_stream(stream, 'ubus listener stream closed') + perform(cmd:shutdown_op(0.2)) + if not self.closed then perform(sleep.sleep_op(1.0)) end + end + end + self:_stop_ubus_listener() +end + +local function trigger_summary(trigger) + trigger = trigger or {} + local env = trigger.env or trigger.payload or trigger + return string.format('%s event action=%s interface=%s device=%s directory=%s', + tostring(trigger.source or trigger.kind or 'hotplug'), + tostring(env.ACTION or env.action or '?'), + tostring(env.INTERFACE or env.interface or '?'), + tostring(env.DEVICE or env.DEVICENAME or env.DEVNAME or env.device or '?'), + tostring(trigger.directory or env.SUBSYSTEM or '?')) +end + +function Observer:handle_socket_stream(st) + self:_track_stream(st) + while not self.closed do + local line = perform(st:read_line_op()) + if line == nil then break end + local rec = cjson.decode(line) + local trig = normalise_hotplug_record(rec) + if trig then + local trigger_log_level = is_transient_mwan_action(trig) and 'debug' or 'info' + if trig.source == 'mwan3' or trig.kind == 'mwan3' then + log(self, trigger_log_level, { what = 'network_observer_mwan3_trigger', summary = trigger_summary(trig), trigger = trig }) + elseif trig.directory == 'iface' or trig.directory == 'net' then + log(self, trigger_log_level, { what = 'network_observer_hotplug_trigger', summary = trigger_summary(trig), trigger = trig }) + end + self:ingest(trig) + end + end + self:_untrack_stream(st) + close_stream(st) +end + +function Observer:socket_server() + pcall(function () file.unlink(self.socket_path) end) + local s, err = socket.listen_unix(self.socket_path, { ephemeral = true }) + if not s then + log(self, 'warn', { what = 'hotplug_socket_listen_failed', path = self.socket_path, err = tostring(err) }) + return + end + self.listener = s + log(self, 'debug', { what = 'hotplug_socket_listening', path = self.socket_path }) + while not self.closed do + local st, aerr = perform(s:accept_op()) + if st then + self.scope:spawn(function () self:handle_socket_stream(st) end) + elseif aerr and not self.closed then + log(self, 'warn', { what = 'hotplug_socket_accept_failed', err = tostring(aerr) }) + perform(sleep.sleep_op(0.5)) + end + end + s:close() +end + +function Observer:start(scope) + if self.closed then return nil, 'observer closed' end + if self.scope then return true, nil end + if type(scope) ~= 'table' then return nil, 'scope required' end + local child, err = scope:child() + if not child then return nil, err or 'observer scope create failed' end + self.scope = child + child:spawn(function () self:coalescer() end) + if self.enable_socket then child:spawn(function () self:socket_server() end) end + if self.enable_ubus then child:spawn(function () self:ubus_listener() end) end + return true, nil +end + +function Observer:terminate(reason) + if self.closed then return true, nil end + self.closed = true + self.tx:close(reason or 'observer terminated') + self:_stop_ubus_listener() + for st in pairs(self.active_streams) do terminate_stream(st, reason or 'observer terminated') end + self.active_streams = {} + if self.listener and self.listener.close then self.listener:close() end + if self.scope then self.scope:cancel(reason or 'observer terminated') end + pcall(function () file.unlink(self.socket_path) end) + return true, nil +end + +return M diff --git a/src/services/hal/backends/network/providers/openwrt/shaper.lua b/src/services/hal/backends/network/providers/openwrt/shaper.lua new file mode 100644 index 00000000..536e92ce --- /dev/null +++ b/src/services/hal/backends/network/providers/openwrt/shaper.lua @@ -0,0 +1,50 @@ +-- services/hal/backends/network/providers/openwrt/shaper.lua +-- Dispatcher for Devicecode OpenWrt shaping link kinds. + +local u32_shaper = require 'services.hal.backends.network.providers.openwrt.tc_u32_shaper' +local mark_shaper = require 'services.hal.backends.network.providers.openwrt.tc_mark_shaper' +local shaping_marks = require 'services.hal.backends.network.providers.openwrt.shaping_marks' + +local M = {} +local function is_plain_table(x) return type(x) == 'table' and getmetatable(x) == nil end +local function copy(v) if type(v) ~= 'table' then return v end local o = {}; for k,val in pairs(v) do o[k]=copy(val) end; return o end +local function sorted_keys(t) local ks={}; for k in pairs(t or {}) do ks[#ks+1]=k end; table.sort(ks,function(a,b)return tostring(a) 0 or b > 0 do + local aa, bb = a % 2, b % 2 + if aa == 1 and bb == 1 then res = res + bitv end + a = math.floor(a / 2); b = math.floor(b / 2); bitv = bitv * 2 + end + return res +end + +local M = {} + +local DEFAULT_MASK = '0x00f00000' +local DEFAULT_CONTROL = '0x00100000' +local DEFAULT_CLIENT = '0x00200000' + +local function is_plain_table(x) return type(x) == 'table' and getmetatable(x) == nil end + +local function exec_spec(argv, opts) + opts = opts or {} + local spec = { stdin = opts.stdin or 'null', stdout = opts.stdout or 'pipe', stderr = opts.stderr or 'stdout' } + for i = 1, #argv do spec[i] = argv[i] end + return spec +end + +local function hex_number(v) + if type(v) == 'number' then return v end + if type(v) ~= 'string' then return nil end + local hx = v:match('^%s*0x(%x+)%s*$') + if hx then return tonumber(hx, 16) end + local dec = v:match('^%s*(%d+)%s*$') + return dec and tonumber(dec) or nil +end + +local function mark_spec(req) + local marks = is_plain_table(req and req.marks) and req.marks or {} + local mask = marks.mask or req.mark_mask or DEFAULT_MASK + local control = marks.control or req.control_mark or DEFAULT_CONTROL + local client = marks.client or req.client_mark or DEFAULT_CLIENT + return tostring(mask), tostring(control), tostring(client) +end + +local function quote_restore_arg(s) + s = tostring(s or '') + if s:find('[\r\n]') then error('newline in iptables-restore argument') end + if s:find('%s') or s:find('"') then return '"' .. s:gsub('(["\\])', '\\%1') .. '"' end + return s +end + +local function restore_line(argv) + local out = {} + for i = 1, #argv do out[i] = quote_restore_arg(argv[i]) end + return table.concat(out, ' ') +end + +function M.default_marks() + return { mask = DEFAULT_MASK, control = DEFAULT_CONTROL, client = DEFAULT_CLIENT } +end + +function M.validate_marks(req, mwan_mask) + local mask, control, client = mark_spec(req) + local mn, cn, cln = hex_number(mask), hex_number(control), hex_number(client) + if not (mn and cn and cln) then return nil, 'invalid shaping mark values' end + if cn == cln then return nil, 'control and client marks must differ' end + if mwan_mask then + local mm = hex_number(mwan_mask) + if mm and band(mn, mm) ~= 0 then return nil, 'shaping mark mask overlaps MWAN mask' end + end + return { mask = mask, control = control, client = client }, nil +end + +local function link_iface(link) + return link.iface or link.device or link.interface or link.linux_interface +end + +function M.build_restore(req) + req = req or {} + local mask, control, client = mark_spec(req) + local lines = { '*mangle' } + local outputs, forwards = {}, {} + for link_id, link in pairs(req.links or {}) do + if is_plain_table(link) and link.enabled ~= false and link.kind == 'wan_mark' then + local iface = link_iface(link) + if type(iface) == 'string' and iface ~= '' then + outputs[#outputs + 1] = iface + forwards[#forwards + 1] = iface + end + end + end + table.sort(outputs); table.sort(forwards) + lines[#lines + 1] = ':DEVICECODE_SHAPING_OUTPUT - [0:0]' + lines[#lines + 1] = ':DEVICECODE_SHAPING_FORWARD - [0:0]' + lines[#lines + 1] = '-F DEVICECODE_SHAPING_OUTPUT' + lines[#lines + 1] = '-F DEVICECODE_SHAPING_FORWARD' + for _, iface in ipairs(outputs) do + lines[#lines + 1] = restore_line({ '-A', 'DEVICECODE_SHAPING_OUTPUT', '-o', iface, '-m', 'comment', '--comment', 'devicecode-shaping router exempt', '-j', 'MARK', '--set-xmark', control .. '/' .. mask }) + lines[#lines + 1] = restore_line({ '-A', 'DEVICECODE_SHAPING_OUTPUT', '-o', iface, '-m', 'comment', '--comment', 'devicecode-shaping save router mark', '-j', 'CONNMARK', '--save-mark', '--mask', mask }) + end + for _, iface in ipairs(forwards) do + lines[#lines + 1] = restore_line({ '-A', 'DEVICECODE_SHAPING_FORWARD', '-o', iface, '-m', 'comment', '--comment', 'devicecode-shaping client', '-j', 'MARK', '--set-xmark', client .. '/' .. mask }) + lines[#lines + 1] = restore_line({ '-A', 'DEVICECODE_SHAPING_FORWARD', '-o', iface, '-m', 'comment', '--comment', 'devicecode-shaping save client mark', '-j', 'CONNMARK', '--save-mark', '--mask', mask }) + end + lines[#lines + 1] = 'COMMIT' + lines[#lines + 1] = '' + return table.concat(lines, '\n'), { mask = mask, control = control, client = client } +end + + +local function call_runner(run_cmd, argv) + if type(run_cmd) ~= 'function' then return true end + local ok = run_cmd(argv) + return ok == true +end + +local function ensure_jumps(run_cmd) + if type(run_cmd) ~= 'function' then return true end + call_runner(run_cmd, { 'iptables', '-t', 'mangle', '-N', 'DEVICECODE_SHAPING_OUTPUT' }) + call_runner(run_cmd, { 'iptables', '-t', 'mangle', '-N', 'DEVICECODE_SHAPING_FORWARD' }) + if not call_runner(run_cmd, { 'iptables', '-t', 'mangle', '-C', 'OUTPUT', '-j', 'DEVICECODE_SHAPING_OUTPUT' }) then + call_runner(run_cmd, { 'iptables', '-t', 'mangle', '-A', 'OUTPUT', '-j', 'DEVICECODE_SHAPING_OUTPUT' }) + end + if not call_runner(run_cmd, { 'iptables', '-t', 'mangle', '-C', 'FORWARD', '-j', 'DEVICECODE_SHAPING_FORWARD' }) then + call_runner(run_cmd, { 'iptables', '-t', 'mangle', '-A', 'FORWARD', '-j', 'DEVICECODE_SHAPING_FORWARD' }) + end + return true +end + +local function default_restore(content) + local cmd = exec_mod.command(exec_spec({ 'iptables-restore', '--noflush' }, { stdin = 'pipe' })) + local stdin, serr = cmd:stdin_stream() + if not stdin then return nil, serr or 'failed to open iptables-restore stdin' end + local ok, werr = stdin:write(content or '') + if not ok then + pcall(function() stdin:terminate('iptables-restore write failed') end) + fibers.perform(cmd:shutdown_op(0.2)) + return nil, 'failed to write iptables-restore input: ' .. tostring(werr) + end + stdin:close() + local out, st, code, sig, err = fibers.perform(cmd:combined_output_op()) + if st == 'exited' and code == 0 then return true, nil, out or '' end + local detail = err or out or ('status=' .. tostring(st)) + if st == 'exited' then detail = tostring(detail) .. ' (exit ' .. tostring(code) .. ')' + elseif st == 'signalled' then detail = tostring(detail) .. ' (signal ' .. tostring(sig) .. ')' end + return nil, detail, out or '' +end + +function M.apply(req, opts) + opts = opts or {} + local restore, marks = M.build_restore(req or {}) + if not restore or restore == '' then return { ok = true, applied = false, marks = marks, backend = 'openwrt' } end + ensure_jumps(opts.run_cmd) + local runner = opts.run_restore or opts.restore + if type(runner) == 'function' then + local ok, err, out = runner(restore) + return { ok = ok == true, applied = ok == true, err = err, output = out, restore = restore, marks = marks, backend = 'openwrt' } + end + local ok, err, out = default_restore(restore) + return { ok = ok == true, applied = ok == true, err = err, output = out, restore = restore, marks = marks, backend = 'openwrt' } +end + +return M diff --git a/src/services/hal/backends/network/providers/openwrt/speedtest.lua b/src/services/hal/backends/network/providers/openwrt/speedtest.lua new file mode 100644 index 00000000..021c1f5b --- /dev/null +++ b/src/services/hal/backends/network/providers/openwrt/speedtest.lua @@ -0,0 +1,160 @@ +-- services/hal/backends/network/providers/openwrt/speedtest.lua +-- HAL-side WAN speed test using `mwan3 use `. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local exec = require 'fibers.io.exec' +local file = require 'fibers.io.file' + +local perform = fibers.perform +local M = {} + +local DEFAULT_URL = 'https://proof.ovh.net/files/100Mb.dat' +local HARD_MAX_DURATION_S = 1 +local DEFAULT_SAMPLE_INTERVAL_S = 0.06 +local DEFAULT_NO_IMPROVEMENT_SAMPLES = 2 + + +local function owned_process_flags() + local flags = { process_group = true } + if type(exec.supports) == 'function' and exec.supports('parent_death_signal') then + flags.parent_death_signal = 'TERM' + end + return flags +end + +local function read_counter(path) + local f, err = file.open(path, 'r') + if not f then return nil, err end + local data, rerr = f:read_all() + f:close() + if rerr ~= nil then return nil, rerr end + local n = tonumber((tostring(data or ''):gsub('%s+', ''))) + if not n then return nil, 'counter parse failed' end + return n, nil +end + +local function monotonic() + return fibers.now and fibers.now() or os.clock() +end + +local function record_sample(state, mbps) + if mbps <= 0 then return false end + state.samples = state.samples + 1 + if mbps > state.highest then + state.second_highest = state.highest + state.highest = mbps + state.consecutive_no_improvement = 0 + return true + end + if mbps > state.second_highest then state.second_highest = mbps end + state.consecutive_no_improvement = state.consecutive_no_improvement + 1 + return false +end + +local function averaged_top_two(state) + if state.samples <= 0 then return 0 end + if state.samples == 1 or state.second_highest <= 0 then return state.highest end + return (state.highest + state.second_highest) / 2 +end + +function M.run_op(req, opts) + return fibers.run_scope_op(function(scope) + req = req or {} + opts = opts or {} + local iface = req.interface + local dev = req.device or req.linux_interface or req.ifname + if type(iface) ~= 'string' or iface == '' then return { ok = false, err = 'speedtest interface required', backend = 'openwrt' } end + if type(dev) ~= 'string' or dev == '' then dev = iface end + local url = req.url or opts.url or DEFAULT_URL + local duration = tonumber(req.max_duration_s or opts.max_duration_s) or HARD_MAX_DURATION_S + if duration <= 0 or duration > HARD_MAX_DURATION_S then duration = HARD_MAX_DURATION_S end + local sample_s = tonumber(req.sample_interval_s or opts.sample_interval_s) or DEFAULT_SAMPLE_INTERVAL_S + if sample_s <= 0 then sample_s = DEFAULT_SAMPLE_INTERVAL_S end + local no_improvement_limit = tonumber(req.no_improvement_samples or opts.no_improvement_samples) or DEFAULT_NO_IMPROVEMENT_SAMPLES + no_improvement_limit = math.max(1, math.floor(no_improvement_limit)) + local argv = { 'mwan3', 'use', iface, 'wget', '-O', '/dev/null', tostring(url) } + local run_cmd = opts.run_cmd or req.run_cmd + if type(run_cmd) == 'function' then + local ok, out, err = run_cmd(argv) + local mbps = tonumber(out) + if ok == true and mbps ~= nil then + return { ok = true, backend = 'openwrt', interface = iface, device = dev, mbps = mbps, peak_mbps = mbps } + end + return { ok = false, code = 'speedtest_failed', backend = 'openwrt', interface = iface, device = dev, err = tostring(err or out or 'speedtest failed') } + end + local spec = { + stdin = 'null', + stdout = 'pipe', + stderr = 'null', + shutdown_grace = 0.2, + flags = owned_process_flags(), + } + for i = 1, #argv do spec[i] = argv[i] end + local cmd = exec.command(spec) + local started, serr = cmd:stdout_stream() + if not started then return { ok = false, code = 'start_failed', err = serr or 'speedtest start failed', backend = 'openwrt' } end + local counter = '/sys/class/net/' .. tostring(dev) .. '/statistics/rx_bytes' + local start_b, berr = read_counter(counter) + if not start_b then + perform(cmd:shutdown_op(0.2)) + return { + ok = false, + code = 'counter_unavailable', + err = 'counter read failed: ' .. tostring(berr or 'unknown'), + backend = 'openwrt', + interface = iface, + device = dev, + counter = counter, + } + end + local t0 = monotonic() + local prev_t, prev_b = t0, start_b + local samples = { + highest = 0, + second_highest = 0, + samples = 0, + consecutive_no_improvement = 0, + } + while (monotonic() - t0) < duration do + local remaining = duration - (monotonic() - t0) + if remaining <= 0 then break end + perform(sleep.sleep_op(math.min(sample_s, remaining))) + local t = monotonic() + local b = read_counter(counter) + if not b then break end + local dt = t - prev_t + local db = b - prev_b + if dt > 0 and db >= 0 then + local mbps = (db * 8) / (dt * 1000 * 1000) + record_sample(samples, mbps) + end + prev_t, prev_b = t, b + if samples.highest > 0 and samples.consecutive_no_improvement >= no_improvement_limit then break end + end + perform(cmd:shutdown_op(0.2)) + local mbps = averaged_top_two(samples) + if mbps <= 0 and (prev_b - start_b) <= 0 then + return { ok = false, code = 'insufficient_data', backend = 'openwrt', interface = iface, device = dev, data_mib = 0, duration_s = prev_t - t0 } + end + return { + ok = true, + backend = 'openwrt', + interface = iface, + device = dev, + mbps = mbps, + peak_mbps = mbps, + raw_peak_mbps = samples.highest, + second_peak_mbps = samples.second_highest > 0 and samples.second_highest or nil, + sample_count = samples.samples, + data_mib = (prev_b - start_b) / (1024 * 1024), + bytes = (prev_b - start_b), + duration_s = prev_t - t0, + } + end):wrap(function(status, _report, result) + if status ~= 'ok' then return { ok = false, code = 'worker_failed', err = tostring(result or status), backend = 'openwrt' } end + return result + end) +end + +return M diff --git a/src/services/hal/backends/network/providers/openwrt/tc_mark_shaper.lua b/src/services/hal/backends/network/providers/openwrt/tc_mark_shaper.lua new file mode 100644 index 00000000..07d45afb --- /dev/null +++ b/src/services/hal/backends/network/providers/openwrt/tc_mark_shaper.lua @@ -0,0 +1,139 @@ +-- services/hal/backends/network/providers/openwrt/tc_mark_shaper.lua +-- Mark-based WAN shaper. The mangle/connmark contract is owned by shaping_marks.lua. + +local fibers = require 'fibers' +local exec_mod = require 'fibers.io.exec' + +local M = {} +local ACTIVE_RUN_CMD = nil + +local function is_plain_table(x) return type(x) == 'table' and getmetatable(x) == nil end +local function ifb_name(iface) return 'ifb_' .. tostring(iface or ''):gsub('[^%w]', '_'):sub(1, 11) end +local function exec_spec(argv) local spec = { stdin = 'null', stdout = 'pipe', stderr = 'stdout' }; for i=1,#argv do spec[i]=argv[i] end; return spec end +local function classid(minor) return '1:' .. tostring(minor) end + +local function run_cmd(argv) + if type(ACTIVE_RUN_CMD) == 'function' then + local ok, out, err, code = ACTIVE_RUN_CMD(argv) + if ok == true then return true, out or '', err, code or 0 end + return nil, out or '', err or out or 'command failed', code + end + local cmd = exec_mod.command(exec_spec(argv)) + local out, st, code = fibers.perform(cmd:combined_output_op()) + if st == 'exited' and code == 0 then return true, out or '', nil, code end + return nil, out or '', out or ('exit ' .. tostring(code)), code +end + +local function try_cmd(argv) run_cmd(argv); return true end +local function must_cmd(argv, label) + local ok, out, err, code = run_cmd(argv) + if ok == true then return true, nil end + return nil, (label or table.concat(argv, ' ')) .. ': ' .. tostring(err or out or ('exit ' .. tostring(code))) +end + +local function append_fq(argv, fq) + fq = fq or {} + argv[#argv + 1] = 'fq_codel' + if fq.limit then argv[#argv+1]='limit'; argv[#argv+1]=tostring(fq.limit) end + if fq.flows then argv[#argv+1]='flows'; argv[#argv+1]=tostring(fq.flows) end + if fq.quantum then argv[#argv+1]='quantum'; argv[#argv+1]=tostring(fq.quantum) end + if fq.target then argv[#argv+1]='target'; argv[#argv+1]=tostring(fq.target) end + if fq.interval then argv[#argv+1]='interval'; argv[#argv+1]=tostring(fq.interval) end + if fq.memory_limit then argv[#argv+1]='memory_limit'; argv[#argv+1]=tostring(fq.memory_limit) end + if fq.ecn == false then argv[#argv+1]='noecn' elseif fq.ecn ~= nil then argv[#argv+1]='ecn' end +end + +local function class_argv(dev, parent, minor, cfg) + cfg = cfg or {} + local rate = tostring(cfg.rate or cfg.limit or '1gbit') + local ceil = tostring(cfg.ceil or cfg.limit or rate) + local argv = { 'tc', 'class', 'replace', 'dev', dev, 'parent', parent, 'classid', classid(minor), 'htb', 'rate', rate, 'ceil', ceil } + if cfg.burst then argv[#argv+1]='burst'; argv[#argv+1]=tostring(cfg.burst) end + if cfg.cburst then argv[#argv+1]='cburst'; argv[#argv+1]=tostring(cfg.cburst) end + if cfg.prio then argv[#argv+1]='prio'; argv[#argv+1]=tostring(cfg.prio) end + return argv +end + +local function fq_argv(dev, minor, fq) + local argv = { 'tc', 'qdisc', 'replace', 'dev', dev, 'parent', classid(minor) } + append_fq(argv, fq) + return argv +end + +local function ensure_root(dev, cfg) + try_cmd({ 'tc', 'qdisc', 'del', 'dev', dev, 'root' }) + local ok, err = must_cmd({ 'tc', 'qdisc', 'add', 'dev', dev, 'root', 'handle', '1:', 'htb', 'default', '20' }, 'add root htb ' .. dev) + if not ok then return nil, err end + ok, err = must_cmd(class_argv(dev, '1:', 1, cfg.root or { rate = '1gbit', ceil = '1gbit' }), 'root class ' .. dev) + if not ok then return nil, err end + ok, err = must_cmd(class_argv(dev, '1:1', 10, cfg.control or { rate = '1gbit', ceil = '1gbit' }), 'control class ' .. dev) + if not ok then return nil, err end + ok, err = must_cmd(class_argv(dev, '1:1', 20, cfg.client or { rate = cfg.limit or '1gbit', ceil = cfg.limit or '1gbit' }), 'client class ' .. dev) + if not ok then return nil, err end + must_cmd(fq_argv(dev, 10, cfg.control_fq_codel or cfg.fq_codel), 'control fq ' .. dev) + return must_cmd(fq_argv(dev, 20, cfg.client_fq_codel or cfg.fq_codel), 'client fq ' .. dev) +end + +local function add_mark_filters(dev, marks) + marks = marks or {} + local control = tostring(marks.control or '0x00100000') + local client = tostring(marks.client or '0x00200000') + local mask = tostring(marks.mask or '0x00f00000') + try_cmd({ 'tc', 'filter', 'del', 'dev', dev, 'parent', '1:', 'protocol', 'ip', 'prio', '10' }) + try_cmd({ 'tc', 'filter', 'del', 'dev', dev, 'parent', '1:', 'protocol', 'ip', 'prio', '20' }) + local ok, err = must_cmd({ 'tc', 'filter', 'add', 'dev', dev, 'parent', '1:', 'protocol', 'ip', 'prio', '10', 'handle', control .. '/' .. mask, 'fw', 'flowid', '1:10' }, 'control fw filter ' .. dev) + if not ok then return nil, err end + return must_cmd({ 'tc', 'filter', 'add', 'dev', dev, 'parent', '1:', 'protocol', 'ip', 'prio', '20', 'handle', client .. '/' .. mask, 'fw', 'flowid', '1:20' }, 'client fw filter ' .. dev) +end + +local function ensure_ingress_redirect(iface, ifb, marks) + try_cmd({ 'ip', 'link', 'add', ifb, 'type', 'ifb' }) + local ok, err = must_cmd({ 'ip', 'link', 'set', 'dev', ifb, 'up' }, 'ifb up') + if not ok then return nil, err end + try_cmd({ 'tc', 'qdisc', 'del', 'dev', iface, 'ingress' }) + ok, err = must_cmd({ 'tc', 'qdisc', 'add', 'dev', iface, 'handle', 'ffff:', 'ingress' }, 'add ingress ' .. iface) + if not ok then return nil, err end + local mask = tostring((marks and marks.mask) or '0x00f00000') + -- ctinfo restores the saved connmark into skb mark before the packet is redirected to the IFB. + return must_cmd({ 'tc', 'filter', 'add', 'dev', iface, 'parent', 'ffff:', 'protocol', 'ip', 'prio', '1', 'u32', 'match', 'u32', '0', '0', 'action', 'ctinfo', 'cpmark', mask, 'action', 'mirred', 'egress', 'redirect', 'dev', ifb }, 'ingress ctinfo redirect ' .. iface) +end + +local function apply_one(link, marks) + local iface = link.iface or link.device or link.interface or link.linux_interface + if type(iface) ~= 'string' or iface == '' then return nil, 'wan_mark link missing iface' end + if link.egress ~= nil and link.egress.enabled ~= false then + local ok, err = ensure_root(iface, link.egress) + if not ok then return nil, 'egress: ' .. tostring(err) end + ok, err = add_mark_filters(iface, marks) + if not ok then return nil, 'egress filters: ' .. tostring(err) end + end + if link.ingress ~= nil and link.ingress.enabled ~= false then + local ifb = link.ingress.ifb or link.ifb or ifb_name(iface) + local ok, err = ensure_ingress_redirect(iface, ifb, marks) + if not ok then return nil, 'ingress redirect: ' .. tostring(err) end + ok, err = ensure_root(ifb, link.ingress) + if not ok then return nil, 'ingress ifb: ' .. tostring(err) end + ok, err = add_mark_filters(ifb, marks) + if not ok then return nil, 'ingress filters: ' .. tostring(err) end + end + return true, nil +end + +function M.apply(req, opts) + opts = opts or {}; req = req or {} + local previous = ACTIVE_RUN_CMD + ACTIVE_RUN_CMD = opts.run_cmd or req.run_cmd + local marks = req.marks or { mask = '0x00f00000', control = '0x00100000', client = '0x00200000' } + local applied = {} + for id, link in pairs(req.links or {}) do + if is_plain_table(link) and link.kind == 'wan_mark' and link.enabled ~= false then + local ok, err = apply_one(link, marks) + if ok ~= true then ACTIVE_RUN_CMD = previous; return { ok = false, err = tostring(id) .. ': ' .. tostring(err), backend = 'openwrt' } end + applied[id] = { ok = true, iface = link.iface or link.device or link.interface, kind = 'wan_mark' } + end + end + ACTIVE_RUN_CMD = previous + return { ok = true, applied = true, changed = next(applied) ~= nil, links = applied, backend = 'openwrt' } +end + +return M diff --git a/src/services/hal/backends/network/providers/openwrt/tc_u32_shaper.lua b/src/services/hal/backends/network/providers/openwrt/tc_u32_shaper.lua new file mode 100644 index 00000000..0f1c8dab --- /dev/null +++ b/src/services/hal/backends/network/providers/openwrt/tc_u32_shaper.lua @@ -0,0 +1,1547 @@ +-- services/hal/backends/network/providers/openwrt/tc_u32_shaper.lua +-- +-- Strict per-host HTB shaper using u32 hashing + exact /32 rules. +-- +-- Features +-- * Egress shaping on iface (match dst by default) +-- * Ingress shaping via IFB (mirror ingress -> shape IFB egress) +-- * Prefix support: /20 .. /32 (up to 4096 addresses; /20 practical ceiling) +-- * Declarative host set: +-- - all_hosts=true expands to every address in the subnet +-- - hosts={} becomes an override table in all_hosts mode +-- - by default, network/broadcast are skipped for /0.. /30 +-- (include_network/include_broadcast to include them) +-- * True deltas: +-- - host filters updated only when membership changes +-- - host HTB classes updated only when changed/new +-- - per-host fq_codel leaves updated only when changed/new (removed when deleted) +-- * Bulk programming via tc -batch for filters, HTB classes, and fq_codel add/del +-- +-- Notes +-- * fq_codel leaves are applied via delete+add (no qdisc replace) for compatibility. +-- * Bucket-local filter deletes are attempted; if unsupported, the filter table is rebuilt. +-- * Lua 5.1 / LuaJIT compatible. + +local fibers = require 'fibers' +local file_mod = require 'fibers.io.file' +local exec_mod = require 'fibers.io.exec' + +local safe = require 'coxpcall' + +local bit = rawget(_G, 'bit') or require 'bit32' + +local perform = fibers.perform +local band, bor, bxor, lshift, rshift = bit.band, bit.bor, bit.bxor, bit.lshift, bit.rshift + +local M = {} + +local TC_BATCH_CHUNK = 512 + +-- Provider-owned command runner override. The OpenWrt network provider +-- serialises shaping calls, and this module also keeps backend-local STATE, so +-- the scoped override is deliberately module-local rather than a public API. +local ACTIVE_RUN_CMD = nil + +-- STATE[iface] = { egress = st, ingress = st } +-- st = { sig=..., dev=..., ifb?=..., ids=..., hosts=plan, dirty?=true } +local STATE = {} + +-------------------------------------------------------------------------------- +-- Small helpers +-------------------------------------------------------------------------------- + +local function is_plain_table(x) return type(x) == 'table' and getmetatable(x) == nil end + +local function argv_push(argv, ...) + local n = select('#', ...) + for i = 1, n do argv[#argv + 1] = select(i, ...) end +end + +local function exec_spec(argv) + local spec = { stdin = 'null', stdout = 'pipe', stderr = 'stdout' } + for i = 1, #argv do spec[i] = argv[i] end + return spec +end + +local function sorted_keys(t) + local ks = {} + for k in pairs(t or {}) do ks[#ks + 1] = k end + table.sort(ks, function(a, b) return tostring(a) < tostring(b) end) + return ks +end + +local function shallow_copy(t) + local o = {} + for k, v in pairs(t or {}) do o[k] = v end + return o +end + +-- 32-bit multiply mod 2^32 without relying on 64-bit integer arithmetic. +local function mul32(a, b) + a = tonumber(a) or 0 + b = tonumber(b) or 0 + local a_lo = band(a, 0xffff) + local a_hi = rshift(a, 16) + local b_lo = band(b, 0xffff) + local b_hi = rshift(b, 16) + + local lo = a_lo * b_lo + local mid = a_hi * b_lo + a_lo * b_hi + return band(lo + lshift(mid, 16), 0xffffffff) +end + +-- 32-bit FNV-1a (stable across runs; used for IFB suffixes). +local function fnv1a32(s) + s = tostring(s or '') + local h = 2166136261 + for i = 1, #s do + h = bxor(h, s:byte(i)) + h = mul32(h, 16777619) + end + return band(h, 0xffffffff) +end + +local function ifb_suffix_hex4(iface) + return string.format('%04x', band(fnv1a32(iface), 0xffff)) +end + +local function sanitise_ifb_name(iface) + -- Linux ifname max is 15 chars. "ifb_" + iface may overflow. + local raw = tostring(iface or '') + + -- Keep conservative characters for IFB device names. + local clean = raw:gsub('[^%w]', '_') + if clean == '' then clean = '_' end + + local base = 'ifb_' .. clean + if #base <= 15 then + return base + end + + -- Collision-avoiding truncation: ifb__ + -- Must be <= 15 chars. + local suf = ifb_suffix_hex4(raw) -- 4 chars + local prefix_len = 15 - 4 - 1 - #suf -- "ifb_" + "_" + suf + if prefix_len < 1 then prefix_len = 1 end + return 'ifb_' .. clean:sub(1, prefix_len) .. '_' .. suf +end + +local function str_contains(hay, needle) + return type(hay) == 'string' and hay:find(needle, 1, true) ~= nil +end + +local function log(logger, level, payload) + if type(logger) == 'function' then logger(level, payload) end +end + +-------------------------------------------------------------------------------- +-- Command runners (single + tc -batch) +-------------------------------------------------------------------------------- + +local function run_cmd(argv) + if type(ACTIVE_RUN_CMD) == 'function' then + local ok, out, err, code, st, sig = ACTIVE_RUN_CMD(argv) + if ok == true then + return true, out or '', err, code or 0, st or 'exited', sig + end + return nil, out or '', err or out or 'command failed', code, st, sig + end + + local cmd = exec_mod.command(exec_spec(argv)) + local out, st, code, sig, err = perform(cmd:combined_output_op()) + local ok = (st == 'exited' and code == 0) + return ok, out or '', err, code, st, sig +end + +local function is_benign_try_error(argv, out, err, _code) + local a1, a2, a3 = argv[1], argv[2], argv[3] + local msg = (out or '') .. '\n' .. tostring(err or '') + + if a1 == 'tc' and a2 == 'qdisc' and a3 == 'del' then + if str_contains(msg, 'Cannot delete qdisc with handle of zero.') then return true end + if str_contains(msg, 'Cannot find specified qdisc on specified device.') then return true end + if str_contains(msg, 'Error: Invalid handle.') then return true end + end + + if a1 == 'tc' and a2 == 'filter' and a3 == 'del' then + if str_contains(msg, 'Cannot find specified filter chain.') then return true end + if str_contains(msg, 'RTNETLINK answers: No such file or directory') then return true end + end + + if a1 == 'ip' and a2 == 'link' and a3 == 'add' then + if str_contains(msg, 'File exists') then return true end + end + + return false +end + +local function try_cmd(argv, logger) + local ok, out, err, code = run_cmd(argv) + if not ok and not is_benign_try_error(argv, out, err, code) then + log(logger, 'warn', { + what = 'cmd_failed', + cmd = table.concat(argv, ' '), + out = out ~= '' and out or nil, + err = err and tostring(err) or nil, + code = code, + }) + end + return ok, out, err, code +end + +local function must_cmd(argv, logger, label) + local ok, out, err, code = run_cmd(argv) + if not ok then + log(logger, 'error', { + what = 'cmd_failed_fatal', + label = label, + cmd = table.concat(argv, ' '), + out = out ~= '' and out or nil, + err = err and tostring(err) or nil, + code = code, + }) + return nil, (label or table.concat(argv, ' ')) .. ': ' .. tostring(err or out or ('exit ' .. tostring(code))) + end + return true, nil +end + +local function tc_batch_line(argv) + -- tc -batch lines omit the leading "tc". + if type(argv) ~= 'table' or argv[1] ~= 'tc' then + return nil, 'argv must begin with "tc"' + end + local parts = {} + for i = 2, #argv do + local s = tostring(argv[i]) + if s:find('[\r\n]') then return nil, 'newline in token' end + if s:find('%s') then return nil, 'whitespace in token: ' .. s end + parts[#parts + 1] = s + end + return table.concat(parts, ' '), nil +end + +local function run_tc_batch(cmds) + if type(cmds) ~= 'table' or #cmds == 0 then + return true, '', nil, 0 + end + + local lines = {} + for i = 1, #cmds do + local line, lerr = tc_batch_line(cmds[i]) + if not line then + return nil, '', 'batch encode failed: ' .. tostring(lerr), 255 + end + lines[#lines + 1] = line + end + + -- Use fibers.io.file tmpfile to avoid blocking Lua I/O. Keep the file open + -- (and flushed) while tc reads it; close afterwards to unlink. + local tmp, terr = file_mod.tmpfile('rw-r--r--') + if not tmp then + return nil, '', 'tmpfile failed: ' .. tostring(terr), 255 + end + + local path = tmp:filename() + if not path or path == '' then + tmp:close() + return nil, '', 'tmpfile has no filename', 255 + end + + local content = table.concat(lines, '\n') .. '\n' + local n, werr = tmp:write(content) + if not n then + tmp:close() + return nil, '', 'write batch file failed: ' .. tostring(werr), 255 + end + + local fok, ferr = tmp:flush() + if not fok then + tmp:close() + return nil, '', 'flush batch file failed: ' .. tostring(ferr), 255 + end + + local ok, out, err, code = run_cmd({ 'tc', '-batch', path }) + tmp:close() -- unlink-on-close + return ok, out, err, code +end + +local function must_tc_batch_chunked(cmds, logger, label, chunk_size) + chunk_size = tonumber(chunk_size) or TC_BATCH_CHUNK + if chunk_size < 1 then chunk_size = TC_BATCH_CHUNK end + if #cmds == 0 then return true, nil end + + local i, chunk_no = 1, 0 + while i <= #cmds do + chunk_no = chunk_no + 1 + local j = math.min(i + chunk_size - 1, #cmds) + local chunk = {} + for k = i, j do chunk[#chunk + 1] = cmds[k] end + + local ok, out, err, code = run_tc_batch(chunk) + if not ok then + log(logger, 'error', { + what = 'batch_failed_fatal', + label = label, + chunk = chunk_no, + count = #chunk, + out = out ~= '' and out or nil, + err = err and tostring(err) or nil, + code = code, + }) + return nil, (label or 'tc -batch') .. ': ' .. tostring(err or out or ('exit ' .. tostring(code))) + end + + i = j + 1 + end + + return true, nil +end + +-------------------------------------------------------------------------------- +-- IPv4 helpers +-------------------------------------------------------------------------------- + +local function parse_ipv4(s) + if type(s) ~= 'string' then return nil end + local a, b, c, d = s:match('^(%d+)%.(%d+)%.(%d+)%.(%d+)$') + a, b, c, d = tonumber(a), tonumber(b), tonumber(c), tonumber(d) + if not (a and b and c and d) then return nil end + if a > 255 or b > 255 or c > 255 or d > 255 then return nil end + return bor(lshift(a, 24), lshift(b, 16), lshift(c, 8), d) +end + +local function ipv4_to_string(u) + u = tonumber(u) or 0 + return string.format('%d.%d.%d.%d', + band(rshift(u, 24), 0xff), + band(rshift(u, 16), 0xff), + band(rshift(u, 8), 0xff), + band(u, 0xff)) +end + +local function mask_from_prefix(pfx) + pfx = tonumber(pfx) + if not pfx or pfx < 0 or pfx > 32 then return nil end + if pfx == 0 then return 0 end + local m = 0 + for i = 1, pfx do m = bor(m, lshift(1, 32 - i)) end + return m +end + +local function parse_cidr(cidr) + if type(cidr) ~= 'string' then return nil, nil, 'cidr must be a string' end + local ip_s, pfx_s = cidr:match('^([^/]+)/(%d+)$') + if not ip_s then return nil, nil, 'bad cidr format' end + local ip = parse_ipv4(ip_s) + local pfx = tonumber(pfx_s) + if not ip or not pfx then return nil, nil, 'bad cidr value' end + if pfx < 20 or pfx > 32 then return nil, nil, 'supported prefixes are /20.. /32' end + local mask = mask_from_prefix(pfx) + return band(ip, mask), pfx, nil +end + +local function host_count_from_prefix(pfx) + return 2 ^ (32 - pfx) +end + +local function ip_in_subnet(ip_u, net_u, pfx) + local mask = mask_from_prefix(pfx) + return band(ip_u, mask) == band(net_u, mask) +end + +local function host_offset(ip_u, net_u) + local off = tonumber(ip_u) - tonumber(net_u) + return (off >= 0) and off or nil +end + +local function should_skip_specials(pfx, off, total, cfg) + if pfx >= 31 then return false end + if off == 0 and not (cfg and cfg.include_network) then return true end + if off == total - 1 and not (cfg and cfg.include_broadcast) then return true end + return false +end + +-------------------------------------------------------------------------------- +-- u32 helpers +-------------------------------------------------------------------------------- + +local function u32_handle_hex(n) + n = tonumber(n) or 0 + if n < 0 then n = 0 end + return string.format('%x', n) +end + +local function u32_table_ref(handle_num) + return u32_handle_hex(handle_num) .. ':' +end + +local function u32_bucket_ref(handle_num, bucket_num) + return u32_handle_hex(handle_num) .. ':' .. u32_handle_hex(bucket_num) .. ':' +end + +-------------------------------------------------------------------------------- +-- Signatures (per-host only; built once when planning) +-------------------------------------------------------------------------------- + +local function fq_signature(fq) + if not is_plain_table(fq) then return '' end + local ks = sorted_keys(fq) + local parts = {} + for i = 1, #ks do + local k = ks[i] + parts[#parts + 1] = tostring(k) .. '=' .. tostring(fq[k]) + end + return table.concat(parts, ';') +end + +local function htb_signature(htb) + if not is_plain_table(htb) then return '' end + return table.concat({ + 'r=' .. tostring(htb.rate), + 'c=' .. tostring(htb.ceil), + 'b=' .. tostring(htb.burst), + 'cb=' .. tostring(htb.cburst), + 'p=' .. tostring(htb.prio), + 'q=' .. tostring(htb.quantum), + }, '|') +end + +-------------------------------------------------------------------------------- +-- tc layout helpers +-------------------------------------------------------------------------------- + +local function default_ids() + return { + root_major = 1, + root_class_minor = 1, + pool_minor = 20, + inner_major = 20, + inner_root_minor = 1, + default_minor = 100, + base_minor = 1000, + host_table_handle = 1, + outer_prio = 100, + link_prio = 1, + host_prio = 99, + } +end + +local function layout_mode(cfg) + local m = cfg and (cfg.mode or cfg.host_mode or cfg.burst_model) + if m == 'budgeted_peak' or m == 'borrow_to_ceil' then return m end + return 'borrow_to_ceil' +end + +local function budgeted_peak(cfg) return layout_mode(cfg) == 'budgeted_peak' end + +local function classid(major, minor) return tostring(major) .. ':' .. tostring(minor) end +local function qdisc_handle(major) return tostring(major) .. ':' end + +local function budgeted_peak_has_aggregate(cfg) + if not budgeted_peak(cfg) then return false end + if cfg and cfg.segment_aggregate == false then return false end + if cfg and cfg.segment_aggregate == true then return true end + -- Canonical segment shaping sets pool_* only when the operator configured + -- an aggregate segment download/upload limit. + return cfg and (cfg.pool_class ~= nil or cfg.pool_rate ~= nil or cfg.pool_ceil ~= nil or cfg.rate ~= nil or cfg.ceil ~= nil) +end + +local function budgeted_peak_flat(cfg) + return budgeted_peak(cfg) and not budgeted_peak_has_aggregate(cfg) +end + +local function filter_parent(ids, cfg) + if budgeted_peak_flat(cfg) then return qdisc_handle(ids.root_major) end + return qdisc_handle(ids.inner_major) +end + +local function host_classid(ids, cfg, minor) + if budgeted_peak_flat(cfg) then return classid(ids.root_major, minor) end + return classid(ids.inner_major, minor) +end + +local function host_parent_classid(ids, cfg) + if budgeted_peak_flat(cfg) then return qdisc_handle(ids.root_major) end + if budgeted_peak_has_aggregate(cfg) then return qdisc_handle(ids.inner_major) end + return classid(ids.inner_major, ids.inner_root_minor) +end + +local function default_classid(ids, cfg) + if budgeted_peak_flat(cfg) then return classid(ids.root_major, ids.default_minor) end + return classid(ids.inner_major, ids.default_minor) +end + +local function peak_qdisc_handle(rec) + return qdisc_handle(rec.minor) +end + +local function peak_classid(rec) + return classid(rec.minor, 1) +end + +local function fq_parent_classid(rec) + return rec.leaf_classid or rec.classid +end + +local function host_table_handle(ids, cfg) + -- In flat budgeted_peak the filters live under the root qdisc (normally + -- handle 1:), so avoid using table handle 1: by default. The old shaper + -- used 100:. + if budgeted_peak_flat(cfg) and ids.host_table_handle == 1 then return 256 end + return ids.host_table_handle +end + +local function bool_flag(v, yes, no) + if v == nil then return nil end + return v and yes or no +end + +local function append_fq_codel_args(argv, fq) + fq = fq or {} + argv[#argv + 1] = 'fq_codel' + + if fq.limit ~= nil then argv_push(argv, 'limit', tostring(fq.limit)) end + if fq.flows ~= nil then argv_push(argv, 'flows', tostring(fq.flows)) end + if fq.quantum ~= nil then argv_push(argv, 'quantum', tostring(fq.quantum)) end + if fq.target ~= nil then argv_push(argv, 'target', tostring(fq.target)) end + if fq.interval ~= nil then argv_push(argv, 'interval', tostring(fq.interval)) end + if fq.memory_limit ~= nil then argv_push(argv, 'memory_limit', tostring(fq.memory_limit)) end + if fq.drop_batch ~= nil then argv_push(argv, 'drop_batch', tostring(fq.drop_batch)) end + if fq.ce_threshold ~= nil then argv_push(argv, 'ce_threshold', tostring(fq.ce_threshold)) end + + local ecn = bool_flag(fq.ecn, 'ecn', 'noecn') + if ecn then argv[#argv + 1] = ecn end +end + +local function htb_class_replace_argv(dev, parent, class_id, cfg) + cfg = cfg or {} + local rate = tostring(cfg.rate or '1gbit') + local ceil = tostring(cfg.ceil or rate) + local burst = (cfg.burst ~= nil) and tostring(cfg.burst) or nil + local cburst = (cfg.cburst ~= nil) and tostring(cfg.cburst) or nil + local prio = (cfg.prio ~= nil) and tostring(cfg.prio) or nil + local quantum = (cfg.quantum ~= nil) and tostring(cfg.quantum) or nil + + local argv = { + 'tc', 'class', 'replace', 'dev', dev, + 'parent', parent, + 'classid', class_id, + 'htb', + 'rate', rate, + } + if burst then argv_push(argv, 'burst', burst) end + argv_push(argv, 'ceil', ceil) + if cburst then argv_push(argv, 'cburst', cburst) end + if prio then argv_push(argv, 'prio', prio) end + if quantum then argv_push(argv, 'quantum', quantum) end + return argv +end + +local function fq_qdisc_argv(op, dev, parent_classid, fq) + local argv = { 'tc', 'qdisc', op, 'dev', dev, 'parent', parent_classid } + append_fq_codel_args(argv, fq) + return argv +end + +-------------------------------------------------------------------------------- +-- IFB ingress redirect +-------------------------------------------------------------------------------- + +local function ensure_ifb(ifb, logger) + try_cmd({ 'ip', 'link', 'add', ifb, 'type', 'ifb' }, logger) + return must_cmd({ 'ip', 'link', 'set', 'dev', ifb, 'up' }, logger, 'ifb up') +end + +local function ensure_ingress_redirect(iface, ifb, logger) + try_cmd({ 'tc', 'qdisc', 'del', 'dev', iface, 'ingress' }, logger) + local ok, err = must_cmd({ 'tc', 'qdisc', 'add', 'dev', iface, 'handle', 'ffff:', 'ingress' }, logger, + 'add ingress qdisc') + if not ok then return nil, err end + + return must_cmd({ + 'tc', 'filter', 'add', 'dev', iface, 'parent', 'ffff:', + 'protocol', 'ip', 'u32', + 'match', 'u32', '0', '0', + 'action', 'mirred', 'egress', 'redirect', 'dev', ifb, + }, logger, 'add mirred redirect') +end + +-------------------------------------------------------------------------------- +-- Scaffold + top-level maintenance +-------------------------------------------------------------------------------- + +local function direction_params(kind, cfg) + if kind == 'egress' then + local m = cfg.match or 'dst' + return { match_field = m, hash_at = (m == 'src') and 12 or 16 } + end + local m = cfg.match or 'src' + return { match_field = m, hash_at = (m == 'src') and 12 or 16 } +end + +local function scaffold_signature(kind, spec, cfg, ids, dev) + local dp = direction_params(kind, cfg) + return table.concat({ + kind, dev, spec.cidr, dp.match_field, tostring(dp.hash_at), layout_mode(cfg), + budgeted_peak_has_aggregate(cfg) and 'agg' or 'noagg', + tostring(ids.root_major), tostring(ids.pool_minor), + tostring(ids.inner_major), tostring(ids.inner_root_minor), + tostring(ids.default_minor), tostring(ids.base_minor), + tostring(host_table_handle(ids, cfg)), + tostring(ids.outer_prio), tostring(ids.link_prio), tostring(ids.host_prio), + }, '|') +end + +local function reconcile_default_fq(dev, ids, cfg, logger) + local fq_cfg = cfg and cfg.default_fq_codel + local parent = default_classid(ids, cfg) + if fq_cfg == nil or fq_cfg == false then + try_cmd({ 'tc', 'qdisc', 'del', 'dev', dev, 'parent', parent }, logger) + return true, nil + end + -- delete+add (no replace) + try_cmd({ 'tc', 'qdisc', 'del', 'dev', dev, 'parent', parent }, logger) + return must_cmd(fq_qdisc_argv('add', dev, parent, fq_cfg), logger, 'default fq_codel add') +end + +local function rebuild_scaffold(kind, spec, cfg, dev, ids, logger) + local dp = direction_params(kind, cfg) + local net_s = ipv4_to_string(spec.net_u) .. '/' .. tostring(spec.pfx) + + try_cmd({ 'tc', 'qdisc', 'del', 'dev', dev, 'root' }, logger) + + if budgeted_peak(cfg) then + if budgeted_peak_has_aggregate(cfg) then + local cmds = { + { 'tc', 'qdisc', 'add', 'dev', dev, 'root', + 'handle', qdisc_handle(ids.root_major), + 'htb', 'default', tostring(ids.root_class_minor) }, + + htb_class_replace_argv(dev, qdisc_handle(ids.root_major), + classid(ids.root_major, ids.root_class_minor), + cfg.root_class or { rate = (cfg.root_rate or '1gbit'), ceil = (cfg.root_ceil or cfg.root_rate or '1gbit') }), + htb_class_replace_argv(dev, qdisc_handle(ids.root_major), + classid(ids.root_major, ids.pool_minor), + cfg.pool_class or { + rate = (cfg.pool_rate or cfg.rate or '1gbit'), + ceil = (cfg.pool_ceil or cfg.ceil or cfg.pool_rate or cfg.rate or '1gbit'), + burst = cfg.pool_burst, + cburst = cfg.pool_cburst, + }), + { 'tc', 'qdisc', 'add', 'dev', dev, + 'parent', classid(ids.root_major, ids.pool_minor), + 'handle', qdisc_handle(ids.inner_major), + 'htb', 'default', tostring(ids.default_minor) }, + htb_class_replace_argv(dev, qdisc_handle(ids.inner_major), + classid(ids.inner_major, ids.default_minor), + cfg.default_class or { + rate = (cfg.default_rate or cfg.host_rate or '1gbit'), + ceil = (cfg.default_ceil or cfg.host_ceil or cfg.default_rate or cfg.host_rate or '1gbit'), + burst = (cfg.default_burst or cfg.host_burst), + cburst = (cfg.default_cburst or cfg.host_cburst), + }), + { 'tc', 'filter', 'add', 'dev', dev, + 'parent', qdisc_handle(ids.root_major), + 'protocol', 'ip', + 'prio', tostring(ids.outer_prio), + 'u32', 'match', 'ip', dp.match_field, net_s, + 'flowid', classid(ids.root_major, ids.pool_minor) }, + } + local ok, err = must_tc_batch_chunked(cmds, logger, 'rebuild budgeted_peak aggregate scaffold', 128) + if not ok then return nil, err end + return reconcile_default_fq(dev, ids, cfg, logger) + end + + local cmds = { + { 'tc', 'qdisc', 'add', 'dev', dev, 'root', + 'handle', qdisc_handle(ids.root_major), + 'htb', 'default', tostring(ids.default_minor) }, + htb_class_replace_argv(dev, qdisc_handle(ids.root_major), + classid(ids.root_major, ids.default_minor), + cfg.default_class or { + rate = (cfg.default_rate or cfg.host_rate or '1gbit'), + ceil = (cfg.default_ceil or cfg.host_ceil or cfg.default_rate or cfg.host_rate or '1gbit'), + burst = (cfg.default_burst or cfg.host_burst), + cburst = (cfg.default_cburst or cfg.host_cburst), + }), + } + local ok, err = must_tc_batch_chunked(cmds, logger, 'rebuild budgeted_peak flat scaffold', 128) + if not ok then return nil, err end + return reconcile_default_fq(dev, ids, cfg, logger) + end + + local cmds = { + { 'tc', 'qdisc', 'add', 'dev', dev, 'root', + 'handle', qdisc_handle(ids.root_major), + 'htb', 'default', tostring(ids.root_class_minor) }, + + htb_class_replace_argv(dev, qdisc_handle(ids.root_major), + classid(ids.root_major, ids.root_class_minor), + cfg.root_class or { rate = (cfg.root_rate or '1gbit'), ceil = (cfg.root_ceil or cfg.root_rate or '1gbit') }), + htb_class_replace_argv(dev, classid(ids.root_major, ids.root_class_minor), + classid(ids.root_major, ids.pool_minor), + cfg.pool_class or { + rate = (cfg.pool_rate or cfg.rate or '1gbit'), + ceil = (cfg.pool_ceil or cfg.ceil or cfg.pool_rate or cfg.rate or '1gbit'), + burst = cfg.pool_burst, + cburst = cfg.pool_cburst, + }), + + { 'tc', 'qdisc', 'add', 'dev', dev, + 'parent', classid(ids.root_major, ids.pool_minor), + 'handle', qdisc_handle(ids.inner_major), + 'htb', 'default', tostring(ids.default_minor) }, + + htb_class_replace_argv(dev, qdisc_handle(ids.inner_major), + classid(ids.inner_major, ids.inner_root_minor), + { + rate = (cfg.pool_rate or cfg.rate or '1gbit'), + ceil = (cfg.pool_ceil or cfg.ceil or cfg.pool_rate or cfg.rate or '1gbit'), + burst = cfg.pool_burst, + cburst = cfg.pool_cburst, + }), + + htb_class_replace_argv(dev, classid(ids.inner_major, ids.inner_root_minor), + classid(ids.inner_major, ids.default_minor), + cfg.default_class or { + rate = (cfg.default_rate or cfg.host_rate or '1gbit'), + ceil = (cfg.default_ceil or cfg.host_ceil or cfg.default_rate or cfg.host_rate or '1gbit'), + burst = (cfg.default_burst or cfg.host_burst), + cburst = (cfg.default_cburst or cfg.host_cburst), + }), + + { 'tc', 'filter', 'add', 'dev', dev, + 'parent', qdisc_handle(ids.root_major), + 'protocol', 'ip', + 'prio', tostring(ids.outer_prio), + 'u32', 'match', 'ip', dp.match_field, net_s, + 'flowid', classid(ids.root_major, ids.pool_minor) }, + } + + local ok, err = must_tc_batch_chunked(cmds, logger, 'rebuild scaffold', 128) + if not ok then return nil, err end + return reconcile_default_fq(dev, ids, cfg, logger) +end + +local function apply_top_classes(dev, ids, cfg, logger) + if budgeted_peak(cfg) then + local cmds = {} + if budgeted_peak_has_aggregate(cfg) then + cmds[#cmds + 1] = htb_class_replace_argv(dev, qdisc_handle(ids.root_major), + classid(ids.root_major, ids.root_class_minor), + cfg.root_class or { rate = (cfg.root_rate or '1gbit'), ceil = (cfg.root_ceil or cfg.root_rate or '1gbit') }) + cmds[#cmds + 1] = htb_class_replace_argv(dev, qdisc_handle(ids.root_major), + classid(ids.root_major, ids.pool_minor), + cfg.pool_class or { + rate = (cfg.pool_rate or cfg.rate or '1gbit'), + ceil = (cfg.pool_ceil or cfg.ceil or cfg.pool_rate or cfg.rate or '1gbit'), + burst = cfg.pool_burst, + cburst = cfg.pool_cburst, + }) + cmds[#cmds + 1] = htb_class_replace_argv(dev, qdisc_handle(ids.inner_major), + classid(ids.inner_major, ids.default_minor), + cfg.default_class or { + rate = (cfg.default_rate or cfg.host_rate or '1gbit'), + ceil = (cfg.default_ceil or cfg.host_ceil or cfg.default_rate or cfg.host_rate or '1gbit'), + burst = (cfg.default_burst or cfg.host_burst), + cburst = (cfg.default_cburst or cfg.host_cburst), + }) + else + cmds[#cmds + 1] = htb_class_replace_argv(dev, qdisc_handle(ids.root_major), + classid(ids.root_major, ids.default_minor), + cfg.default_class or { + rate = (cfg.default_rate or cfg.host_rate or '1gbit'), + ceil = (cfg.default_ceil or cfg.host_ceil or cfg.default_rate or cfg.host_rate or '1gbit'), + burst = (cfg.default_burst or cfg.host_burst), + cburst = (cfg.default_cburst or cfg.host_cburst), + }) + end + local ok, err = must_tc_batch_chunked(cmds, logger, 'budgeted_peak top class refresh', 128) + if not ok then return nil, err end + return reconcile_default_fq(dev, ids, cfg, logger) + end + + local cmds = { + -- Refresh root class as well; otherwise root_rate/root_ceil changes only + -- take effect on a full scaffold rebuild. + htb_class_replace_argv(dev, qdisc_handle(ids.root_major), + classid(ids.root_major, ids.root_class_minor), + cfg.root_class or { + rate = (cfg.root_rate or '1gbit'), + ceil = (cfg.root_ceil or cfg.root_rate or '1gbit'), + }), + htb_class_replace_argv(dev, classid(ids.root_major, ids.root_class_minor), + classid(ids.root_major, ids.pool_minor), + cfg.pool_class or { + rate = (cfg.pool_rate or cfg.rate or '1gbit'), + ceil = (cfg.pool_ceil or cfg.ceil or cfg.pool_rate or cfg.rate or '1gbit'), + burst = cfg.pool_burst, + cburst = cfg.pool_cburst, + }), + htb_class_replace_argv(dev, qdisc_handle(ids.inner_major), + classid(ids.inner_major, ids.inner_root_minor), + { + rate = (cfg.pool_rate or cfg.rate or '1gbit'), + ceil = (cfg.pool_ceil or cfg.ceil or cfg.pool_rate or cfg.rate or '1gbit'), + burst = cfg.pool_burst, + cburst = cfg.pool_cburst, + }), + htb_class_replace_argv(dev, classid(ids.inner_major, ids.inner_root_minor), + classid(ids.inner_major, ids.default_minor), + cfg.default_class or { + rate = (cfg.default_rate or cfg.host_rate or '1gbit'), + ceil = (cfg.default_ceil or cfg.host_ceil or cfg.default_rate or cfg.host_rate or '1gbit'), + burst = (cfg.default_burst or cfg.host_burst), + cburst = (cfg.default_cburst or cfg.host_cburst), + }), + } + local ok, err = must_tc_batch_chunked(cmds, logger, 'top class refresh', 128) + if not ok then return nil, err end + return reconcile_default_fq(dev, ids, cfg, logger) +end + +-------------------------------------------------------------------------------- +-- Host plan +-------------------------------------------------------------------------------- + +local function build_host_plan(spec, cfg, ids) + local hosts_raw = cfg.hosts + if hosts_raw == nil then hosts_raw = {} end + if not is_plain_table(hosts_raw) then + return nil, 'hosts must be a table keyed by IPv4 string' + end + + local overrides = {} + local ok_keys = sorted_keys(hosts_raw) + for i = 1, #ok_keys do + local ip_s = ok_keys[i] + local hcfg = hosts_raw[ip_s] + if not is_plain_table(hcfg) then + return nil, 'host entry for ' .. tostring(ip_s) .. ' must be a table' + end + local ip_u = parse_ipv4(ip_s) + if not ip_u then return nil, 'invalid host IP ' .. tostring(ip_s) end + if not ip_in_subnet(ip_u, spec.net_u, spec.pfx) then + return nil, 'host ' .. ip_s .. ' is outside subnet ' .. spec.cidr + end + overrides[ip_s] = hcfg + end + + local function make_rec(ip_s, hcfg) + local ip_u = parse_ipv4(ip_s) + if not ip_u then return nil, 'invalid host IP ' .. tostring(ip_s) end + if not ip_in_subnet(ip_u, spec.net_u, spec.pfx) then + return nil, 'host ' .. ip_s .. ' is outside subnet ' .. spec.cidr + end + + local off = host_offset(ip_u, spec.net_u) + if off == nil then return nil, 'failed host offset for ' .. ip_s end + if off >= host_count_from_prefix(spec.pfx) then return nil, 'host offset out of range for ' .. ip_s end + + local bucket = band(ip_u, 0xff) + local minor = ids.base_minor + off + if minor == ids.default_minor then + return nil, 'classid collision for host ' .. ip_s .. '; change base_minor/default_minor' + end + if minor > 65534 then + return nil, 'class minor too large for host ' .. ip_s + end + + local sustained = hcfg.sustained_rate or hcfg.rate or cfg.host_rate or '1mbit' + local peak = hcfg.peak_rate or hcfg.peak or hcfg.ceil or cfg.host_ceil or sustained + local burst = hcfg.burst_budget or hcfg.burst or cfg.host_burst + local cburst = hcfg.cburst or cfg.host_cburst or burst + + local htb + local peak_htb = nil + if budgeted_peak(cfg) then + -- The host budget class enforces long-term behaviour. Its ceil is + -- deliberately the sustained rate; the child peak class caps how + -- quickly the burst allowance may be spent. + htb = { + rate = sustained, + ceil = hcfg.sustained_ceil or sustained, + burst = burst, + cburst = cburst, + prio = hcfg.prio or cfg.host_prio, + quantum = hcfg.quantum or cfg.host_quantum, + } + peak_htb = { + rate = peak, + ceil = hcfg.peak_ceil or peak, + burst = hcfg.peak_burst or cfg.host_peak_burst, + cburst = hcfg.peak_cburst or cfg.host_peak_cburst, + prio = hcfg.peak_prio or cfg.host_peak_prio, + quantum = hcfg.peak_quantum or cfg.host_peak_quantum, + } + else + htb = { + rate = sustained, + ceil = peak, + burst = burst, + cburst = cburst, + prio = hcfg.prio or cfg.host_prio, + quantum = hcfg.quantum or cfg.host_quantum, + } + end + + local fq = nil + if hcfg.fq_codel == false then + fq = false + elseif is_plain_table(hcfg.fq_codel) or is_plain_table(cfg.fq_codel) then + fq = shallow_copy(cfg.fq_codel or {}) + if is_plain_table(hcfg.fq_codel) then + for k, v in pairs(hcfg.fq_codel) do fq[k] = v end + end + end + + local rec = { + ip_s = ip_s, + ip_u = ip_u, + offset = off, + bucket = bucket, + minor = minor, + classid = host_classid(ids, cfg, minor), + htb = htb, + peak_htb = peak_htb, + fq = fq, + cfg = cfg, + } + if budgeted_peak(cfg) then + rec.peak_qdisc = peak_qdisc_handle(rec) + rec.peak_classid = peak_classid(rec) + rec.leaf_classid = rec.peak_classid + rec.htb_sig = htb_signature(htb) .. '|peak:' .. htb_signature(peak_htb) + else + rec.leaf_classid = rec.classid + rec.htb_sig = htb_signature(htb) + end + rec.fq_sig = fq_signature(fq) + return rec, nil + end + + local out = {} + if cfg.all_hosts == true then + local total = host_count_from_prefix(spec.pfx) + for off = 0, total - 1 do + if not should_skip_specials(spec.pfx, off, total, cfg) then + local ip_s = ipv4_to_string((tonumber(spec.net_u) or 0) + off) + local rec, err = make_rec(ip_s, overrides[ip_s] or {}) + if not rec then return nil, err end + out[ip_s] = rec + end + end + else + for i = 1, #ok_keys do + local ip_s = ok_keys[i] + local rec, err = make_rec(ip_s, overrides[ip_s] or {}) + if not rec then return nil, err end + out[ip_s] = rec + end + end + + return out, nil +end + +-------------------------------------------------------------------------------- +-- Deltas (membership + per-host) +-------------------------------------------------------------------------------- + +local function host_htb_changed(old, new_) + if not old then return true end + if old.classid ~= new_.classid then return true end + return old.htb_sig ~= new_.htb_sig +end + +local function host_fq_changed(old, new_) + if not old then + return is_plain_table(new_.fq) or (new_.fq == false) + end + if old.classid ~= new_.classid then return true end + + local okind = (old.fq == false) and 'false' or (is_plain_table(old.fq) and 'table' or 'nil') + local nkind = (new_.fq == false) and 'false' or (is_plain_table(new_.fq) and 'table' or 'nil') + if okind ~= nkind then return true end + + return old.fq_sig ~= new_.fq_sig +end + +local function membership_delta(prev_plan, new_plan) + prev_plan, new_plan = prev_plan or {}, new_plan or {} + + local affected = {} + local changed = false + + -- additions + bucket/class changes + for ip_s, nrec in pairs(new_plan) do + local orec = prev_plan[ip_s] + if not orec then + affected[nrec.bucket] = true + changed = true + else + if orec.bucket ~= nrec.bucket or orec.classid ~= nrec.classid then + affected[orec.bucket] = true + affected[nrec.bucket] = true + changed = true + end + end + end + + -- removals + for ip_s, orec in pairs(prev_plan) do + if not new_plan[ip_s] then + affected[orec.bucket] = true + changed = true + end + end + + return changed, affected +end + +-------------------------------------------------------------------------------- +-- Host filters (full + delta) +-------------------------------------------------------------------------------- + +local function host_rule_argv(dev, ids, match_field, rec) + return { + 'tc', 'filter', 'add', 'dev', dev, + 'parent', filter_parent(ids, rec.cfg or {}), + 'protocol', 'ip', + 'prio', tostring(ids.host_prio), + 'u32', + 'ht', u32_bucket_ref(host_table_handle(ids, rec.cfg or {}), rec.bucket), + 'match', 'ip', match_field, rec.ip_s .. '/32', + 'flowid', rec.classid, + } +end + +local function rebuild_host_filters_full(kind, spec, cfg, dev, ids, plan, logger) + local dp = direction_params(kind, cfg) + local net_s = ipv4_to_string(spec.net_u) .. '/' .. tostring(spec.pfx) + + if ids.link_prio == ids.host_prio then + return nil, 'link_prio and host_prio must differ' + end + + -- best-effort clear the filter priorities we own + try_cmd( + { 'tc', 'filter', 'del', 'dev', dev, 'parent', filter_parent(ids, cfg), 'protocol', 'ip', 'prio', tostring( + ids + .link_prio) }, logger) + try_cmd( + { 'tc', 'filter', 'del', 'dev', dev, 'parent', filter_parent(ids, cfg), 'protocol', 'ip', 'prio', tostring( + ids + .host_prio) }, logger) + + local batch = { + { + 'tc', 'filter', 'add', 'dev', dev, + 'parent', filter_parent(ids, cfg), + 'protocol', 'ip', + 'prio', tostring(ids.host_prio), + 'handle', u32_table_ref(host_table_handle(ids, cfg)), + 'u32', 'divisor', '256', + }, + { + 'tc', 'filter', 'add', 'dev', dev, + 'parent', filter_parent(ids, cfg), + 'protocol', 'ip', + 'prio', tostring(ids.link_prio), + 'u32', + 'link', u32_table_ref(host_table_handle(ids, cfg)), + 'hashkey', 'mask', '0x000000ff', 'at', tostring(dp.hash_at), + 'match', 'ip', dp.match_field, net_s, + }, + } + + local ips = sorted_keys(plan) + for i = 1, #ips do + batch[#batch + 1] = host_rule_argv(dev, ids, dp.match_field, plan[ips[i]]) + end + + return must_tc_batch_chunked(batch, logger, 'rebuild host filters') +end + +local function delete_bucket_rules(dev, ids, cfg, bucket, logger) + local argv = { + 'tc', 'filter', 'del', 'dev', dev, + 'parent', filter_parent(ids, cfg), + 'protocol', 'ip', + 'prio', tostring(ids.host_prio), + 'u32', + 'ht', u32_bucket_ref(host_table_handle(ids, cfg), bucket), + } + + local ok, out, err, code = run_cmd(argv) + if ok or is_benign_try_error(argv, out, err, code) then + return true, nil + end + + local msg = tostring(err or out or ('exit ' .. tostring(code))) + if str_contains(msg, 'Illegal "ht"') or str_contains(msg, 'Illegal \"ht\"') then + return nil, 'bucket_delete_unsupported' + end + + log(logger, 'warn', { + what = 'cmd_failed', + cmd = table.concat(argv, ' '), + out = out ~= '' and out or nil, + err = err and tostring(err) or nil, + code = code, + }) + + return nil, 'bucket_delete_failed' +end + +local function reconcile_host_filters_delta(kind, spec, cfg, dev, ids, prev_plan, new_plan, logger) + local changed, affected = membership_delta(prev_plan, new_plan) + if not changed then return true, nil end + + -- Attempt bucket-local delete; fall back to full rebuild if unsupported. + local buckets = sorted_keys(affected) + for i = 1, #buckets do + local b = tonumber(buckets[i]) + local ok, _ = delete_bucket_rules(dev, ids, cfg, b, logger) + if not ok then + return rebuild_host_filters_full(kind, spec, cfg, dev, ids, new_plan, logger) + end + end + + local match_field = direction_params(kind, cfg).match_field + local batch = {} + local ips = sorted_keys(new_plan) + for i = 1, #ips do + local rec = new_plan[ips[i]] + if affected[rec.bucket] then + batch[#batch + 1] = host_rule_argv(dev, ids, match_field, rec) + end + end + + return must_tc_batch_chunked(batch, logger, 'reconcile host filters delta') +end + +-------------------------------------------------------------------------------- +-- Host classes + fq_codel leaves (true deltas; batched) +-------------------------------------------------------------------------------- + +local function peak_qdisc_argv(op, dev, rec) + return { 'tc', 'qdisc', op, 'dev', dev, 'parent', rec.classid, 'handle', rec.peak_qdisc, 'htb', 'default', '1' } +end + +local function reconcile_host_classes_and_fq(dev, ids, prev_plan, new_plan, logger) + prev_plan = prev_plan or {} + + local qdisc_del, class_del = {}, {} + local budget_class_upd, peak_qdisc_upd, peak_class_upd, qdisc_add = {}, {}, {}, {} + + local function del_host_qdiscs(rec) + if not rec then return end + if is_plain_table(rec.fq) then + qdisc_del[#qdisc_del + 1] = { 'tc', 'qdisc', 'del', 'dev', dev, 'parent', fq_parent_classid(rec) } + end + if rec.peak_qdisc then + qdisc_del[#qdisc_del + 1] = { 'tc', 'qdisc', 'del', 'dev', dev, 'parent', rec.classid } + end + end + + -- stale removals + for ip_s, old in pairs(prev_plan) do + if not new_plan[ip_s] then + del_host_qdiscs(old) + class_del[#class_del + 1] = { 'tc', 'class', 'del', 'dev', dev, 'classid', old.classid } + end + end + + -- changes/additions + local ips = sorted_keys(new_plan) + for i = 1, #ips do + local rec = new_plan[ips[i]] + local old = prev_plan[rec.ip_s] + + -- If the host remains present but its classid or leaf topology changes, + -- remove the old per-host qdiscs before replacing classes. + local topology_changed = old and (old.classid ~= rec.classid or old.leaf_classid ~= rec.leaf_classid or old.peak_qdisc ~= rec.peak_qdisc) + if topology_changed then + del_host_qdiscs(old) + class_del[#class_del + 1] = { 'tc', 'class', 'del', 'dev', dev, 'classid', old.classid } + end + + if host_htb_changed(old, rec) then + budget_class_upd[#budget_class_upd + 1] = + htb_class_replace_argv(dev, host_parent_classid(ids, rec.cfg or {}), rec.classid, rec.htb) + if rec.peak_htb then + -- The peak qdisc is only a container for the peak class. On OpenWrt, + -- `tc qdisc replace ... htb` against an existing classful HTB qdisc + -- with children can fail with "Change operation not supported by + -- specified qdisc". For steady-state rate changes, keep the + -- existing qdisc and replace only the classes. Add the qdisc only + -- when the host is new or the old topology was explicitly removed. + if (not old) or topology_changed then + peak_qdisc_upd[#peak_qdisc_upd + 1] = peak_qdisc_argv('add', dev, rec) + end + peak_class_upd[#peak_class_upd + 1] = htb_class_replace_argv(dev, rec.peak_qdisc, rec.peak_classid, rec.peak_htb) + end + end + + if host_fq_changed(old, rec) then + if old and is_plain_table(old.fq) and not (old.classid ~= rec.classid or old.leaf_classid ~= rec.leaf_classid or old.peak_qdisc ~= rec.peak_qdisc) then + qdisc_del[#qdisc_del + 1] = { 'tc', 'qdisc', 'del', 'dev', dev, 'parent', fq_parent_classid(old) } + end + if is_plain_table(rec.fq) then + qdisc_add[#qdisc_add + 1] = fq_qdisc_argv('add', dev, fq_parent_classid(rec), rec.fq) + end + end + end + + -- Phase 1: qdisc deletes (batch, with best-effort fallback) + if #qdisc_del > 0 then + local ok, _ = must_tc_batch_chunked(qdisc_del, logger, 'host qdisc deletes', 256) + if not ok then + for i = 1, #qdisc_del do try_cmd(qdisc_del[i], logger) end + end + end + + -- Phase 2: class deletes + if #class_del > 0 then + local ok, err = must_tc_batch_chunked(class_del, logger, 'class deletes', 512) + if not ok then return nil, err end + end + + -- Phase 3: host budget class updates + if #budget_class_upd > 0 then + local ok, err = must_tc_batch_chunked(budget_class_upd, logger, 'host budget class updates', 512) + if not ok then return nil, err end + end + + -- Phase 4: per-host peak qdisc updates + if #peak_qdisc_upd > 0 then + local ok, err = must_tc_batch_chunked(peak_qdisc_upd, logger, 'host peak qdisc updates', 256) + if not ok then return nil, err end + end + + -- Phase 5: per-host peak class updates + if #peak_class_upd > 0 then + local ok, err = must_tc_batch_chunked(peak_class_upd, logger, 'host peak class updates', 512) + if not ok then return nil, err end + end + + -- Phase 6: fq_codel leaf adds + if #qdisc_add > 0 then + local ok, err = must_tc_batch_chunked(qdisc_add, logger, 'fq qdisc adds', 256) + if not ok then return nil, err end + end + + return true, nil +end + +-------------------------------------------------------------------------------- +-- Per-direction apply/clear +-------------------------------------------------------------------------------- + +local function prune_iface_state(iface) + local st = STATE[iface] + if st and st.egress == nil and st.ingress == nil then + STATE[iface] = nil + end +end + +local function mark_dirty(st) + -- Force a full rebuild next run (scaffold + filters), to avoid state drift + -- after partial kernel programming. + if not st then return end + st.dirty = true + st.sig = nil + st.hosts = nil + st.redirect_ready = nil +end + +local function fail_dirty(st, err) + mark_dirty(st) + return nil, err +end + +local function clear_direction(iface, kind, cfg, logger) + local per_iface = STATE[iface] + local prev = per_iface and per_iface[kind] or nil + + if kind == 'egress' then + try_cmd({ 'tc', 'qdisc', 'del', 'dev', iface, 'root' }, logger) + if per_iface then per_iface.egress = nil end + prune_iface_state(iface) + return true, nil + end + + local ifb + if is_plain_table(cfg) and type(cfg.ifb) == 'string' and cfg.ifb ~= '' then + ifb = cfg.ifb + elseif prev and type(prev.ifb) == 'string' and prev.ifb ~= '' then + ifb = prev.ifb + else + ifb = sanitise_ifb_name(iface) + end + + try_cmd({ 'tc', 'qdisc', 'del', 'dev', iface, 'ingress' }, logger) + try_cmd({ 'tc', 'qdisc', 'del', 'dev', ifb, 'root' }, logger) + + if per_iface then per_iface.ingress = nil end + prune_iface_state(iface) + return true, nil +end + +local function apply_direction(iface, kind, spec, cfg, logger) + if cfg == nil or cfg.enabled == false then + return clear_direction(iface, kind, cfg, logger) + end + + local ids = shallow_copy(default_ids()) + for k, v in pairs(cfg.ids or {}) do ids[k] = v end + + -- Ensure we always have a state record we can mark dirty on error. + local per_iface = STATE[iface] or {} + STATE[iface] = per_iface + local st = per_iface[kind] or {} + per_iface[kind] = st + + local dev, ifb = iface, nil + if kind == 'ingress' then + -- Prefer explicit config; otherwise reuse previous IFB if known; otherwise derive. + ifb = cfg.ifb or st.ifb or sanitise_ifb_name(iface) + + local ok, err = ensure_ifb(ifb, logger) + if not ok then return fail_dirty(st, err) end + + -- The ingress redirect (ingress qdisc + mirred filter) is disruptive to rebuild. + -- Under the assumption that nothing else mutates tc state, we only need to + -- program it once per iface/ifb pair, and again after an internal failure + -- (mark_dirty clears redirect_ready) or if the IFB name changes. + if st.redirect_ready ~= true or st.ifb ~= ifb then + ok, err = ensure_ingress_redirect(iface, ifb, logger) + if not ok then return fail_dirty(st, err) end + st.redirect_ready = true + st.ifb = ifb + end + + dev = ifb + end + + local sig = scaffold_signature(kind, spec, cfg, ids, dev) + local need_scaffold = (st.dirty == true) or (st.sig ~= sig) or (st.dev ~= dev) + + if need_scaffold then + local ok, err = rebuild_scaffold(kind, spec, cfg, dev, ids, logger) + if not ok then return fail_dirty(st, err) end + st.sig, st.dev, st.ifb, st.ids, st.hosts = sig, dev, ifb, shallow_copy(ids), nil + st.dirty = nil + else + local ok, err = apply_top_classes(dev, ids, cfg, logger) + if not ok then return fail_dirty(st, err) end + end + + local plan, perr = build_host_plan(spec, cfg, ids) + if not plan then return fail_dirty(st, perr) end + + local prev_plan = st.hosts or {} + + -- Membership drives filter work; limits drive class/fq work. + local mem_changed, _affected = membership_delta(prev_plan, plan) + local lim_changed = false + + if mem_changed then + local ok, err + if need_scaffold or not st.hosts then + ok, err = rebuild_host_filters_full(kind, spec, cfg, dev, ids, plan, logger) + else + ok, err = reconcile_host_filters_delta(kind, spec, cfg, dev, ids, prev_plan, plan, logger) + end + if not ok then return fail_dirty(st, err) end + lim_changed = true -- membership implies class/fq reconciliation work may be needed + else + -- Check per-host deltas only when membership is stable. + for ip_s, rec in pairs(plan) do + local old = prev_plan[ip_s] + if host_htb_changed(old, rec) or host_fq_changed(old, rec) then + lim_changed = true + break + end + end + end + + -- Always reconcile when membership changed (stale removals), otherwise only if limits changed. + if mem_changed or lim_changed then + local ok, err = reconcile_host_classes_and_fq(dev, ids, prev_plan, plan, logger) + if not ok then return fail_dirty(st, err) end + end + + st.hosts = plan + st.ids = shallow_copy(ids) + st.dev = dev + st.ifb = ifb + st.dirty = nil + + return true, nil +end + +------------------------------------------------------------------------ +-- Public API +------------------------------------------------------------------------ + +--- Apply bi-directional per-host shaping. +--- +--- spec = { +--- iface = "eth2", +--- subnet = "10.12.0.0/20", +--- log = function(level, payload) ... end, -- optional +--- +--- egress = { +--- enabled = true, +--- match = "dst", -- default "dst" +--- pool_rate = "100mbit", +--- pool_ceil = "100mbit", +--- pool_burst = "64kb", +--- host_rate = "2mbit", -- defaults for hosts +--- host_ceil = "5mbit", +--- host_burst = "32kb", +--- host_cburst = "32kb", +--- fq_codel = { flows=1024, limit=10240, memory_limit="16Mb", target="5ms", interval="100ms" }, +--- default_rate = "100mbit", -- unmatched hosts in subnet (optional) +--- default_ceil = "100mbit", +--- +--- -- Expand to every host in the subnet (except network/broadcast by default for /0../30) +--- all_hosts = true, +--- include_network = false, -- optional; default false +--- include_broadcast = false, -- optional; default false +--- +--- -- Optional per-host overrides (also the full host set when all_hosts=false) +--- hosts = { +--- ["10.12.0.2"] = { rate="1mbit", ceil="2mbit", burst="16kb" }, +--- ["10.12.15.2"] = { rate="3mbit", ceil="4mbit", fq_codel={ flows=2048, memory_limit="32Mb" } }, +--- }, +--- }, +--- +--- ingress = { +--- enabled = true, +--- ifb = "ifb_eth2", -- optional (auto if omitted) +--- match = "src", -- choose "dst" if shaping downloads to LAN hosts on WAN IFB +--- pool_rate = "100mbit", +--- pool_ceil = "100mbit", +--- host_rate = "2mbit", +--- host_ceil = "5mbit", +--- host_burst = "32kb", +--- fq_codel = { flows=1024, limit=10240, memory_limit="16Mb" }, +--- all_hosts = true, +--- hosts = { ... same shape (overrides) ... } +--- } +--- } +--- +--- Reconciliation behaviour: +--- * Host filters are updated only when membership changes (adds/removes). +--- * Host HTB classes are updated only for changed/new hosts. +--- * Per-host fq_codel leaves are updated only for changed/new hosts (and removed for deleted hosts). +local function apply_one(spec) + spec = spec or {} + local iface = spec.iface + if type(iface) ~= 'string' or iface == '' then + return nil, 'iface is required' + end + + local cidr = spec.subnet or spec.cidr + local net_u, pfx, cerr = parse_cidr(cidr) + if not net_u then return nil, cerr end + + local logger = (type(spec.log) == 'function') and spec.log or nil + local parsed = { cidr = cidr, net_u = net_u, pfx = pfx } + + local ok, err = apply_direction(iface, 'egress', parsed, spec.egress, logger) + if not ok then return nil, 'egress: ' .. tostring(err) end + + ok, err = apply_direction(iface, 'ingress', parsed, spec.ingress, logger) + if not ok then return nil, 'ingress: ' .. tostring(err) end + + return true, nil +end + + +local function first_host_cidr(spec) + local function scan(cfg) + if not is_plain_table(cfg) or not is_plain_table(cfg.hosts) then return nil end + local ks = sorted_keys(cfg.hosts) + for i = 1, #ks do + local ip = tostring(ks[i]) + if ip:match('^%d+%.%d+%.%d+%.%d+$') then return ip .. '/32' end + end + return nil + end + return scan(spec.egress) or scan(spec.ingress) +end + +local function apply_request(req, opts) + req = req or {} + opts = opts or {} + local links = req.links or req.policy + if is_plain_table(links) then + local applied, changed = {}, false + for _, link_id in ipairs(sorted_keys(links)) do + local link = links[link_id] + if is_plain_table(link) then + local one = shallow_copy(link) + one.iface = one.iface or one.device or one.linux_interface or one.interface + one.subnet = one.subnet or one.cidr or req.subnet or req.cidr or first_host_cidr(one) + if type(one.iface) ~= 'string' or one.iface == '' then + return { ok = false, err = 'shaping link ' .. tostring(link_id) .. ' missing iface/device', backend = 'openwrt' } + end + if type(one.subnet) ~= 'string' or one.subnet == '' then + return { ok = false, err = 'shaping link ' .. tostring(link_id) .. ' missing subnet/cidr', backend = 'openwrt' } + end + local ok, err = apply_one(one) + if ok ~= true then + return { ok = false, err = tostring(link_id) .. ': ' .. tostring(err), backend = 'openwrt' } + end + applied[link_id] = { ok = true, iface = one.iface, subnet = one.subnet } + changed = true + end + end + return { ok = true, applied = true, changed = changed, backend = 'openwrt', links = applied } + end + + local ok, err = apply_one(req) + if ok ~= true then return { ok = false, err = tostring(err), backend = 'openwrt' } end + return { ok = true, applied = true, changed = true, backend = 'openwrt' } +end + +function M.apply(req, opts) + opts = opts or {} + local previous = ACTIVE_RUN_CMD + ACTIVE_RUN_CMD = opts.run_cmd or (req and req.run_cmd) + local ok, result = safe.pcall(apply_request, req or {}, opts) + ACTIVE_RUN_CMD = previous + if not ok then return { ok = false, err = tostring(result), backend = 'openwrt' } end + return result +end + +function M.clear(iface, opts) + opts = opts or {} + local previous = ACTIVE_RUN_CMD + ACTIVE_RUN_CMD = opts.run_cmd or opts.runner + if type(iface) ~= 'string' or iface == '' then + ACTIVE_RUN_CMD = previous + return nil, 'iface is required' + end + + local logger = (type(opts.log) == 'function') and opts.log or nil + local st = STATE[iface] + local ifb = opts.ifb or (st and st.ingress and st.ingress.ifb) or sanitise_ifb_name(iface) + + try_cmd({ 'tc', 'qdisc', 'del', 'dev', iface, 'root' }, logger) + try_cmd({ 'tc', 'qdisc', 'del', 'dev', iface, 'ingress' }, logger) + try_cmd({ 'tc', 'qdisc', 'del', 'dev', ifb, 'root' }, logger) + + if opts.delete_ifb then + try_cmd({ 'ip', 'link', 'del', ifb }, logger) + end + + STATE[iface] = nil + ACTIVE_RUN_CMD = previous + return true, nil +end + +return M diff --git a/src/services/hal/backends/openwrt/uci_manager.lua b/src/services/hal/backends/openwrt/uci_manager.lua new file mode 100644 index 00000000..3b2d9bd1 --- /dev/null +++ b/src/services/hal/backends/openwrt/uci_manager.lua @@ -0,0 +1,1519 @@ +-- services/hal/backends/openwrt/uci_manager.lua +-- Scoped UCI commit manager. +-- +-- This module is OpenWrt HAL infrastructure. New strict code should use the +-- Op-returning submit_op/commit_op path. The compatibility singleton may expose +-- blocking commit() only at the edge while preserving old wifi service callers. +-- +-- Ownership contract: +-- * before admission succeeds, the caller still owns staged changes; +-- * after admission succeeds, this manager owns a normalised copy of the +-- commit record; +-- * caller cancellation after admission abandons only that caller's wait, not +-- the already-admitted commit. +-- +-- UCI model covered here: +-- * named section creation through set(config, section, stype) +-- * anonymous section creation through add(config, stype), returning a +-- session-local alias which may be used by later staged changes; +-- * scalar and list option set; +-- * add_list / del_list implemented against the manager-owned cursor; +-- * delete option / section; +-- * rename option / section where the libuci-lua binding supports it; +-- * reorder section where the libuci-lua binding supports it; +-- * commit, revert-on-failure and deduplicated reload/restart commands. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' +local queue = require 'devicecode.support.queue' +local file = require 'fibers.io.file' + +local M = {} +local Manager = {} +Manager.__index = Manager + +local ActivationRunner = {} +ActivationRunner.__index = ActivationRunner + +local Session = {} +Session.__index = Session + +local function elapsed_ms(t0) + if not t0 then return nil end + return math.floor(((fibers.now() - t0) * 1000) + 0.5) +end + +local function exec_spec(argv, opts) + opts = opts or {} + local spec = {} + for i = 1, #argv do spec[i] = argv[i] end + spec.stdin = opts.stdin ~= nil and opts.stdin or 'null' + spec.stdout = opts.stdout ~= nil and opts.stdout or 'null' + spec.stderr = opts.stderr ~= nil and opts.stderr or 'null' + return spec +end + +local function log_manager(self, level, payload) + local logger = self and self._logger + if logger and type(logger[level]) == 'function' then + payload = payload or {} + payload.component = payload.component or 'uci_manager' + logger[level](logger, payload) + end +end + +local function trace_fields(trace) + local out = {} + if type(trace) == 'table' then + for k, v in pairs(trace) do out[k] = v end + end + return out +end + +local function shallow_copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +local function deep_copy(v) + if type(v) ~= 'table' then return v end + local out = {} + for k, vv in pairs(v) do out[k] = deep_copy(vv) end + return out +end + +local function copy_changes(changes) + local out = {} + for i, ch in ipairs(changes or {}) do + out[i] = deep_copy(ch) + end + return out +end + +local function is_array(t) + if type(t) ~= 'table' then return false end + local n = 0 + for k in pairs(t) do + if type(k) ~= 'number' or k < 1 or k % 1 ~= 0 then return false end + if k > n then n = k end + end + return n == #t +end + +local function array_copy(t) + local out = {} + for i = 1, #(t or {}) do out[i] = t[i] end + return out +end + +local function ensure_pkg_file(confdir, pkg) + if type(confdir) ~= 'string' or confdir == '' or type(pkg) ~= 'string' or pkg == '' then return true, nil end + local path = confdir .. '/' .. pkg + + local f = file.open(path, 'r') + if f then f:close(); return true, nil end + + local err + f, err = file.open(path, 'w') + if not f then return nil, tostring(err) end + local ok, werr = fibers.perform(f:write_op('')) + local cok, cerr = f:close() + if ok == false or ok == nil then return nil, tostring(werr or 'failed to create UCI package file ' .. path) end + if cok == false or cok == nil then return nil, tostring(cerr or 'failed to close UCI package file ' .. path) end + return true, nil +end + +local function is_uci_identifier(s) + return type(s) == 'string' and s ~= '' and s:match('^[A-Za-z0-9_]+$') ~= nil +end + +-- UCI section types (e.g. wifi-device, wifi-iface) may contain hyphens. +local function is_uci_section_type(s) + return type(s) == 'string' and s ~= '' and s:match('^[A-Za-z0-9_%-]+$') ~= nil +end + +local function validate_pkg(s, label) + if not is_uci_identifier(s) then + return nil, label .. ' must be a non-empty UCI identifier' + end + return true, nil +end + +local function validate_section_ref(s, label) + if type(s) ~= 'string' or s == '' then + return nil, label .. ' required' + end + return true, nil +end + +local function to_uci_value(v) + local tv = type(v) + if tv == 'boolean' then return v and '1' or '0' end + if tv == 'number' or tv == 'string' then return tostring(v) end + if tv == 'table' then + if not is_array(v) then return nil, 'UCI table values must be arrays' end + local out = {} + for i = 1, #v do + local cv, err = to_uci_value(v[i]) + if cv == nil then return nil, err end + out[i] = cv + end + return out + end + if v == nil then return nil end + return tostring(v) +end + +local function as_list(v) + if v == nil then return {} end + if type(v) == 'table' then + local out = {} + for i = 1, #v do out[i] = tostring(v[i]) end + return out + end + return { tostring(v) } +end + +local function validate_change(record, ch, i) + if type(ch) ~= 'table' then return nil, 'change ' .. i .. ' must be a table' end + if ch.config ~= nil and ch.config ~= record.config then + return nil, 'change ' .. i .. ' config differs from record.config' + end + local op = ch.op + if op ~= 'set' and op ~= 'delete' and op ~= 'add' and op ~= 'add_list' + and op ~= 'del_list' and op ~= 'rename' and op ~= 'reorder' then + return nil, 'change ' .. i .. ' has unsupported op' + end + + if op == 'add' then + if not is_uci_section_type(ch.stype or ch.section_type or ch.option) then + return nil, 'change ' .. i .. ' section type must be a valid UCI section type' + end + if ch.alias ~= nil and (type(ch.alias) ~= 'string' or ch.alias == '') then + return nil, 'change ' .. i .. ' alias must be a non-empty string when present' + end + return true, nil + end + + local ok, serr = validate_section_ref(ch.section, 'change ' .. i .. ' section') + if ok ~= true then return nil, serr end + + if op == 'set' then + if type(ch.option) ~= 'string' or ch.option == '' then + return nil, 'change ' .. i .. ' option/section type required' + end + if ch.value == nil then + if not is_uci_section_type(ch.option) then + return nil, 'change ' .. i .. ' named section type must be a UCI identifier' + end + else + local _, verr = to_uci_value(ch.value) + if verr then return nil, 'change ' .. i .. ': ' .. tostring(verr) end + end + elseif op == 'delete' then + if ch.option ~= nil and type(ch.option) ~= 'string' then + return nil, 'change ' .. i .. ' option must be a string when present' + end + elseif op == 'add_list' or op == 'del_list' then + if type(ch.option) ~= 'string' or ch.option == '' then + return nil, 'change ' .. i .. ' option required' + end + if ch.value == nil then return nil, 'change ' .. i .. ' value required' end + local _, verr = to_uci_value(ch.value) + if verr then return nil, 'change ' .. i .. ': ' .. tostring(verr) end + elseif op == 'rename' then + if ch.option ~= nil and type(ch.option) ~= 'string' then + return nil, 'change ' .. i .. ' option must be a string when present' + end + if type(ch.name or ch.new_name) ~= 'string' or (ch.name or ch.new_name) == '' then + return nil, 'change ' .. i .. ' rename target required' + end + elseif op == 'reorder' then + if type(ch.position) ~= 'number' then return nil, 'change ' .. i .. ' position must be a number' end + end + return true, nil +end + +local function validate_argv(argv, label) + if type(argv) ~= 'table' or #argv == 0 then return nil, label .. ' must be a non-empty argv array' end + for i = 1, #argv do + if type(argv[i]) ~= 'string' or argv[i] == '' then + return nil, label .. '[' .. i .. '] must be a non-empty string' + end + end + return true, nil +end + +local function semantic_reload_to_argv(entry) + local kind = entry.kind or entry.action + local target = entry.target or entry.service + if kind ~= 'reload' and kind ~= 'restart' then + return nil, 'unsupported reload action: ' .. tostring(kind) + end + if type(target) ~= 'string' or target == '' then + return nil, 'reload target required' + end + if target == 'wifi' or target == 'wireless' then + return { '/sbin/wifi', kind }, nil + end + return { '/etc/init.d/' .. target, kind }, nil +end + +local function normalise_legacy_argv(argv) + -- Existing radio/band providers pass legacy command shorthands such as + -- { 'wifi', 'reload' } and { 'service', 'dawn', 'restart' }. Preserve this + -- compatibility at the HAL edge while encouraging new code to use semantic + -- reload tables. + if argv[1] == 'wifi' then + local out = { '/sbin/wifi' } + for i = 2, #argv do out[#out + 1] = argv[i] end + return out + end + if argv[1] == 'service' and type(argv[2]) == 'string' and type(argv[3]) == 'string' then + local out = { '/etc/init.d/' .. argv[2], argv[3] } + for i = 4, #argv do out[#out + 1] = argv[i] end + return out + end + return argv +end + +local function normalise_restart_cmds(cmds) + if cmds == nil then return {}, nil end + if type(cmds) ~= 'table' then return nil, 'restart_cmds must be a table' end + local out = {} + for i = 1, #cmds do + local entry = cmds[i] + local argv, err + local wait = true + if type(entry) == 'table' and type(entry[1]) == 'string' then + local ok, verr = validate_argv(entry, 'restart_cmds[' .. i .. ']') + if ok ~= true then return nil, verr end + wait = entry.wait ~= false and entry.async ~= true + argv = normalise_legacy_argv(array_copy(entry)) + elseif type(entry) == 'table' then + wait = entry.wait ~= false and entry.async ~= true + argv, err = semantic_reload_to_argv(entry) + if not argv then return nil, err end + else + return nil, 'restart_cmds[' .. i .. '] must be an argv or semantic reload table' + end + argv.wait = wait + out[#out + 1] = argv + end + return out, nil +end + +local function normalise_record(record) + if type(record) ~= 'table' then return nil, 'record must be a table' end + if type(record.config) ~= 'string' or record.config == '' then return nil, 'record.config required' end + local ok_pkg, pkg_err = validate_pkg(record.config, 'record.config') + if ok_pkg ~= true then return nil, pkg_err end + if type(record.changes) ~= 'table' then return nil, 'record.changes must be a table' end + + local out = { + config = record.config, + changes = copy_changes(record.changes), + reply_tx = record.reply_tx, + replace_package = record.replace_package == true, + } + for i, ch in ipairs(out.changes) do + local ok, err = validate_change(out, ch, i) + if ok ~= true then return nil, err end + ch.config = out.config + end + local restarts, rerr = normalise_restart_cmds(record.restart_cmds) + if not restarts then return nil, rerr end + out.restart_cmds = restarts + return out, nil +end + +local function resolve_section(record, aliases, section) + if aliases and aliases[section] then return aliases[section] end + return section +end + +local function revert_record(cursor, record) + if cursor and type(cursor.revert) == 'function' then + pcall(function () cursor:revert(record.config) end) + end +end + +local function record_packages(records) + local seen, out = {}, {} + for _, record in ipairs(records or {}) do + if record and record.config and not seen[record.config] then + seen[record.config] = true + out[#out + 1] = record.config + end + end + table.sort(out) + return out +end + +local function copy_package_table(pkg) + local out = {} + for k, v in pairs(pkg or {}) do + if type(k) == 'string' and k:sub(1, 1) ~= '.' then + out[k] = deep_copy(v) + end + end + return out +end + +local function snapshot_packages(cursor, packages) + local out = {} + if not cursor then return out, nil end + if type(cursor.get_all) ~= 'function' then return nil, 'uci get_all unavailable; cannot snapshot for rollback' end + for _, pkg in ipairs(packages or {}) do + if type(cursor.load) == 'function' then pcall(function () cursor:load(pkg) end) end + out[pkg] = copy_package_table(cursor:get_all(pkg) or {}) + end + return out, nil +end + +local function delete_all_sections(cursor, pkg) + if not cursor then return true, nil end + if type(cursor.get_all) ~= 'function' then return nil, 'uci get_all unavailable; cannot clear package for rollback' end + local cur = cursor:get_all(pkg) or {} + for secname, rec in pairs(cur) do + if type(secname) == 'string' and secname:sub(1, 1) ~= '.' and type(rec) == 'table' then + local ok, err = cursor:delete(pkg, secname) + if ok ~= true then return nil, tostring(err or ('delete failed for ' .. pkg .. '.' .. secname)) end + end + end + return true, nil +end + +local function restore_package(cursor, pkg, snapshot) + if not cursor then return true, nil end + if type(cursor.revert) == 'function' then pcall(function () cursor:revert(pkg) end) end + local ok, err = delete_all_sections(cursor, pkg) + if ok ~= true then return nil, err end + for secname, rec in pairs(snapshot or {}) do + if type(secname) == 'string' and secname:sub(1, 1) ~= '.' and type(rec) == 'table' then + local stype = rec['.type'] + if type(stype) ~= 'string' or stype == '' then + return nil, 'rollback snapshot missing section type for ' .. pkg .. '.' .. secname + end + ok, err = cursor:set(pkg, secname, stype) + if ok ~= true then return nil, tostring(err or ('restore create failed for ' .. pkg .. '.' .. secname)) end + for opt, value in pairs(rec) do + if type(opt) == 'string' and opt:sub(1, 1) ~= '.' then + local uval, verr = to_uci_value(value) + if verr then return nil, verr end + ok, err = cursor:set(pkg, secname, opt, uval) + if ok ~= true then return nil, tostring(err or ('restore set failed for ' .. pkg .. '.' .. secname .. '.' .. opt)) end + end + end + end + end + ok, err = cursor:commit(pkg) + if ok ~= true then return nil, tostring(err or ('rollback commit failed for ' .. pkg)) end + return true, nil +end + +local function restore_packages(cursor, snapshots, packages) + local restored, errors = {}, {} + for _, pkg in ipairs(packages or {}) do + local ok, err = restore_package(cursor, pkg, snapshots and snapshots[pkg] or {}) + if ok == true then + restored[#restored + 1] = pkg + else + errors[#errors + 1] = tostring(pkg) .. ': ' .. tostring(err) + end + end + if #errors > 0 then return nil, table.concat(errors, '; '), restored end + return true, nil, restored +end + +local function cursor_add_list(cursor, config, section, option, value) + if type(cursor.add_list) == 'function' then + local ok, err = cursor:add_list(config, section, option, value) + if ok then return true, nil end + -- Some Lua bindings do not expose add_list; if present but rejected, report it. + return nil, tostring(err or 'uci add_list failed') + end + local cur = as_list(cursor:get(config, section, option)) + cur[#cur + 1] = value + local ok, err = cursor:set(config, section, option, cur) + if not ok then return nil, tostring(err or 'uci add_list(set) failed') end + return true, nil +end + +local function cursor_del_list(cursor, config, section, option, value) + if type(cursor.del_list) == 'function' then + local ok, err = cursor:del_list(config, section, option, value) + if ok then return true, nil end + return nil, tostring(err or 'uci del_list failed') + end + local cur = as_list(cursor:get(config, section, option)) + local out = {} + local removed = false + for i = 1, #cur do + if tostring(cur[i]) == tostring(value) then + removed = true + else + out[#out + 1] = cur[i] + end + end + if not removed then return true, nil end + if #out == 0 then + local ok, err = cursor:delete(config, section, option) + if not ok then return nil, tostring(err or 'uci del_list(delete) failed') end + return true, nil + end + local ok, err = cursor:set(config, section, option, out) + if not ok then return nil, tostring(err or 'uci del_list(set) failed') end + return true, nil +end + +local function cursor_rename_option_fallback(cursor, config, section, option, new_name) + local value = cursor:get(config, section, option) + if value == nil then return nil, 'uci rename option failed: option not found' end + local ok, err = cursor:set(config, section, new_name, value) + if not ok then return nil, tostring(err or 'uci rename option(set) failed') end + ok, err = cursor:delete(config, section, option) + if not ok then return nil, tostring(err or 'uci rename option(delete) failed') end + return true, nil +end + +local function cursor_rename_section_fallback(cursor, config, section, new_name) + if not is_uci_identifier(new_name) then + return nil, 'uci rename section target must be a UCI identifier' + end + if type(cursor.get_all) ~= 'function' then + return nil, 'uci rename unavailable in Lua binding and get_all fallback unavailable' + end + local all = cursor:get_all(config, section) + if type(all) ~= 'table' then return nil, 'uci rename section failed: section not found' end + local stype = all['.type'] + if type(stype) ~= 'string' or stype == '' then + return nil, 'uci rename section failed: section type missing' + end + local ok, err = cursor:set(config, new_name, stype) + if not ok then return nil, tostring(err or 'uci rename section(create) failed') end + for k, v in pairs(all) do + if type(k) == 'string' and k:sub(1, 1) ~= '.' then + ok, err = cursor:set(config, new_name, k, v) + if not ok then return nil, tostring(err or ('uci rename section(set ' .. tostring(k) .. ') failed')) end + end + end + ok, err = cursor:delete(config, section) + if not ok then return nil, tostring(err or 'uci rename section(delete old) failed') end + return true, nil +end + +local function cursor_rename(cursor, config, section, option, new_name) + if type(cursor.rename) == 'function' then + local ok, err + if option ~= nil then + ok, err = cursor:rename(config, section, option, new_name) + else + ok, err = cursor:rename(config, section, new_name) + end + if ok then return true, nil end + return nil, tostring(err or 'uci rename failed') + end + if option ~= nil then + return cursor_rename_option_fallback(cursor, config, section, option, new_name) + end + return cursor_rename_section_fallback(cursor, config, section, new_name) +end + +local function apply_with_cursor(cursor, record) + if not cursor then + return true, nil -- explicit fake/no-uci mode for tests and non-OpenWrt hosts. + end + if type(cursor.load) == 'function' then pcall(function () cursor:load(record.config) end) end + if record.replace_package == true then + local ok, err = delete_all_sections(cursor, record.config) + if ok ~= true then return nil, err end + end + local aliases = {} + for _, change in ipairs(record.changes) do + local op = change.op + local section = resolve_section(record, aliases, change.section) + if op == 'add' then + if type(cursor.add) ~= 'function' then return nil, 'uci add unavailable in Lua binding' end + local stype = change.stype or change.section_type or change.option + local name, err = cursor:add(record.config, stype) + if not name then return nil, tostring(err or 'uci add failed') end + if change.alias then aliases[change.alias] = name end + elseif op == 'set' then + if change.value == nil then + local ok, err = cursor:set(record.config, section, change.option) + if not ok then return nil, tostring(err or 'uci set section failed') end + else + local value, verr = to_uci_value(change.value) + if verr then return nil, verr end + local ok, err = cursor:set(record.config, section, change.option, value) + if not ok then return nil, tostring(err or 'uci set failed') end + end + elseif op == 'delete' then + local ok, err + if change.option ~= nil then + ok, err = cursor:delete(record.config, section, change.option) + else + ok, err = cursor:delete(record.config, section) + end + if not ok then return nil, tostring(err or 'uci delete failed') end + elseif op == 'add_list' then + local values = type(change.value) == 'table' and change.value or { change.value } + for i = 1, #values do + local value, verr = to_uci_value(values[i]) + if verr then return nil, verr end + local ok, err = cursor_add_list(cursor, record.config, section, change.option, value) + if not ok then return nil, err end + end + elseif op == 'del_list' then + local values = type(change.value) == 'table' and change.value or { change.value } + for i = 1, #values do + local value, verr = to_uci_value(values[i]) + if verr then return nil, verr end + local ok, err = cursor_del_list(cursor, record.config, section, change.option, value) + if not ok then return nil, err end + end + elseif op == 'rename' then + local new_name = change.name or change.new_name + local ok, err = cursor_rename(cursor, record.config, section, change.option, new_name) + if ok and change.option == nil and aliases[change.section] then aliases[change.section] = new_name end + if not ok then return nil, tostring(err or 'uci rename failed') end + elseif op == 'reorder' then + if type(cursor.reorder) ~= 'function' then return nil, 'uci reorder unavailable in Lua binding' end + local ok, err = cursor:reorder(record.config, section, math.floor(change.position)) + if not ok then return nil, tostring(err or 'uci reorder failed') end + end + end + local ok, err = cursor:commit(record.config) + if not ok then return nil, tostring(err or 'uci commit failed') end + return true, nil +end + +local function default_run_cmd(argv) + local exec = require 'fibers.io.exec' + local cmd = exec.command(exec_spec(argv)) + local status, code = fibers.perform(cmd:run_op()) + if status ~= 'exited' or code ~= 0 then + return false, table.concat(argv, ' ') .. ' exited with status=' .. tostring(status) .. ' code=' .. tostring(code) + end + return true, nil +end + +local NO_ACTIVATION = {} + +local function restart_key(argv) + return table.concat(argv, '\0') +end + +local function trim_restarts(record_results) + local seen = {} + local entries = {} + for _, res in ipairs(record_results) do + if res.ok and res.record and res.record.restart_cmds then + for _, argv in ipairs(res.record.restart_cmds) do + local key = restart_key(argv) + local entry = seen[key] + if not entry then + entry = { argv = argv, wait = argv.wait ~= false, sessions = {} } + seen[key] = entry + entries[#entries + 1] = entry + elseif argv.wait ~= false then + -- If any caller requires a synchronous restart, keep the shared + -- deduplicated command synchronous for all sessions that depend on it. + entry.wait = true + end + local duplicate = false + for _, s in ipairs(entry.sessions) do + if s == res then duplicate = true; break end + end + if not duplicate then entry.sessions[#entry.sessions + 1] = res end + end + end + end + return entries +end + +local function split_activation_entries(entries) + local sync, async = {}, {} + for _, entry in ipairs(entries or {}) do + if entry.wait ~= false then sync[#sync + 1] = entry else async[#async + 1] = entry end + end + return sync, async +end + +local function activation_entry_count(entries) + return #(entries or {}) +end + +local function activation_argvs(entries) + local out = {} + for i, entry in ipairs(entries or {}) do out[i] = array_copy(entry.argv or {}) end + return out +end + +local function new_activation_runner(manager) + local tx, rx = mailbox.new(8, { full = 'drop_oldest' }) + return setmetatable({ + _manager = manager, + _tx = tx, + _rx = rx, + _closed = false, + _next_id = 0, + _latest_id = 0, + _running_id = nil, + _status = { state = 'idle' }, + }, ActivationRunner) +end + +function ActivationRunner:submit(entries, trace) + if self._closed then return nil, 'activation runner closed' end + if activation_entry_count(entries) == 0 then + return { state = 'none', commands = 0 }, nil + end + self._next_id = (self._next_id or 0) + 1 + local item = { + id = self._next_id, + entries = entries, + trace = trace_fields(trace), + submitted_at = fibers.now(), + } + self._latest_id = item.id + local ok, err = queue.try_admit_now(self._tx, item) + if ok ~= true then return nil, err or 'activation runner busy' end + self._status = { + state = self._running_id and 'queued' or 'scheduled', + generation = item.id, + commands = activation_entry_count(entries), + submitted_at = item.submitted_at, + } + log_manager(self._manager, 'info', (function () + local p = trace_fields(trace) + p.what = 'uci_activation_scheduled' + p.activation_id = item.id + p.commands = activation_entry_count(entries) + p.argvs = activation_argvs(entries) + return p + end)()) + return { + state = 'scheduled', + generation = item.id, + commands = activation_entry_count(entries), + }, nil +end + +function ActivationRunner:status() + return deep_copy(self._status or { state = 'idle' }) +end + +function ActivationRunner:_next_item(owner_scope) + local arms = { item = self._rx:recv_op() } + if owner_scope and type(owner_scope.close_op) == 'function' then arms.closed = owner_scope:close_op() end + local which, item = fibers.perform(fibers.named_choice(arms)) + if which == 'closed' or item == nil then return nil end + + local drained = 0 + while true do + local newer = queue.try_now(self._rx:recv_op(), NO_ACTIVATION) + if newer == NO_ACTIVATION or newer == nil then break end + item = newer + drained = drained + 1 + end + if drained > 0 then + log_manager(self._manager, 'info', (function () + local p = trace_fields(item.trace) + p.what = 'uci_activation_coalesced' + p.activation_id = item.id + p.dropped = drained + return p + end)()) + end + return item +end + +function ActivationRunner:_run_entry(entry, trace, activation_id) + local argv = entry.argv or {} + local t0 = fibers.now() + log_manager(self._manager, 'info', (function () + local p = trace_fields(trace) + p.what = 'uci_activation_command_begin' + p.activation_id = activation_id + p.argv = array_copy(argv) + return p + end)()) + local ok, err = self._manager._run_cmd(argv) + log_manager(self._manager, ok == true and 'info' or 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_activation_command_done' + p.activation_id = activation_id + p.argv = array_copy(argv) + p.ok = ok == true + p.err = err + p.elapsed_ms = elapsed_ms(t0) + return p + end)()) + return ok == true, err +end + +function ActivationRunner:_run(owner_scope) + while self._closed ~= true do + local item = self:_next_item(owner_scope) + if item == nil then return end + self._running_id = item.id + local t0 = fibers.now() + self._status = { + state = 'running', + generation = item.id, + commands = activation_entry_count(item.entries), + started_at = t0, + } + log_manager(self._manager, 'info', (function () + local p = trace_fields(item.trace) + p.what = 'uci_activation_runner_begin' + p.activation_id = item.id + p.commands = activation_entry_count(item.entries) + p.argvs = activation_argvs(item.entries) + return p + end)()) + + local ok, err = true, nil + for _, entry in ipairs(item.entries or {}) do + local rok, rerr = self:_run_entry(entry, item.trace, item.id) + if rok ~= true then ok, err = false, rerr; break end + end + + local stale = item.id < (self._latest_id or item.id) + self._status = { + state = ok and 'done' or 'failed', + generation = item.id, + commands = activation_entry_count(item.entries), + ok = ok == true, + err = err, + stale = stale, + elapsed_ms = elapsed_ms(t0), + } + if ok ~= true then + self._manager._last_activation_error = { + activation_id = item.id, + err = tostring(err or 'activation failed'), + } + end + log_manager(self._manager, ok == true and 'info' or 'warn', (function () + local p = trace_fields(item.trace) + p.what = 'uci_activation_runner_done' + p.activation_id = item.id + p.ok = ok == true + p.err = err + p.stale = stale + p.elapsed_ms = elapsed_ms(t0) + return p + end)()) + self._running_id = nil + end +end + +function ActivationRunner:start(scope) + if self._closed then return nil, 'activation runner closed' end + if self._scope then return true, nil end + scope = scope or fibers.current_scope() + if not scope then return nil, 'scope required' end + self._scope = scope + local ok, err = scope:spawn(function (worker_scope) self:_run(worker_scope) end) + if ok ~= true then return nil, err or 'activation runner spawn failed' end + return true, nil +end + +function ActivationRunner:terminate(reason) + if self._closed then return true, nil end + self._closed = true + self._tx:close(reason or 'activation runner terminated') + self._scope = nil + return true, nil +end + +local function make_cursor(opts) + if opts and opts.cursor ~= nil then return opts.cursor, nil end + local ok, uci_or_err = pcall(require, 'uci') + if not ok or not uci_or_err or type(uci_or_err.cursor) ~= 'function' then + if opts and opts.allow_fake == false then + return nil, 'uci module unavailable' + end + return nil, 'uci module unavailable; manager running in fake mode' + end + return uci_or_err.cursor(opts and opts.confdir, opts and opts.savedir), nil +end + +function M.new(opts) + opts = opts or {} + local tx, rx = mailbox.new(opts.queue_len or 16, { full = opts.full or 'reject_newest' }) + local cursor, cursor_note = make_cursor(opts) + if cursor == nil and opts.allow_fake == false then + return nil, cursor_note or 'uci unavailable' + end + local confdir = opts.confdir + if confdir == nil and opts.cursor == nil and cursor ~= nil then confdir = '/etc/config' end + local mgr = setmetatable({ + _tx = tx, + _rx = rx, + _scope = nil, + _closed = false, + _cursor = cursor, + _cursor_note = cursor_note, + _fake = cursor == nil, + _confdir = confdir, + _savedir = opts.savedir, + _debounce_s = tonumber(opts.debounce_s) or 0.1, + _run_cmd = opts.run_cmd or default_run_cmd, + _run_cmd_explicit = type(opts.run_cmd) == 'function', + _logger = opts.logger, + _pending = {}, + }, Manager) + mgr._activation = new_activation_runner(mgr) + return mgr +end + +function Manager:start(owner_scope) + if self._closed then return nil, 'closed' end + if self._scope then return true, nil end + owner_scope = owner_scope or fibers.current_scope() + if not owner_scope then return nil, 'owner scope required' end + + -- The UCI manager is long-lived owned work under the supplied lifetime + -- scope. Do not install finalisers on owner_scope here: lua-fibers only + -- permits scope:finally on an already-started scope from inside that same + -- scope. Provider apply calls may start the manager from a request fibre, + -- so create a private child scope and install its finaliser before it starts. + local child, cerr = owner_scope:child() + if not child then return nil, cerr or 'uci manager scope create failed' end + + self._scope = child + child:finally(function (_, status, primary) + self:terminate(primary or status or 'uci manager closed') + end) + + local aok, aerr = self._activation:start(child) + if aok ~= true then + self._scope = nil + child:cancel(aerr or 'activation runner start failed') + return nil, aerr or 'activation runner start failed' + end + + local ok, err = child:spawn(function (worker_scope) self:_run(worker_scope) end) + if ok ~= true then + self._activation:terminate(err or 'uci manager spawn failed') + self._scope = nil + child:cancel(err or 'uci manager spawn failed') + return nil, err or 'uci manager spawn failed' + end + + return true, nil +end + +function Manager:_collect_batch(first_item, owner_scope) + local batch = { first_item } + while self._closed ~= true do + local arms = { + more = self._rx:recv_op(), + timeout = sleep.sleep_op(self._debounce_s):wrap(function () return true end), + } + if owner_scope and type(owner_scope.close_op) == 'function' then + arms.closed = owner_scope:close_op() + end + local which, item = fibers.perform(fibers.named_choice(arms)) + if which == 'timeout' or which == 'closed' then break end + if item == nil then break end + if item.transaction then + self._pending[#self._pending + 1] = item + break + end + batch[#batch + 1] = item + end + return batch +end + +function Manager:_ensure_packages(packages, trace) + local t0 = fibers.now() + log_manager(self, 'debug', (function () + local p = trace_fields(trace) + p.what = 'uci_ensure_packages_begin' + p.packages = packages or {} + return p + end)()) + if self._fake == true then + log_manager(self, 'debug', (function () + local p = trace_fields(trace) + p.what = 'uci_ensure_packages_done' + p.fake = true + p.elapsed_ms = elapsed_ms(t0) + return p + end)()) + return true, nil + end + if type(self._confdir) == 'string' and self._confdir ~= '' then + local ok, err = file.mkdir_p(self._confdir) + if ok ~= true then + log_manager(self, 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_ensure_packages_done' + p.ok = false + p.err = 'failed to create UCI confdir ' .. self._confdir .. ': ' .. tostring(err) + p.elapsed_ms = elapsed_ms(t0) + return p + end)()) + return nil, 'failed to create UCI confdir ' .. self._confdir .. ': ' .. tostring(err) + end + end + for _, pkg in ipairs(packages or {}) do + local pkg_t0 = fibers.now() + local ok, err = ensure_pkg_file(self._confdir, pkg) + log_manager(self, ok == true and 'debug' or 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_ensure_package_file' + p.package = pkg + p.ok = ok == true + p.err = err + p.elapsed_ms = elapsed_ms(pkg_t0) + return p + end)()) + if ok ~= true then return nil, err end + if self._cursor and type(self._cursor.load) == 'function' then pcall(function () self._cursor:load(pkg) end) end + end + log_manager(self, 'debug', (function () + local p = trace_fields(trace) + p.what = 'uci_ensure_packages_done' + p.ok = true + p.elapsed_ms = elapsed_ms(t0) + return p + end)()) + return true, nil +end + +function Manager:_run_restart_entry(entry, trace) + local wait = entry.wait ~= false + log_manager(self, 'debug', (function () + local p = trace_fields(trace) + p.what = wait and 'uci_activation_begin' or 'uci_activation_schedule_begin' + p.argv = array_copy(entry.argv or {}) + p.wait = wait + return p + end)()) + if wait then + local t0 = fibers.now() + local ok, err = self._run_cmd(entry.argv) + log_manager(self, ok == true and 'debug' or 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_activation_done' + p.argv = array_copy(entry.argv or {}) + p.ok = ok == true + p.err = err + p.elapsed_ms = elapsed_ms(t0) + return p + end)()) + return ok, err + end + local activation, err = self:_schedule_activation({ entry }, trace) + return activation ~= nil, err +end + +function Manager:_schedule_activation(entries, trace) + if not entries or #entries == 0 then return { state = 'none', commands = 0 }, nil end + if not self._activation then return nil, 'activation runner unavailable' end + return self._activation:submit(entries, trace) +end + +function Manager:_apply_batch(batch) + local results = {} + local pkgs = record_packages((function () local rs = {}; for _, item in ipairs(batch or {}) do rs[#rs + 1] = item.record end; return rs end)()) + local eok, eerr = self:_ensure_packages(pkgs) + if eok ~= true then + for _, item in ipairs(batch or {}) do + if item.reply_tx then queue.try_admit_now(item.reply_tx, { ok = false, err = tostring(eerr) }) end + end + return + end + for _, item in ipairs(batch) do + local record = item.record + local ok, err + if self._closed then + ok, err = nil, 'uci manager closed' + else + ok, err = apply_with_cursor(self._cursor, record) + if ok ~= true then revert_record(self._cursor, record) end + end + results[#results + 1] = { item = item, record = record, ok = ok == true, err = err or '' } + end + + local restart_entries = trim_restarts(results) + local sync_entries, async_entries = split_activation_entries(restart_entries) + for _, entry in ipairs(sync_entries) do + local rok, rerr = self:_run_restart_entry(entry) + if rok ~= true then + for _, res in ipairs(entry.sessions) do + res.ok = false + res.err = res.err ~= '' and (res.err .. '; ' .. tostring(rerr)) or tostring(rerr) + end + end + end + local activation, aerr = nil, nil + if #async_entries > 0 then + activation, aerr = self:_schedule_activation(async_entries) + if activation == nil then + for _, entry in ipairs(async_entries) do + for _, res in ipairs(entry.sessions) do + res.ok = false + res.err = res.err ~= '' and (res.err .. '; ' .. tostring(aerr)) or tostring(aerr) + end + end + end + end + + for _, res in ipairs(results) do + if res.item.reply_tx then + local ok = queue.try_admit_now(res.item.reply_tx, { ok = res.ok, err = res.err, activation = activation }) + if ok ~= true then + -- The caller has stopped waiting. This is not a manager failure. + end + end + end +end + + +function Manager:_apply_transaction(item) + local t0 = fibers.now() + local tx = item.transaction or {} + local records = tx.records or {} + local packages = tx.packages or record_packages(records) + local trace = tx.trace or {} + log_manager(self, 'debug', (function () + local p = trace_fields(trace) + p.what = 'uci_transaction_begin' + p.packages = packages + p.records = #records + return p + end)()) + local eok, eerr = self:_ensure_packages(packages, trace) + if eok ~= true then + log_manager(self, 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_transaction_done' + p.ok = false + p.status = 'failed_no_change' + p.err = tostring(eerr) + p.elapsed_ms = elapsed_ms(t0) + return p + end)()) + if item.reply_tx then queue.try_admit_now(item.reply_tx, { ok = false, status = 'failed_no_change', err = tostring(eerr), packages = packages }) end + return + end + + local result = { + ok = true, + status = 'ok', + packages = packages, + rollback = { attempted = false, ok = nil, packages = {} }, + timings = {}, + } + + local phase = fibers.now() + log_manager(self, 'debug', (function () + local p = trace_fields(trace) + p.what = 'uci_snapshot_begin' + p.packages = packages + return p + end)()) + local snapshots, snap_err = snapshot_packages(self._cursor, packages) + result.timings.snapshot_ms = elapsed_ms(phase) + log_manager(self, snapshots and 'debug' or 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_snapshot_done' + p.ok = snapshots ~= nil + p.err = snap_err + p.elapsed_ms = elapsed_ms(phase) + return p + end)()) + if snapshots == nil then + result.ok = false + result.status = 'failed_no_change' + result.err = snap_err + result.timings.total_ms = elapsed_ms(t0) + log_manager(self, 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_transaction_done' + p.ok = false + p.status = result.status + p.err = result.err + p.elapsed_ms = result.timings.total_ms + return p + end)()) + if item.reply_tx then queue.try_admit_now(item.reply_tx, result) end + return + end + + local record_results = {} + local failed = nil + for _, record in ipairs(records) do + local rt0 = fibers.now() + log_manager(self, 'debug', (function () + local p = trace_fields(trace) + p.what = 'uci_record_apply_begin' + p.config = record.config + p.changes = #(record.changes or {}) + p.replace_package = record.replace_package == true + return p + end)()) + local ok, err + if self._closed then + ok, err = nil, 'uci manager closed' + else + ok, err = apply_with_cursor(self._cursor, record) + end + local record_ms = elapsed_ms(rt0) + log_manager(self, ok == true and 'debug' or 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_record_apply_done' + p.config = record.config + p.ok = ok == true + p.err = err + p.elapsed_ms = record_ms + return p + end)()) + local res = { record = record, ok = ok == true, err = err or '', elapsed_ms = record_ms } + record_results[#record_results + 1] = res + result.timings['commit_' .. tostring(record.config) .. '_ms'] = record_ms + if ok ~= true then + failed = { step = 'uci_commit', config = record.config, err = err } + break + end + end + + if not failed then + phase = fibers.now() + local restart_entries = trim_restarts(record_results) + log_manager(self, 'debug', (function () + local p = trace_fields(trace) + p.what = 'uci_activation_phase_begin' + p.entries = #restart_entries + p.fake = self._fake == true + return p + end)()) + if self._fake == true and self._run_cmd_explicit ~= true then + -- In fake-UCI mode there is no OpenWrt system to reload. Unit tests and + -- non-OpenWrt hosts should still exercise validation, reconciliation and + -- transaction/rollback semantics without attempting /etc/init.d commands. + else + local sync_entries, async_entries = split_activation_entries(restart_entries) + for _, entry in ipairs(sync_entries) do + local rok, rerr = self:_run_restart_entry(entry, trace) + if rok ~= true then + failed = { step = 'restart', argv = entry.argv, err = rerr } + break + end + end + if not failed and #async_entries > 0 then + local activation, aerr = self:_schedule_activation(async_entries, trace) + if activation then + result.activation = activation + else + failed = { step = 'activation_schedule', err = aerr } + end + end + end + result.timings.activation_ms = elapsed_ms(phase) + log_manager(self, failed and 'warn' or 'debug', (function () + local p = trace_fields(trace) + p.what = 'uci_activation_phase_done' + p.ok = failed == nil + p.err = failed and failed.err or nil + p.activation = result.activation + p.elapsed_ms = result.timings.activation_ms + return p + end)()) + end + + if failed then + result.ok = false + result.failed_step = failed.step + result.failed_config = failed.config + result.err = tostring(failed.err or failed.step or 'uci transaction failed') + if tx.rollback ~= false then + result.rollback.attempted = true + phase = fibers.now() + log_manager(self, 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_rollback_begin' + p.failed_step = failed.step + p.failed_config = failed.config + return p + end)()) + local rok, rerr, restored = restore_packages(self._cursor, snapshots, packages) + result.timings.rollback_ms = elapsed_ms(phase) + result.rollback.ok = rok == true + result.rollback.packages = restored or {} + log_manager(self, rok == true and 'info' or 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_rollback_done' + p.ok = rok == true + p.err = rerr + p.elapsed_ms = result.timings.rollback_ms + return p + end)()) + if rok == true then + result.status = 'failed_rolled_back' + else + result.status = 'failed_rollback_failed' + result.rollback.err = tostring(rerr) + end + else + result.status = 'failed_partial' + end + end + + result.timings.total_ms = elapsed_ms(t0) + log_manager(self, result.ok == true and 'debug' or 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_transaction_done' + p.ok = result.ok == true + p.status = result.status + p.err = result.err + p.failed_step = result.failed_step + p.failed_config = result.failed_config + p.activation = result.activation + p.elapsed_ms = result.timings.total_ms + return p + end)()) + + if item.reply_tx then + local sent = queue.try_admit_now(item.reply_tx, result) + log_manager(self, sent == true and 'debug' or 'warn', (function () + local p = trace_fields(trace) + p.what = 'uci_transaction_reply' + p.sent = sent == true + p.elapsed_ms = elapsed_ms(t0) + return p + end)()) + end +end + +function Manager:_run(owner_scope) + while self._closed ~= true do + local item = table.remove(self._pending, 1) + if item == nil then + local arms = { item = self._rx:recv_op() } + if owner_scope and type(owner_scope.close_op) == 'function' then + arms.closed = owner_scope:close_op() + end + local which, got = fibers.perform(fibers.named_choice(arms)) + if which == 'closed' or got == nil then return end + item = got + end + if item.transaction then + self:_apply_transaction(item) + else + local batch = self:_collect_batch(item, owner_scope) + self:_apply_batch(batch) + end + end +end + + +function Manager:transaction_op(spec, opts) + spec = spec or {} + opts = opts or {} + local admitted_flag = false + return fibers.run_scope_op(function () + if self._closed then return { ok = false, status = 'closed', err = 'closed' }, false end + local records = {} + for i, record in ipairs(spec.records or {}) do + local normalised, err = normalise_record(record) + if not normalised then + return { ok = false, status = 'invalid', err = 'record ' .. tostring(i) .. ': ' .. tostring(err) }, false + end + records[#records + 1] = normalised + end + if #records == 0 then return { ok = true, status = 'ok', packages = {}, rollback = { attempted = false } }, false end + local reply_tx, reply_rx = mailbox.new(1, { full = 'reject_newest' }) + local admitted, aerr = queue.try_admit_now(self._tx, { + transaction = { + records = records, + packages = spec.packages or record_packages(records), + rollback = spec.rollback ~= false, + trace = opts.trace, + }, + reply_tx = reply_tx, + }) + if admitted ~= true then + return { ok = false, status = 'busy', err = 'uci_manager_busy: ' .. tostring(aerr or 'not_admitted') }, false + end + admitted_flag = true + if type(opts.on_admitted) == 'function' then opts.on_admitted() end + local result = fibers.perform(reply_rx:recv_op()) + if result == nil then return { ok = false, status = 'closed', err = 'uci manager closed' }, true end + return result, true + end):wrap(function (status, _report, result, admitted) + if status ~= 'ok' then return { ok = false, status = 'failed', err = tostring(result or status) }, admitted_flag end + return result, (admitted == true) or admitted_flag + end) +end + +function Manager:submit_op(record, opts) + opts = opts or {} + local admitted_flag = false + return fibers.run_scope_op(function () + if self._closed then return false, 'closed', false end + local normalised, err = normalise_record(record) + if not normalised then return false, err, false end + local reply_tx, reply_rx = mailbox.new(1, { full = 'reject_newest' }) + local admitted, aerr = queue.try_admit_now(self._tx, { + record = normalised, + reply_tx = reply_tx, + }) + if admitted ~= true then + return false, 'uci_manager_busy: ' .. tostring(aerr or 'not_admitted'), false + end + admitted_flag = true + if type(opts.on_admitted) == 'function' then + local ok_admit, admit_err = pcall(opts.on_admitted) + if ok_admit ~= true then return false, tostring(admit_err or 'on_admitted failed'), true end + end + local result = fibers.perform(reply_rx:recv_op()) + if result == nil then return false, 'uci manager closed', true end + return result.ok == true, result.err, true + end):wrap(function (status, _report, ok, err, admitted) + if status ~= 'ok' then return false, tostring(ok or status), admitted_flag end + return ok == true, err, (admitted == true) or admitted_flag + end) +end + +function Manager:new_session() + return setmetatable({ _manager = self, _changes = {}, _alias_n = 0 }, Session) +end + +function Manager:terminate(reason) + if self._closed then return true, nil end + self._closed = true + local why = reason or 'uci manager terminated' + if self._activation then self._activation:terminate(why) end + self._tx:close(why) + local scope = self._scope + self._scope = nil + if scope then scope:cancel(why) end + return true, nil +end + +function Manager:activation_status() + if not self._activation then return { state = 'unavailable' } end + return self._activation:status() +end + +function Manager:fake_mode() + return self._fake, self._cursor_note +end + +function Session:set(config, section, option, value) + self._changes[#self._changes + 1] = { + op = 'set', + config = config, + section = section, + option = option, + value = value, + } + return true +end + +function Session:add(config, stype, alias) + self._alias_n = (self._alias_n or 0) + 1 + alias = alias or ('__dc_uci_pending_' .. tostring(self._alias_n)) + self._changes[#self._changes + 1] = { + op = 'add', + config = config, + stype = stype, + alias = alias, + } + return alias +end + +function Session:add_list(config, section, option, value) + self._changes[#self._changes + 1] = { + op = 'add_list', + config = config, + section = section, + option = option, + value = value, + } + return true +end + +function Session:del_list(config, section, option, value) + self._changes[#self._changes + 1] = { + op = 'del_list', + config = config, + section = section, + option = option, + value = value, + } + return true +end + +function Session:delete(config, section, option) + self._changes[#self._changes + 1] = { + op = 'delete', + config = config, + section = section, + option = option, + } + return true +end + +function Session:rename(config, section, option_or_name, maybe_name) + local option, name + if maybe_name == nil then + name = option_or_name + else + option = option_or_name + name = maybe_name + end + self._changes[#self._changes + 1] = { + op = 'rename', + config = config, + section = section, + option = option, + name = name, + } + return true +end + +function Session:reorder(config, section, position) + self._changes[#self._changes + 1] = { + op = 'reorder', + config = config, + section = section, + position = position, + } + return true +end + +function Session:commit_op(config, restart_cmds) + return fibers.run_scope_op(function () + local changes = copy_changes(self._changes) + local ok, err = fibers.perform(self._manager:submit_op({ + config = config, + changes = changes, + restart_cmds = restart_cmds, + }, { + on_admitted = function () self._changes = {} end, + })) + return ok == true, err + end):wrap(function (status, _report, ok, err) + if status ~= 'ok' then return false, tostring(ok or status) end + return ok == true, err + end) +end + +function Session:commit(config, restart_cmds) + return fibers.perform(self:commit_op(config, restart_cmds)) +end + +M._normalise_record_for_test = normalise_record + +return M diff --git a/src/services/hal/backends/openwrt/uci_singleton_compat.lua b/src/services/hal/backends/openwrt/uci_singleton_compat.lua new file mode 100644 index 00000000..90b9b6f4 --- /dev/null +++ b/src/services/hal/backends/openwrt/uci_singleton_compat.lua @@ -0,0 +1,95 @@ +-- services/hal/backends/openwrt/uci_singleton_compat.lua +-- Compatibility surface for old non-Op UCI session callers. +-- +-- New code should use the scoped UCI manager directly. This wrapper preserves +-- the existing wifi-facing surface while avoiding a root-scope singleton. +-- +-- Ownership rule: +-- * production code should bind a manager owned by the OpenWrt HAL backend; +-- * unbound current-scope startup is available only when explicitly requested +-- through ensure_started({ allow_local_manager = true }). + +local fibers = require 'fibers' +local uci_manager = require 'services.hal.backends.openwrt.uci_manager' + +local M = {} + +local manager = nil +local manager_owned = 'unbound' + +local function ensure_bound_or_local(opts) + if manager then return manager, nil end + opts = opts or {} + if opts.allow_local_manager ~= true then + return nil, 'UCI compatibility manager is not bound; OpenWrt HAL must bind_manager(manager)' + end + local mgr, err = uci_manager.new(opts) + if not mgr then return nil, err end + local scope = fibers.current_scope() + if not scope then return nil, 'no current scope for local UCI compatibility manager' end + local ok, serr = mgr:start(scope) + if ok ~= true then return nil, serr end + manager = mgr + manager_owned = 'local_current_scope' + return manager, nil +end + +function M.bind_manager(mgr) + if type(mgr) ~= 'table' or type(mgr.new_session) ~= 'function' then + return nil, 'invalid UCI manager' + end + manager = mgr + manager_owned = 'bound' + return true, nil +end + +function M.clear_bound_manager() + manager = nil + manager_owned = 'unbound' + return true, nil +end + +function M.ensure_started(opts) + local _, err = ensure_bound_or_local(opts) + return err == nil, err +end + +function M.manager_owner() + return manager_owned +end + +function M.new_session(opts) + local mgr, err = ensure_bound_or_local(opts) + if not mgr then error(err or 'uci compatibility manager unavailable', 2) end + return mgr:new_session() +end + +local function read_cursor() + local ok, uci_or_err = pcall(require, 'uci') + if not ok or not uci_or_err or type(uci_or_err.cursor) ~= 'function' then return nil end + return uci_or_err.cursor() +end + +function M.get_value(config, section, option) + local c = read_cursor() + if not c then return nil end + return c:get(config, section, option) +end + +function M.section_exists(config, section) + local c = read_cursor() + if not c then return false end + return c:get(config, section) ~= nil +end + +function M.get_sections(config, stype) + local c = read_cursor() + local names = {} + if not c then return names end + c:foreach(config, stype, function (s) + if s['.name'] then names[#names + 1] = s['.name'] end + end) + return names +end + +return M diff --git a/src/services/hal/backends/radio/contract.lua b/src/services/hal/backends/radio/contract.lua new file mode 100644 index 00000000..df316868 --- /dev/null +++ b/src/services/hal/backends/radio/contract.lua @@ -0,0 +1,35 @@ +---Radio backend contract definition. +---All radio backends must implement the functions listed in RADIO_BACKEND_FUNCTIONS. + +local RADIO_BACKEND_FUNCTIONS = { + 'get_meta', + 'apply', + 'clear', + 'start_client_monitor', + 'watch_clients_op', + 'get_connected_macs', + 'get_iface_info', + 'get_iface_survey', + 'get_station_info', +} + +---Validate that a backend table implements all required functions. +---@param backend table +---@return boolean ok +---@return string err Empty string on success. +local function validate(backend) + if type(backend) ~= 'table' then + return false, "backend must be a table" + end + for _, fn_name in ipairs(RADIO_BACKEND_FUNCTIONS) do + if type(backend[fn_name]) ~= 'function' then + return false, "backend missing required function: " .. fn_name + end + end + return true, "" +end + +return { + RADIO_BACKEND_FUNCTIONS = RADIO_BACKEND_FUNCTIONS, + validate = validate, +} diff --git a/src/services/hal/backends/radio/provider.lua b/src/services/hal/backends/radio/provider.lua new file mode 100644 index 00000000..9353dc6b --- /dev/null +++ b/src/services/hal/backends/radio/provider.lua @@ -0,0 +1,32 @@ +---Radio backend provider. +---Iterates registered backends and returns the first supported one. + +local contract = require "services.hal.backends.radio.contract" + +local BACKENDS = { + "services.hal.backends.radio.providers.openwrt", +} + +---Instantiate a new backend for the named radio from the first supported provider. +---@param name string UCI radio section name (e.g. "radio0") +---@return table|nil backend instance +---@return string err "" on success +local function new(name) + for _, path in ipairs(BACKENDS) do + local ok, provider = pcall(require, path) + if ok and provider.is_supported and provider.is_supported() then + local backend = provider.backend.new(name) + local valid, verr = contract.validate(backend) + if valid then + return backend, "" + else + return nil, "backend " .. path .. " failed contract check: " .. verr + end + end + end + return nil, "no supported radio backend found on this device" +end + +return { + new = new, +} diff --git a/src/services/hal/backends/radio/providers/openwrt/impl.lua b/src/services/hal/backends/radio/providers/openwrt/impl.lua new file mode 100644 index 00000000..5bed0d8c --- /dev/null +++ b/src/services/hal/backends/radio/providers/openwrt/impl.lua @@ -0,0 +1,433 @@ +---OpenWrt radio backend implementation. +---All UCI writes use the shared uci singleton (reactor). Reads use get_value. +---Subprocess management (iw event, iw dev info, iw survey) is handled here. + +local uci = require "services.hal.backends.common.uci" +local fibers = require "fibers" +local op = require "fibers.op" +local scope = require "fibers.scope" +local exec = require "fibers.io.exec" +local file = require "fibers.io.file" + +local SYSFS_STATS = { + 'rx_bytes', 'tx_bytes', + 'rx_packets', 'tx_packets', + 'rx_dropped', 'tx_dropped', + 'rx_errors', 'tx_errors', +} + +local function owned_process_flags() + local flags = { process_group = true } + if type(exec.supports) == 'function' and exec.supports('parent_death_signal') then + flags.parent_death_signal = 'TERM' + end + return flags +end + +local function terminate_command(cmd, sig) + if not cmd or type(cmd.kill) ~= 'function' then return true, nil end + local ok, a, b = pcall(function() return cmd:kill(sig or 15) end) + if not ok then return nil, tostring(a) end + if a == false or a == nil then return nil, tostring(b or 'command kill failed') end + return true, nil +end + +local function shutdown_command_op(cmd, timeout) + return op.guard(function() + if not cmd then return op.always(true, '') end + return cmd:shutdown_op(timeout or 0.2):wrap(function(status, _code, _sig, err) + if status == 'exited' or status == 'signalled' then + return true, '' + end + return false, err or ('command shutdown failed: ' .. tostring(status)) + end) + end) +end + +---Read a single value from /sys/class/net//statistics/ +---@param iface string +---@param stat string +---@return number|nil +---@return string|nil err +local function read_sysfs_stat(iface, stat) + local path = '/sys/class/net/' .. iface .. '/statistics/' .. stat + local f, ferr = file.open(path, 'r') + if not f then return nil, tostring(ferr) end + local content, rerr = f:read_all() + f:close() + if not content then return nil, tostring(rerr) end + local num = tonumber((content or ''):match('(%d+)')) + if num == nil then return nil, stat .. ' invalid number' end + return num, nil +end + +---Parse `iw dev info` output into {txpower, channel, freq, width} +local function parse_iw_dev_info(raw) + if not raw or raw == '' then return nil end + local result = {} + for line in raw:gmatch('[^\r\n]+') do + local txpower = line:match('%s*txpower%s+([%d.]+)%s+dBm') + if txpower then result.txpower = tonumber(txpower) end + + local chan, freq, width = line:match( + '%s*channel%s+(%d+)%s+%(([%d]+)%s*MHz%),%s*width:%s*(%d+)%s*MHz' + ) + if chan then + result.channel = tonumber(chan) + result.freq = tonumber(freq) + result.width = tonumber(width) + end + end + return result +end + +---Parse `iw survey dump` for in-use noise value +local function parse_survey_noise(raw) + if not raw or raw == '' then return nil end + local in_use = false + for line in raw:gmatch('[^\r\n]+') do + if not in_use then + if line:find('%[in use%]') then in_use = true end + else + local val = line:match('noise:%s*(%-?%d+%.?%d*)') + if val then return tonumber(val) end + end + end + return nil +end + +---Parse `iw dev station get ` output into {signal, tx_bytes, rx_bytes} +local function parse_station_info(raw) + if not raw or raw == '' then return nil end + local result = {} + for line in raw:gmatch('[^\r\n]+') do + local sig = line:match('%s*signal:%s*(%-?%d+)%s*dBm') + if sig then result.signal = tonumber(sig) end + + local tx = line:match('%s*tx%s+bytes:%s+(%d+)') + if tx then result.tx_bytes = tonumber(tx) end + + local rx = line:match('%s*rx%s+bytes:%s+(%d+)') + if rx then result.rx_bytes = tonumber(rx) end + end + return result +end + +---Parse `iw dev station dump` output into a list of MAC addresses. +---Each station block starts with "Station (on )". +---@param raw string +---@return string[] list of MAC addresses +local function parse_station_dump_macs(raw) + if not raw or raw == '' then return {} end + local macs = {} + for mac in raw:gmatch('Station%s+([%x:]+)%s+%(on%s+%S+%)') do + macs[#macs + 1] = mac + end + return macs +end + +------------------------------------------------------------------------ +-- RadioBackend class +------------------------------------------------------------------------ + +---@class ClientEvent +---@field mac string +---@field added boolean true = new station, false = del station +---@field interface string + +---@class RadioBackend +---@field name string UCI wifi-device section name (e.g. "radio0") +---@field _monitor table|nil active ClientMonitor, set by start_client_monitor +local RadioBackend = {} +RadioBackend.__index = RadioBackend +RadioBackend.SYSFS_STATS = SYSFS_STATS + +---@param name string UCI radio section name (e.g. "radio0") +---@return RadioBackend +function RadioBackend.new(name) + return setmetatable({ name = name, _monitor = nil }, RadioBackend) +end + +---Get radio metadata from UCI wireless config. +---@return table|nil { path, type } +---@return string err "" on success +function RadioBackend:get_meta() + local path = uci.get_value('wireless', self.name, 'path') + local rtype = uci.get_value('wireless', self.name, 'type') + if not path then + return nil, "could not read wireless." .. self.name .. ".path from UCI" + end + return { path = path, type = rtype or '' }, "" +end + +---Apply the staged radio config table to UCI and reload wireless. +---Uses the shared UCI reactor for debounced writes. +---@param staged table Full staged config as accumulated by the driver +function RadioBackend:apply(staged) + uci.ensure_started() + local session = uci.new_session() + local name = self.name + + -- Delete any existing wifi-iface sections that belong to this radio but + -- are not in the staged interfaces set, so stale sections don't linger. + local staged_iface_names = {} + for _, iface in ipairs(staged.interfaces or {}) do + staged_iface_names[iface.name] = true + end + for _, sec in ipairs(uci.get_sections('wireless', 'wifi-iface')) do + if uci.get_value('wireless', sec, 'device') == name + and not staged_iface_names[sec] then + session:delete('wireless', sec) + end + end + + -- Radio section (wifi-device) — ensure it exists before setting options + session:set('wireless', name, 'wifi-device') + if staged.path and staged.path ~= '' then + session:set('wireless', name, 'path', staged.path) + end + if staged.type and staged.type ~= '' then + session:set('wireless', name, 'type', staged.type) + end + if staged.band then + session:set('wireless', name, 'band', staged.band) + end + if staged.channel ~= nil then + session:set('wireless', name, 'channel', staged.channel) + end + if staged.htmode then + session:set('wireless', name, 'htmode', staged.htmode) + end + if staged.channels then + session:set('wireless', name, 'channels', table.concat(staged.channels, ' ')) + end + if staged.txpower ~= nil then + session:set('wireless', name, 'txpower', staged.txpower) + end + if staged.country then + session:set('wireless', name, 'country', staged.country) + end + if staged.disabled ~= nil then + session:set('wireless', name, 'disabled', staged.disabled) + end + + -- Delete removed interfaces + for _, iface_name in ipairs(staged.deleted_interfaces or {}) do + session:delete('wireless', iface_name) + end + + -- Write interface sections (wifi-iface) + for _, iface in ipairs(staged.interfaces or {}) do + -- Ensure the named section exists with the correct type before setting options + session:set('wireless', iface.name, 'wifi-iface') + session:set('wireless', iface.name, 'ifname', iface.name) + session:set('wireless', iface.name, 'device', name) + session:set('wireless', iface.name, 'mode', iface.mode or 'ap') + session:set('wireless', iface.name, 'ssid', iface.ssid) + session:set('wireless', iface.name, 'encryption', iface.encryption) + session:set('wireless', iface.name, 'key', iface.password or '') + session:set('wireless', iface.name, 'network', iface.network) + if iface.enable_steering then + session:set('wireless', iface.name, 'bss_transition', '1') + session:set('wireless', iface.name, 'ieee80211k', '1') + session:set('wireless', iface.name, 'rrm_neighbor_report', '1') + session:set('wireless', iface.name, 'rrm_beacon_report', '1') + else + session:set('wireless', iface.name, 'bss_transition', '0') + session:set('wireless', iface.name, 'ieee80211k', '0') + session:set('wireless', iface.name, 'rrm_neighbor_report', '0') + session:set('wireless', iface.name, 'rrm_beacon_report', '0') + end + end + + local ok, err = session:commit('wireless', { { 'wifi', 'reload' } }) + if not ok then + error('apply commit failed: ' .. tostring(err)) + end +end + +---Delete all UCI config owned by this radio and reload wireless. +---Removes all wifi-iface sections whose device == self.name, then +---deletes the configurable options from the wifi-device section. +function RadioBackend:clear() + uci.ensure_started() + local session = uci.new_session() + local name = self.name + + -- Delete all wifi-iface sections that belong to this radio + for _, sec in ipairs(uci.get_sections('wireless', 'wifi-iface')) do + if uci.get_value('wireless', sec, 'device') == name then + session:delete('wireless', sec) + end + end + + -- Delete the configurable options on the wifi-device section + -- (leave 'path' and 'type' intact since they are hardware facts) + local OPT_KEYS = { 'band', 'channel', 'channels', 'htmode', 'txpower', + 'country', 'disabled' } + for _, opt in ipairs(OPT_KEYS) do + if uci.get_value('wireless', name, opt) ~= nil then + session:delete('wireless', name, opt) + end + end + + local ok, err = session:commit('wireless', { { 'wifi', 'reload' } }) + if not ok then + error('clear failed: ' .. tostring(err)) + end +end + +---Parse a single `iw event` line into a ClientEvent, or nil if not a station event. +---Matches: ": new station " / ": del station " +---@param line string +---@return ClientEvent|nil +local function parse_client_event_line(line) + local iface, verb, mac = line:match('^(%S-):%s+(%a+)%s+station%s+([%x:]+)$') + if not iface then return nil end + if verb ~= 'new' and verb ~= 'del' then return nil end + return { mac = mac, added = verb == 'new', interface = iface } +end + +---Start the iw event subprocess for client monitoring. +---Stores the process and stream in self._monitor; idempotent if already started. +---@return boolean ok +---@return string err +function RadioBackend:start_client_monitor() + if self._monitor then + return false, 'client monitor already started' + end + local cmd = exec.command { + 'iw', 'event', + stdin = 'null', + stdout = 'pipe', + stderr = 'null', + shutdown_grace = 0.2, + flags = owned_process_flags(), + } + local stdout, err = cmd:stdout_stream() + if not stdout then + return false, 'failed to start iw event: ' .. tostring(err) + end + self._monitor = { cmd = cmd, stdout = stdout } + return true, '' +end + +---Immediate best-effort stop for finalisers. +---@param reason string? +function RadioBackend:terminate(reason) + local mon = self._monitor + self._monitor = nil + if not mon then return true, nil end + if mon.stdout then pcall(function() mon.stdout:terminate(reason or 'radio monitor terminated') end) end + return terminate_command(mon.cmd, 15) +end + +---Gracefully stop the iw event subprocess. +---@param timeout number? +---@return Op +function RadioBackend:stop_client_monitor_op(timeout) + return op.guard(function() + local mon = self._monitor + self._monitor = nil + if not mon then return op.always(true, '') end + if mon.stdout then pcall(function() mon.stdout:terminate('radio monitor stopped') end) end + return shutdown_command_op(mon.cmd, timeout or 0.2) + end) +end + +---Return an op that blocks until the next client connect/disconnect event +---for an interface in the monitor's interfaces_set, then returns a ClientEvent. +---Each perform of the op yields exactly one matching event. +---@return Op +function RadioBackend:watch_clients_op() + return op.guard(function() + if not self._monitor then + return op.always(nil, 'client monitor not started') + end + return scope.run_op(function(s) + while true do + ---@diagnostic disable-next-line: need-check-nil + local line = s:perform(self._monitor.stdout:read_line_op()) --[[@as string?]] + if not line then + return nil, 'iw event stream closed' + end + local ev = parse_client_event_line(line) + if ev then + return ev + end + end + end):wrap(function(st, _, ...) + if st == 'ok' then + return ... + elseif st == 'cancelled' then + return nil, 'cancelled' + else + return nil, (... or 'iw event monitor failed') + end + end) + end) +end + +---Get interface info: txpower, channel, freq, width. +---@param iface string +---@return table|nil { txpower, channel, freq, width } +---@return string err +function RadioBackend:get_iface_info(iface) + local proc = exec.command { 'iw', 'dev', iface, 'info', stdin = 'null', stdout = 'pipe', stderr = 'stdout' } + local out, status, _, _, err = fibers.perform(proc:output_op()) + if status ~= 'exited' or err then return nil, tostring(err or 'iw dev info failed') end + local result = parse_iw_dev_info(out) + if not result then return nil, "could not parse iw dev info output" end + return result, "" +end + +---Get noise floor for an interface via survey dump. +---@param iface string +---@return number|nil noise value +---@return string err +function RadioBackend:get_iface_survey(iface) + local proc = exec.command { 'iw', iface, 'survey', 'dump', stdin = 'null', stdout = 'pipe', stderr = 'stdout' } + local out, status, _, _, err = fibers.perform(proc:output_op()) + if status ~= 'exited' or err then return nil, tostring(err or 'iw survey failed') end + local noise = parse_survey_noise(out) + if noise == nil then return nil, "noise value not found" end + return noise, "" +end + +---Get per-client stats: signal, tx_bytes, rx_bytes. +---@param iface string +---@param mac string +---@return table|nil { signal, tx_bytes, rx_bytes } +---@return string err +function RadioBackend:get_station_info(iface, mac) + local proc = exec.command { 'iw', 'dev', iface, 'station', 'get', mac, stdin = 'null', stdout = 'pipe', stderr = 'stdout' } + local out, status, _, _, err = fibers.perform(proc:output_op()) + if status ~= 'exited' or err then return nil, tostring(err or 'iw station get failed') end + local result = parse_station_info(out) + if not result then return nil, "could not parse station info" end + return result, "" +end + +---Get all currently associated MAC addresses for an interface. +---@param iface string +---@return string[] list of MAC addresses (may be empty) +---@return string err +function RadioBackend:get_connected_macs(iface) + local proc = exec.command { 'iw', 'dev', iface, 'station', 'dump', stdin = 'null', stdout = 'pipe', stderr = 'stdout' } + local out, status, _, _, err = fibers.perform(proc:output_op()) + if status ~= 'exited' or err then return {}, tostring(err or 'iw station dump failed') end + return parse_station_dump_macs(out), '' +end + +---Read a sysfs statistics value for an interface. +---@param iface string +---@param stat string +---@return number|nil +---@return string|nil err +function RadioBackend:read_sysfs_stat(iface, stat) + return read_sysfs_stat(iface, stat) +end + +return { + new = RadioBackend.new, +} diff --git a/src/services/hal/backends/radio/providers/openwrt/init.lua b/src/services/hal/backends/radio/providers/openwrt/init.lua new file mode 100644 index 00000000..8269631a --- /dev/null +++ b/src/services/hal/backends/radio/providers/openwrt/init.lua @@ -0,0 +1,15 @@ +local impl = require "services.hal.backends.radio.providers.openwrt.impl" +local file = require "fibers.io.file" + +---Check whether OpenWrt UCI is available on this device. +---@return boolean +local function is_supported() + local f, _ = file.open('/etc/openwrt_release', 'r') + if f then f:close() return true end + return false +end + +return { + is_supported = is_supported, + backend = impl, +} diff --git a/src/services/hal/backends/time/contract.lua b/src/services/hal/backends/time/contract.lua new file mode 100644 index 00000000..c1da2caf --- /dev/null +++ b/src/services/hal/backends/time/contract.lua @@ -0,0 +1,26 @@ +---@class TimeBackend +---@field start_ntp_monitor fun(self: TimeBackend): boolean, string +---@field ntp_event_op fun(self: TimeBackend): Op +---@field stop fun(self: TimeBackend): boolean, string + +local BACKEND_FUNCTIONS = { + "start_ntp_monitor", + "ntp_event_op", + "stop", +} + +---Check that a time backend provides all required functions. +---@param backend TimeBackend +---@return string error Empty string on success. +local function validate(backend) + for _, func in ipairs(BACKEND_FUNCTIONS) do + if type(backend[func]) ~= "function" then + return "Missing required function: " .. func + end + end + return "" +end + +return { + validate = validate +} diff --git a/src/services/hal/backends/time/provider.lua b/src/services/hal/backends/time/provider.lua new file mode 100644 index 00000000..5377c00c --- /dev/null +++ b/src/services/hal/backends/time/provider.lua @@ -0,0 +1,49 @@ +---Time backend provider factory. +---Selects the appropriate TimeBackend implementation based on the runtime platform. + +local contract = require "services.hal.backends.time.contract" + +local BACKENDS = { + "openwrt" +} + +--- Select and initialize the backend implementation +---@return table backend_impl +local function get_backend_impl() + local backend_impl = nil + for _, backend_name in ipairs(BACKENDS) do + local ok, backend_mod = pcall(require, "services.hal.backends.time.providers." .. backend_name .. ".init") + if ok and type(backend_mod) == "table" and backend_mod.is_supported and backend_mod.is_supported() then + backend_impl = backend_mod.backend + break + end + end + + if backend_impl == nil then + error("No supported time backend found") + end + + return backend_impl +end + +---Create a new TimeBackend instance. +--- +---Detects the platform and instantiates the appropriate backend implementation. +---Fails with an error if no supported backend is found. +--- +---@return TimeBackend +local function new() + local backend_impl = get_backend_impl() + local backend = backend_impl.new() + + local iface_err = contract.validate(backend) + if iface_err ~= "" then + error("Time backend does not implement required interface: " .. tostring(iface_err)) + end + + return backend +end + +return { + new = new, +} diff --git a/src/services/hal/backends/time/providers/openwrt/impl.lua b/src/services/hal/backends/time/providers/openwrt/impl.lua new file mode 100644 index 00000000..c004f0bd --- /dev/null +++ b/src/services/hal/backends/time/providers/openwrt/impl.lua @@ -0,0 +1,216 @@ +---OpenWrt-specific TimeBackend implementation using ubus hotplug.ntp events. + +-- Service modules +local time_types = require "services.hal.types.time" + +-- Fibers modules +local op = require "fibers.op" +local exec = require "fibers.io.exec" + +-- Other modules +local cjson = require "cjson.safe" + +local function owned_process_flags() + local flags = { process_group = true } + if type(exec.supports) == 'function' and exec.supports('parent_death_signal') then + flags.parent_death_signal = 'TERM' + end + return flags +end + +local function terminate_command(cmd, sig) + if not cmd or type(cmd.kill) ~= 'function' then return true, nil end + local ok, a, b = pcall(function() return cmd:kill(sig or 15) end) + if not ok then return nil, tostring(a) end + if a == false or a == nil then return nil, tostring(b or 'command kill failed') end + return true, nil +end + +local function shutdown_command_op(cmd, timeout) + return op.guard(function() + if not cmd then return op.always(true, '') end + return cmd:shutdown_op(timeout or 0.2):wrap(function(status, _code, _sig, err) + if status == 'exited' or status == 'signalled' then + return true, '' + end + return false, err or ('command shutdown failed: ' .. tostring(status)) + end) + end) +end + +---@class OpenWrtTimeBackend : TimeBackend +---@field ntp_monitor_stream Stream? Current ubus listen stream +---@field ntp_monitor_cmd Command? Current ubus listen command +local OpenWrtTimeBackend = {} +OpenWrtTimeBackend.__index = OpenWrtTimeBackend + +---- Private Utilities ---- + +---Recursively convert numeric-looking strings into numbers. +---@param value any +---@return any +local function coerce_numeric_strings(value) + if type(value) == 'string' then + local n = tonumber(value) + if n ~= nil then + return n + end + return value + end + + if type(value) == 'table' then + for k, v in pairs(value) do + value[k] = coerce_numeric_strings(v) + end + return value + end + + return value +end + +---Parse a single ubus listen hotplug.ntp line into a strongly typed NTPEvent. +--- +---Called as a wrap function on read_line_op(), receiving (line, read_err). +---Returns: +--- (NTPEvent, nil) -- success +--- (nil, err_string) -- fatal error; caller should break the monitor loop +--- +---@param line string? +---@param read_err any? +---@return any? +---@return string? +local function parse_ntp_event_line(line, read_err) + if read_err ~= nil then + return nil, "read error: " .. tostring(read_err) + end + + if line == nil or line == "" then + return nil, "stream closed" + end + + local decoded = cjson.decode(line) + if not decoded then + return nil, "decode failed: " .. line + end + + decoded = coerce_numeric_strings(decoded) + local ntp_data = decoded["hotplug.ntp"] + if type(ntp_data) ~= 'table' then + return nil, "missing hotplug.ntp key: " .. line + end + + if type(ntp_data.stratum) ~= 'number' then + return nil, "invalid stratum: " .. line + end + + local action = ntp_data.action or "unknown" + local offset = ntp_data.offset or 0 + local freq_drift_ppm = ntp_data.freq_drift_ppm or 0 + + local ntp_event, event_err = time_types.new.NTPEvent( + ntp_data.stratum, + action, + offset, + freq_drift_ppm + ) + if not ntp_event then + return nil, "NTPEvent construction failed: " .. tostring(event_err) + end + + for k, v in pairs(ntp_data) do + if ntp_event[k] == nil then + ntp_event[k] = v + end + end + + return ntp_event, nil +end + +---- Backend Lifecycle ---- + +---Start monitoring NTP synchronization events via ubus hotplug.ntp. +--- +---@return boolean ok +---@return string error Empty string on success. +function OpenWrtTimeBackend:start_ntp_monitor() + if self.ntp_monitor_cmd then + return false, "NTP monitor already running" + end + + -- Start ubus listen command bound to current scope + self.ntp_monitor_cmd = exec.command{ + 'ubus', 'listen', 'hotplug.ntp', + stdin = 'null', + stdout = 'pipe', + stderr = 'null', + shutdown_grace = 0.2, + flags = owned_process_flags(), + } + local stream, stream_err = self.ntp_monitor_cmd:stdout_stream() + if not stream then + return false, "failed to start ubus listen: " .. tostring(stream_err) + end + + self.ntp_monitor_stream = stream + return true, "" +end + +---Get an operation that yields the next NTP event from the hotplug.ntp stream. +--- +---Returns (NTPEvent, nil) on success, (nil, nil) on a parse error (caller should +---retry), or (nil, err_string) on a fatal error (stream closed or read error). +--- +---@return Op +function OpenWrtTimeBackend:ntp_event_op() + return op.guard(function() + if not self.ntp_monitor_stream then + error("NTP monitor not started") + end + return self.ntp_monitor_stream:read_line_op():wrap(parse_ntp_event_line) + end) +end + +---Stop the NTP monitor and clean up resources. +--- +---@return boolean ok +---@return string error +function OpenWrtTimeBackend:terminate(reason) + local stream = self.ntp_monitor_stream + local cmd = self.ntp_monitor_cmd + self.ntp_monitor_stream = nil + self.ntp_monitor_cmd = nil + if stream then pcall(function() stream:terminate(reason or 'ntp monitor terminated') end) end + return terminate_command(cmd, 15) +end + +function OpenWrtTimeBackend:shutdown_op(timeout) + return op.guard(function() + local stream = self.ntp_monitor_stream + local cmd = self.ntp_monitor_cmd + self.ntp_monitor_stream = nil + self.ntp_monitor_cmd = nil + if stream then pcall(function() stream:terminate('ntp monitor stopped') end) end + return shutdown_command_op(cmd, timeout or 0.2) + end) +end + +function OpenWrtTimeBackend:stop() + local fibers = require 'fibers' + return fibers.perform(self:shutdown_op(0.2)) +end + +---- Constructor ---- + +---Create a new OpenWrt time backend. +--- +---@return OpenWrtTimeBackend +local function new() + return setmetatable({ + ntp_monitor_stream = nil, + ntp_monitor_cmd = nil, + }, OpenWrtTimeBackend) +end + +return { + new = new, +} diff --git a/src/services/hal/backends/time/providers/openwrt/init.lua b/src/services/hal/backends/time/providers/openwrt/init.lua new file mode 100644 index 00000000..ec563590 --- /dev/null +++ b/src/services/hal/backends/time/providers/openwrt/init.lua @@ -0,0 +1,48 @@ +local file = require "fibers.io.file" +local exec = require "fibers.io.exec" +local fibers = require "fibers" + +local backend = require "services.hal.backends.time.providers.openwrt.impl" + +local function is_linux() + local fh, open_err = file.open("/proc/version", "r") + if not fh or open_err then + return false + end + + local content, read_err = fh:read_all() + fh:close() + if not content or read_err then + return false + end + + return content:lower():find("linux") ~= nil +end + +--- Returns true if `ubus` is available and the daemon is reachable +---@return boolean ok +local function has_ubus() + local cmd = exec.command{ + "ubus", "list", + stdin = "null", + stdout = "pipe", + stderr = "null" + } + local _, status, code = fibers.perform(cmd:combined_output_op()) + if status == "exited" and code == 0 then + return true + end + return false +end + +--- Returns true if this is a supported OpenWrt system +---@return boolean +local function is_supported() + local res = is_linux() and has_ubus() + return res +end + +return { + is_supported = is_supported, + backend = backend +} diff --git a/src/services/hal/backends/wired/contract.lua b/src/services/hal/backends/wired/contract.lua new file mode 100644 index 00000000..485b6dce --- /dev/null +++ b/src/services/hal/backends/wired/contract.lua @@ -0,0 +1,22 @@ +-- services/hal/backends/wired/contract.lua +-- Semantic wired-provider backend contract. +-- +-- Providers expose product-level wired surfaces. They do not expose switch ASIC +-- registers, HTTP endpoints, DSA syntax or OpenWrt implementation details. +-- +-- Phase 1 providers are observation-only and must return read_only for control. + +local M = {} + +M.CAP_CLASS = 'wired-provider' +M.SCHEMA = 'devicecode.hal.wired-provider/1' + +function M.read_only(method) + return { + ok = false, + code = 'read_only', + err = tostring(method or 'operation') .. ' is not available on this read-only wired provider', + } +end + +return M diff --git a/src/services/hal/backends/wired/provider.lua b/src/services/hal/backends/wired/provider.lua new file mode 100644 index 00000000..4990f469 --- /dev/null +++ b/src/services/hal/backends/wired/provider.lua @@ -0,0 +1,25 @@ +-- services/hal/backends/wired/provider.lua +-- Thin provider loader for semantic wired-provider backends. + +local M = {} + +local function provider_name(config) + if type(config) ~= 'table' then return nil, 'wired provider config must be a table' end + if type(config.provider) ~= 'string' or config.provider == '' then return nil, 'wired provider config requires provider' end + return config.provider, nil +end + +function M.new(config, opts) + config = config or {} + local name, name_err = provider_name(config) + if not name then return nil, name_err end + local modname = 'services.hal.backends.wired.providers.' .. name + local ok, mod = pcall(require, modname) + if not ok then return nil, ('wired provider %s not available: %s'):format(name, tostring(mod)) end + if type(mod) ~= 'table' or type(mod.new) ~= 'function' then return nil, 'wired provider module must export new(config, opts)' end + local backend, err = mod.new(config, opts or {}) + if not backend then return nil, err end + return backend, nil, name +end + +return M diff --git a/src/services/hal/backends/wired/providers/rtl8380m_http.lua b/src/services/hal/backends/wired/providers/rtl8380m_http.lua new file mode 100644 index 00000000..e7ebf250 --- /dev/null +++ b/src/services/hal/backends/wired/providers/rtl8380m_http.lua @@ -0,0 +1,1058 @@ +-- services/hal/backends/wired/providers/rtl8380m_http.lua +-- +-- Read-only HTTP provider for RTL8380-family PoE/VLAN switches. +-- +-- The provider keeps the manufacturer HTTP/CGI/RSA-login details below HAL and +-- returns provider-shaped raw observations for the Wired service. It deliberately +-- leaves writes unsupported until the control path has dedicated tests against +-- the switch UI's set.cgi forms. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local exec = require 'fibers.io.exec' +local contract = require 'services.hal.backends.wired.contract' +local tablex = require 'shared.table' +local blob_source = require 'devicecode.blob_source' + +local ok_cjson, cjson = pcall(require, 'cjson.safe') +if not ok_cjson then cjson = require 'cjson' end + +local M = {} +local Provider = {} +Provider.__index = Provider + +local DRIVER = 'rtl8380m_http' +local USER_AGENT = 'devicecode-rtl8380m-http/1 lua-http' + +local READ_COMMANDS = { + 'home_main', + 'panel_info', + 'sys_sysinfo', + 'port_port', + 'vlan_create', + 'vlan_conf', + 'vlan_port', + 'vlan_membership', + 'poe_poe', + 'lldp_local', + 'lldp_neighbor', + 'sys_cpumem', + 'rmon_statistics', +} + +local COMMAND_GROUPS = { + panel = { 'home_main', 'panel_info' }, + identity = { 'sys_sysinfo' }, + vlan = { 'home_main', 'vlan_create', 'vlan_conf', 'vlan_port', 'vlan_membership' }, + poe = { 'home_main', 'poe_poe' }, + lldp = { 'lldp_local', 'lldp_neighbor' }, + runtime = { 'sys_cpumem' }, + counters = { 'home_main', 'rmon_statistics' }, +} + +local VLAN_MODE = { + [0] = 'hybrid', + [1] = 'access', + [2] = 'trunk', + [3] = 'tunnel', +} + +local VLAN_ACCEPT_FRAME = { + [0] = 'all', + [1] = 'tag_only', + [2] = 'untag_only', +} + +local VLAN_MEMBERSHIP = { + [0] = 'excluded', + [2] = 'tagged', + [3] = 'untagged', +} + +local function copy(v) return tablex.deep_copy(v) end + +local function merge_table(dst, src) + dst = dst or {} + if type(src) ~= 'table' then return dst end + for k, v in pairs(src) do + if v ~= nil then + if type(v) == 'table' and type(dst[k]) == 'table' then + merge_table(dst[k], v) + else + dst[k] = copy(v) + end + end + end + return dst +end + +local function append_unique(out, seen, value) + value = tostring(value or '') + if value ~= '' and not seen[value] then + seen[value] = true + out[#out + 1] = value + end +end + +local function commands_for_groups(groups) + local out, seen = {}, {} + for _, group in ipairs(groups or {}) do + local commands = COMMAND_GROUPS[tostring(group or '')] + if not commands then return nil, 'unknown command group: ' .. tostring(group) end + for _, cmd in ipairs(commands) do append_unique(out, seen, cmd) end + end + return out, nil +end + +local function trim(s) + return (tostring(s or ''):gsub('^%s+', ''):gsub('%s+$', '')) +end + +local function getenv_ref(v) + if type(v) ~= 'string' then return v end + local name = v:match('^%$([%w_]+)$') + if name then return os.getenv(name) end + return v +end + +local function ensure_base_url(url) + if type(url) ~= 'string' or url == '' then return nil, 'base_url is required' end + if not url:match('^https?://') then return nil, 'base_url must include http:// or https:// scheme' end + if not url:match('/$') then return nil, 'base_url must end with /' end + return url, nil +end + +local function parse_origin(url) + local scheme, hostport = tostring(url or ''):match('^(https?)://([^/]+)') + if not scheme then return nil end + return scheme .. '://' .. hostport +end + +local function cgi_url(base_url, kind, cmd, dummy) + local origin = assert(parse_origin(base_url), 'invalid switch base URL') + local url = origin .. '/cgi/' .. kind .. '.cgi?cmd=' .. tostring(cmd or '') + if dummy then url = url .. '&dummy=' .. tostring(dummy) end + return url +end + +local function shell_quote(s) + s = tostring(s or '') + return "'" .. s:gsub("'", "'\\''") .. "'" +end + + + + +local CookieJar = {} +CookieJar.__index = CookieJar + +function CookieJar.new(initial) + local self = setmetatable({ values = {}, order = {} }, CookieJar) + for name, value in pairs(initial or {}) do + name = tostring(name or '') + if name ~= '' then + self.order[#self.order + 1] = name + self.values[name] = tostring(value or '') + end + end + return self +end + +function CookieJar:set(name, value) + name = trim(name or '') + if name == '' then return end + if self.values[name] == nil then self.order[#self.order + 1] = name end + self.values[name] = tostring(value or '') +end + +function CookieJar:update(set_cookie) + if type(set_cookie) ~= 'string' or set_cookie == '' then return end + local first = set_cookie:match('^%s*([^;]+)') + if not first then return end + local name, value = first:match('^%s*([^=]+)=(.*)$') + if not name then return end + self:set(name, value or '') +end + +function CookieJar:header() + local out = {} + for _, name in ipairs(self.order) do + if self.values[name] ~= nil then out[#out + 1] = name .. '=' .. self.values[name] end + end + return table.concat(out, '; ') +end + +function CookieJar:names() + local out = {} + for _, name in ipairs(self.order) do out[#out + 1] = name end + return out +end + +local function header_values(headers, name) + name = tostring(name or ''):lower() + local values = {} + if type(headers) == 'table' then + local v = headers[name] + if type(v) == 'table' then + for i = 1, #v do values[#values + 1] = tostring(v[i] or '') end + elseif v ~= nil then + values[#values + 1] = tostring(v) + else + for k, hv in pairs(headers) do + if tostring(k):lower() == name then values[#values + 1] = tostring(hv or '') end + end + end + end + return values +end + +local function reset_session(self) + self.jar = CookieJar.new({ cookie_language = 'defLang_en' }) + self.logged_in = false +end + +local function auth_invalid_body(body) + local s = tostring(body or ''):lower() + return s:find('login.html', 1, true) ~= nil + or s:find('home_login', 1, true) ~= nil + or s:find('loginstatus', 1, true) ~= nil +end + +local function request(self, method, url, body, extra_headers) + if not self.http_ref or type(self.http_ref.exchange_op) ~= 'function' then + return nil, 'http capability ref not configured' + end + + local req_headers = { + ['user-agent'] = USER_AGENT, + accept = 'application/json,text/plain,*/*', + connection = 'close', + } + local cookie = self.jar:header() + if cookie ~= '' then req_headers.cookie = cookie end + for k, v in pairs(extra_headers or {}) do req_headers[k] = tostring(v) end + + local sink = blob_source.to_memory() + local args = { + uri = url, + method = tostring(method or 'GET'):upper(), + headers = req_headers, + response_sink = sink, + } + if body ~= nil then args.body_source = blob_source.from_string(body or '') end + if self.response_parser and tostring(url):match('/cgi/[gs]et%.cgi%?') then + args.response_parser = self.response_parser + args.timeout_s = self.timeout_s + args.max_response_bytes = self.max_response_bytes + end + + local reply, err = fibers.perform(self.http_ref:exchange_op(args, { timeout = self.timeout_s })) + if not reply then return nil, err end + local result = reply.result or reply + local status = tonumber(result and result.status or 0) or 0 + local headers = result and result.headers or {} + for _, sc in ipairs(header_values(headers, 'set-cookie')) do self.jar:update(sc) end + return { ok = true, status = status, headers = headers, body = sink:result() } +end + +local function json_decode(body) + local t, err = cjson.decode(tostring(body or '')) + if type(t) ~= 'table' then return nil, err or 'invalid JSON response' end + return t, nil +end + +local b64chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' +local function base64(data) + data = tostring(data or '') + local data_len = #data + return ((data:gsub('.', function(x) + local r, b = '', x:byte() + for i = 8, 1, -1 do r = r .. ((b % 2 ^ i - b % 2 ^ (i - 1) > 0) and '1' or '0') end + return r + end) .. '0000'):gsub('%d%d%d?%d?%d?%d?', function(x) + if #x < 6 then return '' end + local c = 0 + for i = 1, 6 do c = c + ((x:sub(i, i) == '1') and 2 ^ (6 - i) or 0) end + return b64chars:sub(c + 1, c + 1) + end) .. ({ '', '==', '=' })[data_len % 3 + 1]) +end + +local function json_escape(s) + s = tostring(s or '') + return '"' .. s:gsub('\\', '\\\\'):gsub('"', '\\"'):gsub('\n', '\\n'):gsub('\r', '\\r'):gsub('\t', '\\t') .. '"' +end + +local function json_empty_object_at_key(key) + return '{' .. json_escape(key) .. ':{}}' +end + +local function form_encode(fields) + local function enc(s) + s = tostring(s or '') + s = s:gsub('\n', '\r\n') + s = s:gsub('([^%w%-_%.~ ])', function(c) return string.format('%%%02X', c:byte()) end) + return s:gsub(' ', '+') + end + local keys, out = {}, {} + for k in pairs(fields or {}) do keys[#keys + 1] = k end + table.sort(keys) + for _, k in ipairs(keys) do out[#out + 1] = enc(k) .. '=' .. enc(fields[k]) end + return table.concat(out, '&') +end + +local function write_file_raw(path, data) + local f = assert(io.open(path, 'wb')) + f:write(data or '') + f:close() +end + +local function read_file(path) + local f = io.open(path, 'rb') + if not f then return nil end + local data = f:read('*a') + f:close() + return data +end + +local function der_len(n) + if n < 128 then return string.char(n) end + local bytes = {} + while n > 0 do table.insert(bytes, 1, string.char(n % 256)); n = math.floor(n / 256) end + return string.char(0x80 + #bytes) .. table.concat(bytes) +end + +local function der(tag, content) return string.char(tag) .. der_len(#(content or '')) .. (content or '') end +local function hex_to_bin(hex) + hex = tostring(hex or ''):gsub('%s+', ''):gsub(':', '') + if #hex % 2 == 1 then hex = '0' .. hex end + return (hex:gsub('..', function(cc) return string.char(tonumber(cc, 16) or 0) end)) +end +local function der_integer_bin(b) + b = tostring(b or '') + while #b > 1 and b:byte(1) == 0 do b = b:sub(2) end + if #b == 0 then b = string.char(0) end + if b:byte(1) >= 0x80 then b = string.char(0) .. b end + return der(0x02, b) +end +local function der_integer_from_hex(hex) return der_integer_bin(hex_to_bin(hex)) end +local function der_integer_from_number(n) + local bytes = {} + repeat table.insert(bytes, 1, string.char(n % 256)); n = math.floor(n / 256) until n == 0 + return der_integer_bin(table.concat(bytes)) +end +local function pem_wrap(label, der_bytes) + local b, lines = base64(der_bytes), {} + for i = 1, #b, 64 do lines[#lines + 1] = b:sub(i, i + 63) end + return '-----BEGIN ' .. label .. '-----\n' .. table.concat(lines, '\n') .. '\n-----END ' .. label .. '-----\n' +end +local function rsa_public_key_pem(modulus_hex) + local rsa_pub = der(0x30, der_integer_from_hex(modulus_hex) .. der_integer_from_number(65537)) + local alg_id = der(0x30, der(0x06, hex_to_bin('2A864886F70D010101')) .. der(0x05, '')) + return pem_wrap('PUBLIC KEY', der(0x30, alg_id .. der(0x03, string.char(0) .. rsa_pub))) +end + +local function run_openssl(argv) + local spec = { stdin = 'null', stdout = 'null', stderr = 'null' } + for i = 1, #argv do spec[i] = argv[i] end + local cmd = exec.command(spec) + local status, code = fibers.perform(cmd:run_op()) + return status == 'exited' and code == 0 +end + +local function openssl_encrypt_b64(self, modulus_hex, plaintext) + local prefix = os.tmpname() + local pub_path, in_path, out_path = prefix .. '.pub.pem', prefix .. '.plain', prefix .. '.cipher' + write_file_raw(pub_path, rsa_public_key_pem(modulus_hex)) + write_file_raw(in_path, plaintext or '') + local ok = run_openssl({ + self.openssl_bin, + 'pkeyutl', '-encrypt', '-pubin', + '-inkey', pub_path, + '-pkeyopt', 'rsa_padding_mode:pkcs1', + '-in', in_path, + '-out', out_path, + }) + if not ok then + ok = run_openssl({ + self.openssl_bin, + 'rsautl', '-encrypt', '-pubin', '-pkcs', + '-inkey', pub_path, + '-in', in_path, + '-out', out_path, + }) + end + local cipher = read_file(out_path) + os.remove(pub_path); os.remove(in_path); os.remove(out_path) + if not ok or not cipher or cipher == '' then return nil, 'openssl_rsa_encrypt_failed' end + return base64(cipher) +end + +local function response_status_ok(body) + return tostring(body or ''):find('"status"%s*:%s*"ok"') ~= nil +end + +local function login(self, opts) + opts = opts or {} + if self.disable_login then return true, nil end + if self.logged_in and not opts.force then return true, nil end + if not self.username or not self.password then return false, 'switch username/password not configured' end + reset_session(self) + local info_url = cgi_url(self.base_url, 'get', 'home_login') + local info, err = request(self, 'GET', info_url) + if not info or info.status ~= 200 then reset_session(self); return false, err or ('home_login HTTP ' .. tostring(info and info.status)) end + local info_json, jerr = json_decode(info.body) + if not info_json then reset_session(self); return false, 'home_login invalid JSON: ' .. tostring(jerr) end + local modulus = info_json.data and info_json.data.modulus + if not modulus then reset_session(self); return false, 'home_login response missing RSA modulus' end + local encrypted, enc_err = openssl_encrypt_b64(self, modulus, self.password) + if not encrypted then reset_session(self); return false, enc_err end + local form_data = '_ds=1&' .. form_encode({ username = self.username, password = encrypted }) .. '&_de=1' + local auth_url = cgi_url(self.base_url, 'set', 'home_loginAuth', os.time()) + local origin = parse_origin(self.base_url) + local login_html = origin .. '/login.html' + local bodies = { + { body = json_empty_object_at_key(form_data), content_type = 'application/json' }, + { body = form_data, content_type = 'application/json' }, + { body = form_data, content_type = 'application/x-www-form-urlencoded' }, + } + for _, candidate in ipairs(bodies) do + local headers = { + ['content-type'] = candidate.content_type, + ['accept'] = 'application/json, text/javascript, */*; q=0.01', + ['x-requested-with'] = 'XMLHttpRequest', + ['referer'] = login_html, + ['origin'] = origin, + } + request(self, 'POST', auth_url, candidate.body, headers) + for _ = 1, 6 do + local status = request(self, 'GET', cgi_url(self.base_url, 'get', 'home_loginStatus')) + if status and response_status_ok(status.body) then self.logged_in = true; return true, nil end + end + end + reset_session(self) + return false, 'RTL8380 RSA login was not confirmed' +end + +local function get_cmd(self, cmd) + local url = cgi_url(self.base_url, 'get', cmd) + if cmd == 'rmon_statistics' then url = url .. '&time=0' end + local r, err = request(self, 'GET', url) + if not r then return nil, err, 'transport' end + if r.status == 401 or r.status == 403 then return nil, ('%s HTTP %s'):format(cmd, tostring(r.status)), 'auth_invalid' end + if r.status ~= 200 then return nil, ('%s HTTP %s'):format(cmd, tostring(r.status)), 'http' end + local parsed, perr = json_decode(r.body) + if not parsed then + if auth_invalid_body(r.body) then return nil, cmd .. ' returned login page', 'auth_invalid' end + return nil, cmd .. ' invalid JSON: ' .. tostring(perr), 'parse' + end + if parsed.logout then return nil, cmd .. ' returned logout=' .. tostring(parsed.logout) .. ' reason=' .. tostring(parsed.reason), 'auth_invalid' end + return parsed.data or parsed, nil, nil +end + +local function read_commands(self, commands) + local data = {} + for _, cmd in ipairs(commands or {}) do + local d, err, code = get_cmd(self, cmd) + if not d then return nil, err or ('failed to read ' .. cmd), code end + data[cmd] = d + end + return data, nil, nil +end + +local function parse_speed_mbps(v) + if v == nil then return nil end + local n = tonumber(tostring(v):match('(%d+)')) + return n +end + +local function parse_media(v, panel) + local s = tostring(v or ''):lower() + if panel and panel.media == 'fiber' then return 'fiber' end + if s:find('fiber', 1, true) then return 'fiber' end + if s:find('copper', 1, true) then return 'copper' end + return nil +end + +local function parse_vlan_membership_string(s) + local out = {} + s = tostring(s or '') + for token in s:gmatch('%S+') do + local vlan, flags = token:match('^(%d+)([A-Z]+)$') + if vlan then + local rec = { vlan = tonumber(vlan), raw = token } + if flags:find('T', 1, true) then rec.tagged = true end + if flags:find('U', 1, true) then rec.untagged = true end + if flags:find('F', 1, true) then rec.forbidden = true end + if flags:find('P', 1, true) then rec.pvid = true end + out[#out + 1] = rec + end + end + return out +end + +local function vlans_from_membership(parsed) + local tagged, untagged, all = {}, {}, {} + for _, rec in ipairs(parsed or {}) do + if rec.vlan and rec.vlan ~= 4095 and not rec.forbidden then + all[#all + 1] = rec.vlan + if rec.tagged then tagged[#tagged + 1] = rec.vlan end + if rec.untagged then untagged[#untagged + 1] = rec.vlan end + end + end + return all, tagged, untagged +end + +local function is_lag_name(name) return tostring(name or ''):match('^LAG%d+$') ~= nil end + +local function percent(v) + local n = tonumber(v) + if n == nil then return nil end + return n +end + +local function parse_runtime(sys_cpumem) + sys_cpumem = sys_cpumem or {} + return { + cpu = { + utilisation_pct = percent(sys_cpumem.cpu), + }, + memory = { + utilisation_pct = percent(sys_cpumem.mem), + }, + } +end + +local function parse_power(poe) + poe = poe or {} + return { + poe = { + total_power_mw = poe.devPower, + total_power_w = type(poe.devPower) == 'number' and poe.devPower / 1000 or nil, + temperature_c = poe.devTemp, + }, + } +end + +local function parse_surface_counters(row) + if type(row) ~= 'table' then return nil end + local function sum_keys(keys) + local total, seen = 0, false + for _, key in ipairs(keys or {}) do + local n = tonumber(row[key]) + if n then total = total + n; seen = true end + end + return seen and total or nil + end + -- RMON ``oversize`` packets are contextual on VLAN trunks: valid + -- 802.1Q-tagged 1522-byte frames may be counted here by this switch even + -- when Etherlike/FCS/alignment counters remain clean. Keep them visible as + -- RMON detail, but do not fold them into the operator-facing error total. + local hard_errors = sum_keys({ 'CRCAlignErr', 'fragments', 'jabbers' }) + return { + rx = { + bytes = tonumber(row.bytesRec), + packets = tonumber(row.pktsRec), + drops = tonumber(row.dropEvents), + errors = hard_errors, + errors_hard = hard_errors, + broadcast_packets = tonumber(row.bPktsRec), + multicast_packets = tonumber(row.mPktsRec), + }, + rmon = { + crc_align_errors = tonumber(row.CRCAlignErr), + undersize_packets = tonumber(row.undersizePkts), + oversize_packets = tonumber(row.oversizePkts), + fragments = tonumber(row.fragments), + jabbers = tonumber(row.jabbers), + collisions = tonumber(row.collisions), + drop_events = tonumber(row.dropEvents), + }, + size_buckets = { + frames_64 = tonumber(row.frames64B), + frames_65_127 = tonumber(row.frames65127B), + frames_128_255 = tonumber(row.frames128255B), + frames_256_511 = tonumber(row.frames256511B), + frames_512_1023 = tonumber(row.frames5121023B), + frames_over_1024 = tonumber(row.framesOver1024B), + }, + } +end + +local function build_surfaces(data, opts) + opts = opts or { link = true, attachment = true, poe = true } + local home = data.home_main or {} + local ports = home.ports or {} + local panel_ports = (data.panel_info or {}).ports or {} + local port_rows = (data.port_port or {}).ports or {} + local vlan_conf = (data.vlan_conf or {}).ports or {} + local vlan_port = (data.vlan_port or {}).ports or {} + local vlan_membership = (data.vlan_membership or {}).ports or {} + local poe_ports = (data.poe_poe or {}).ports or {} + local surfaces = {} + + local function set_if_present(t, k, v) + if v ~= nil then t[k] = v end + end + + local function maybe_nonempty(t) + for _ in pairs(t or {}) do return t end + return nil + end + + for i, port in ipairs(ports) do + local name = tostring(port.port or port.name or ('port-' .. i)) + local panel = panel_ports[i] + local prow = port_rows[i] + local vconf = vlan_conf[i] + local vp = vlan_port[i] + local vm = vlan_membership[i] + local poe = poe_ports[i] + local surface = { + provider_surface_id = name, + kind = is_lag_name(name) and 'lag' or 'switch-port', + } + local capabilities, raw = {}, {} + + if opts.link then + local media = parse_media(prow and prow.type, panel) + local speed = parse_speed_mbps((panel and panel.speed) or (prow and prow.operSpeed)) + local duplex + if panel and panel.dupFull ~= nil then duplex = panel.dupFull and 'full' or 'half' + elseif prow and type(prow.operDuplex) == 'string' and prow.operDuplex:lower():find('full') then duplex = 'full' + elseif prow and type(prow.operDuplex) == 'string' and prow.operDuplex:lower():find('half') then duplex = 'half' end + + local link = {} + -- Absence of panel/port operational state means "not observed in this + -- command group", not "down". The merge layer preserves the prior link + -- state when this group has no link facts. + if panel and panel.linkup ~= nil then link.state = panel.linkup and 'up' or 'down' + elseif prow and prow.operStatus ~= nil then link.state = prow.operStatus and 'up' or 'down' end + set_if_present(link, 'speed_mbps', speed) + set_if_present(link, 'duplex', duplex) + if panel then set_if_present(link, 'auto_negotiation', panel.autoNego) end + set_if_present(link, 'media', media) + surface.link = maybe_nonempty(link) + set_if_present(raw, 'panel_info', panel) + set_if_present(raw, 'port_port', prow) + end + + if opts.attachment then + local membership = parse_vlan_membership_string((vm and (vm.operVlans or vm.adminVlans)) or '') + local vlans, tagged, untagged = vlans_from_membership(membership) + local mode = VLAN_MODE[(vp and vp.mode) or (vm and vm.mode) or (vconf and vconf.mode)] + local pvid = vp and vp.pvid + if pvid == nil and vconf and vconf.pvid then pvid = (data.vlan_conf and data.vlan_conf.vlan) or 1 end + if mode == 'trunk' or mode == 'hybrid' then capabilities.trunk = true end + if mode == 'access' or mode == 'hybrid' then capabilities.access = true end + local attachment = {} + set_if_present(attachment, 'mode', mode) + set_if_present(attachment, 'pvid', pvid) + if #vlans > 0 or vm ~= nil then attachment.vlans = vlans end + if #tagged > 0 or vm ~= nil then attachment.tagged_vlans = tagged end + if #untagged > 0 or vm ~= nil then attachment.untagged_vlans = untagged end + set_if_present(attachment, 'accept_frame_type', vp and VLAN_ACCEPT_FRAME[vp.accFrameType] or nil) + set_if_present(attachment, 'ingress_filter', vp and vp.ingressFilter or nil) + set_if_present(attachment, 'uplink', vp and vp.uplink or nil) + set_if_present(attachment, 'tpid', vp and vp.tpid or nil) + set_if_present(attachment, 'admin_vlans_raw', vm and vm.adminVlans or nil) + set_if_present(attachment, 'oper_vlans_raw', vm and vm.operVlans or nil) + if vm ~= nil then attachment.oper_vlans = membership end + set_if_present(attachment, 'vlan_membership', vconf and VLAN_MEMBERSHIP[vconf.membership] or nil) + set_if_present(attachment, 'forbidden', vconf and vconf.forbidden or nil) + surface.attachment = maybe_nonempty(attachment) + set_if_present(raw, 'vlan_conf', vconf) + set_if_present(raw, 'vlan_port', vp) + set_if_present(raw, 'vlan_membership', vm) + end + + if opts.poe and poe then + capabilities.poe = true + surface.poe = { + enabled = poe.portEnable == true, + state = poe.portStatus and 'delivering' or 'off', + delivering = poe.portStatus == true, + type = poe.portType, + level = poe.portLevel, + power_limit_mw = poe.portPowerLimit, + watchdog = poe.watchDog == true, + } + set_if_present(raw, 'poe_poe', poe) + end + + surface.capabilities = maybe_nonempty(capabilities) + surface.raw = maybe_nonempty(raw) + surfaces[name] = surface + end + + return surfaces +end + +local function build_surface_counters(data) + local home = data.home_main or {} + local ports = home.ports or {} + local rmon_ports = (data.rmon_statistics or {}).ports or {} + local out = {} + for i, port in ipairs(ports) do + local name = tostring(port.port or port.name or ('port-' .. i)) + local counters = parse_surface_counters(rmon_ports[i]) + if counters then out[name] = counters end + end + return out +end + +local function build_identity(data) + data = data or {} + local sys = data.sys_sysinfo or {} + local home = data.home_main or {} + return { + model = home.model or home.title, + hostname = sys.hostname, + mac = sys.sysMac, + firmware = sys.fwVer, + firmware_date = sys.fwDate, + loader = sys.loaderVer, + loader_date = sys.loaderDate, + serial = sys.syssn, + management_ipv4 = sys.currIpv4, + management_ipv6 = sys.currIpv6, + } +end + +local function base_status(self) + return { + state = 'available', + available = true, + mode = self.mode, + driver = DRIVER, + base_url = self.base_url, + login = self.logged_in and 'confirmed' or (self.disable_login and 'disabled' or 'attempted'), + } +end + +local function build_snapshot(self, data) + local poe = data.poe_poe or {} + return { + ok = true, + provider_id = self.id, + mode = self.mode, + writable = false, + status = base_status(self), + identity = build_identity(data), + surfaces = build_surfaces(data, { link = true, attachment = true, poe = true }), + counters = build_surface_counters(data), + topology = { + lldp_local = data.lldp_local, + lldp_neighbor = data.lldp_neighbor, + }, + runtime = parse_runtime(data.sys_cpumem), + power = parse_power(poe), + raw = self.include_raw and data or nil, + } +end + +local function build_group_observation(self, group, data) + group = tostring(group or '') + local out = { + ok = true, + provider_id = self.id, + group = group, + status = base_status(self), + } + + if group == 'identity' then + out.identity = build_identity(data) + elseif group == 'runtime' then + out.runtime = parse_runtime(data.sys_cpumem) + elseif group == 'counters' then + out.counters = build_surface_counters(data) + elseif group == 'poe' then + out.power = parse_power(data.poe_poe) + out.surfaces = build_surfaces(data, { poe = true }) + elseif group == 'lldp' then + out.topology = { + lldp_local = data.lldp_local, + lldp_neighbor = data.lldp_neighbor, + } + elseif group == 'panel' then + out.surfaces = build_surfaces(data, { link = true }) + elseif group == 'vlan' then + out.surfaces = build_surfaces(data, { attachment = true }) + else + return { ok = false, provider_id = self.id, group = group, status = { state = 'unavailable', available = false, driver = DRIVER, err = 'unknown command group: ' .. group } } + end + + if self.include_raw then out.raw = data end + return out +end + +local function build_groups_observation(self, groups, data) + local out = { + ok = true, + provider_id = self.id, + groups = copy(groups or {}), + status = base_status(self), + } + for _, group in ipairs(groups or {}) do + local partial = build_group_observation(self, group, data) + if not partial or partial.ok ~= true then return partial end + for _, key in ipairs({ 'identity', 'runtime', 'power', 'topology', 'counters' }) do + if type(partial[key]) == 'table' then out[key] = merge_table(out[key] or {}, partial[key]) end + end + if type(partial.surfaces) == 'table' then + out.surfaces = out.surfaces or {} + for surface_id, surface in pairs(partial.surfaces) do + out.surfaces[surface_id] = merge_table(out.surfaces[surface_id] or {}, surface) + end + end + end + if self.include_raw then out.raw = data end + return out +end + +local function require_http_config(config) + local http = config and config.http or nil + if type(http) ~= 'table' then return nil, 'http table is required' end + if type(http.capability) ~= 'string' or http.capability == '' then return nil, 'http.capability is required' end + if http.response_parser ~= 'legacy-http1-close' then return nil, 'http.response_parser must be legacy-http1-close' end + local max_response_bytes = tonumber(http.max_response_bytes) or (1024 * 1024) + if max_response_bytes <= 0 then return nil, 'http.max_response_bytes must be positive when supplied' end + return { capability = http.capability, response_parser = http.response_parser, max_response_bytes = max_response_bytes }, nil +end + + +local CONFIG_FIELDS = { + provider = true, + mode = true, + base_url = true, + username = true, + password = true, + timeout_s = true, + http = true, + openssl_bin = true, + disable_login = true, + include_raw = true, + cookies = true, +} + +local function check_allowed_config(config) + for k in pairs(config or {}) do + if not CONFIG_FIELDS[k] then return nil, 'unsupported rtl8380m_http config field: ' .. tostring(k) end + end + return true, nil +end + +local function configured_http_ref(http_config, opts) + if type(opts.http_client_for) ~= 'function' then return nil, 'http_client_for dependency is required' end + local ref, err = opts.http_client_for(http_config.capability) + if not ref then return nil, err end + return ref, nil +end + +function M.new(config, opts) + config = config or {} + opts = opts or {} + local allowed, allowed_err = check_allowed_config(config) + if not allowed then return nil, allowed_err end + if type(opts.provider_id) ~= 'string' or opts.provider_id == '' then return nil, 'opts.provider_id is required' end + local base_url, base_url_err = ensure_base_url(config.base_url) + if not base_url then return nil, base_url_err end + local username = getenv_ref(config.username) + local password = getenv_ref(config.password) + if type(username) ~= 'string' or username == '' then return nil, 'username is required' end + if type(password) ~= 'string' or password == '' then return nil, 'password is required' end + local http_config, http_config_err = require_http_config(config) + if not http_config then return nil, http_config_err end + local http_ref, http_ref_err = configured_http_ref(http_config, opts) + if not http_ref then return nil, http_ref_err end + local timeout_s = tonumber(config.timeout_s) + if not timeout_s or timeout_s <= 0 then return nil, 'timeout_s must be a positive number' end + return setmetatable({ + id = opts.provider_id, + base_url = base_url, + mode = config.mode or 'read_only', + username = username, + password = password, + timeout_s = timeout_s, + response_parser = http_config.response_parser, + max_response_bytes = http_config.max_response_bytes, + http_ref = http_ref, + openssl_bin = config.openssl_bin or os.getenv('SWITCH_OPENSSL') or 'openssl', + disable_login = config.disable_login == true, + include_raw = config.include_raw == true, + logger = opts.logger, + jar = CookieJar.new(config.cookies or { cookie_language = 'defLang_en' }), + logged_in = false, + }, Provider), nil +end + +function Provider:fetch_snapshot() + if self.client and type(self.client.snapshot) == 'function' then + local data, err = self.client:snapshot(self) + if not data then return { ok = false, provider_id = self.id, status = { state = 'unavailable', available = false, driver = DRIVER, err = err } } end + return build_snapshot(self, data) + end + + local ok_login, lerr = login(self) + if not ok_login then + return { ok = false, provider_id = self.id, status = { state = 'unavailable', available = false, driver = DRIVER, login = 'failed', err = lerr or 'login failed' } } + end + + local data, err, code = read_commands(self, READ_COMMANDS) + if data then return build_snapshot(self, data) end + + if code == 'auth_invalid' then + reset_session(self) + local ok_relogin, relogin_err = login(self, { force = true }) + if not ok_relogin then + return { ok = false, provider_id = self.id, status = { state = 'unavailable', available = false, driver = DRIVER, login = 'failed', err = relogin_err or 're-login failed' } } + end + data, err, code = read_commands(self, READ_COMMANDS) + if data then return build_snapshot(self, data) end + end + + return { ok = false, provider_id = self.id, status = { state = 'unavailable', available = false, driver = DRIVER, login = self.logged_in and 'confirmed' or 'failed', err = err or 'switch snapshot failed' } } +end + +function Provider:fetch_command_group(group_name) + local group = tostring(group_name or '') + local commands = COMMAND_GROUPS[group] + if not commands then + return { + ok = false, + provider_id = self.id, + status = { state = 'unavailable', available = false, driver = DRIVER, err = 'unknown command group: ' .. group }, + } + end + + local ok_login, lerr = login(self) + if not ok_login then + return { ok = false, provider_id = self.id, group = group, commands = commands, status = { state = 'unavailable', available = false, driver = DRIVER, login = 'failed', err = lerr or 'login failed' } } + end + + local data, err, code = read_commands(self, commands) + if code == 'auth_invalid' then + reset_session(self) + local ok_relogin, relogin_err = login(self, { force = true }) + if not ok_relogin then + return { ok = false, provider_id = self.id, group = group, commands = commands, status = { state = 'unavailable', available = false, driver = DRIVER, login = 'failed', err = relogin_err or 're-login failed' } } + end + data, err, code = read_commands(self, commands) + end + + if not data then + return { ok = false, provider_id = self.id, group = group, commands = commands, status = { state = 'unavailable', available = false, driver = DRIVER, login = self.logged_in and 'confirmed' or 'failed', err = err or ('switch command group failed: ' .. group) } } + end + + return { + ok = true, + provider_id = self.id, + group = group, + commands = commands, + status = { state = 'available', available = true, driver = DRIVER, login = self.logged_in and 'confirmed' or 'disabled' }, + raw = data, + } +end + +function Provider:fetch_command_groups(groups) + groups = groups or {} + local commands, cerr = commands_for_groups(groups) + if not commands then + return { ok = false, provider_id = self.id, groups = copy(groups), status = { state = 'unavailable', available = false, driver = DRIVER, err = cerr } } + end + + local ok_login, lerr = login(self) + if not ok_login then + return { ok = false, provider_id = self.id, groups = copy(groups), commands = commands, status = { state = 'unavailable', available = false, driver = DRIVER, login = 'failed', err = lerr or 'login failed' } } + end + + local data, err, code = read_commands(self, commands) + if code == 'auth_invalid' then + reset_session(self) + local ok_relogin, relogin_err = login(self, { force = true }) + if not ok_relogin then + return { ok = false, provider_id = self.id, groups = copy(groups), commands = commands, status = { state = 'unavailable', available = false, driver = DRIVER, login = 'failed', err = relogin_err or 're-login failed' } } + end + data, err, code = read_commands(self, commands) + end + + if not data then + return { ok = false, provider_id = self.id, groups = copy(groups), commands = commands, status = { state = 'unavailable', available = false, driver = DRIVER, login = self.logged_in and 'confirmed' or 'failed', err = err or 'switch command groups failed' } } + end + + local out = build_groups_observation(self, groups, data) + if out and out.ok == true then out.commands = commands end + return out +end + +function Provider:fetch_snapshot_op(_req) + return op.guard(function () + return fibers.run_scope_op(function () + return self:fetch_snapshot() + end):wrap(function (status, _report, result_or_primary, err) + if status == 'ok' then return result_or_primary, err end + return { + ok = false, + provider_id = self.id, + status = { + state = 'unavailable', + available = false, + driver = DRIVER, + err = tostring(result_or_primary or status or 'snapshot failed'), + }, + }, nil + end) + end) +end + +function Provider:snapshot_op(req) return self:fetch_snapshot_op(req) end +function Provider:watch_op(req) return self:fetch_snapshot_op(req) end + +function Provider:observe_groups_op(req) + req = req or {} + local groups = req.groups or {} + if type(groups) ~= 'table' then + return op.always({ ok = false, provider_id = self.id, status = { state = 'unavailable', available = false, driver = DRIVER, err = 'groups must be an array' } }) + end + return op.guard(function () + return fibers.run_scope_op(function () + return self:fetch_command_groups(groups) + end):wrap(function (status, _report, result_or_primary, err) + if status == 'ok' then return result_or_primary, err end + return { + ok = false, + provider_id = self.id, + groups = copy(groups), + status = { + state = 'unavailable', + available = false, + driver = DRIVER, + err = tostring(result_or_primary or status or 'group observation failed'), + }, + }, nil + end) + end) +end + +function Provider:apply_attachments_op(_req) return op.always(contract.read_only('apply_attachments')) end +function Provider:set_poe_op(_req) return op.always(contract.read_only('set_poe')) end +function Provider:bounce_op(_req) return op.always(contract.read_only('bounce')) end +function Provider:terminate(_reason) reset_session(self); return true end + +M._test = { + VLAN_MODE = VLAN_MODE, + VLAN_ACCEPT_FRAME = VLAN_ACCEPT_FRAME, + VLAN_MEMBERSHIP = VLAN_MEMBERSHIP, + parse_vlan_membership_string = parse_vlan_membership_string, + build_snapshot = function(provider_like, data) return build_snapshot(provider_like or { id = 'switch-main', mode = 'read_only' }, data or {}) end, + build_surfaces = build_surfaces, + parse_runtime = parse_runtime, + parse_power = parse_power, + parse_surface_counters = parse_surface_counters, + build_group_observation = function(provider_like, group, data) return build_group_observation(provider_like or { id = 'switch-main', mode = 'read_only' }, group, data or {}) end, + build_groups_observation = function(provider_like, groups, data) return build_groups_observation(provider_like or { id = 'switch-main', mode = 'read_only' }, groups or {}, data or {}) end, + commands_for_groups = commands_for_groups, + auth_invalid_body = auth_invalid_body, + COMMAND_GROUPS = COMMAND_GROUPS, +} + +return M diff --git a/src/services/hal/backends/wired/providers/static.lua b/src/services/hal/backends/wired/providers/static.lua new file mode 100644 index 00000000..975e1682 --- /dev/null +++ b/src/services/hal/backends/wired/providers/static.lua @@ -0,0 +1,77 @@ +-- services/hal/backends/wired/providers/static.lua +-- Static/read-only wired provider, useful for CM5 eth0 and tests. + +local op = require 'fibers.op' +local contract = require 'services.hal.backends.wired.contract' +local tablex = require 'shared.table' + +local M = {} +local Provider = {} +Provider.__index = Provider + +local function copy(v) return tablex.deep_copy(v) end + +local CONFIG_FIELDS = { + provider = true, + mode = true, + surfaces = true, + topology = true, + meta = true, +} + +local function check_allowed_config(config) + for k in pairs(config or {}) do + if not CONFIG_FIELDS[k] then return nil, 'unsupported static wired config field: ' .. tostring(k) end + end + return true, nil +end + +function M.new(config, opts) + config = config or {} + opts = opts or {} + local allowed, allowed_err = check_allowed_config(config) + if not allowed then return nil, allowed_err end + if type(opts.provider_id) ~= 'string' or opts.provider_id == '' then return nil, 'opts.provider_id is required' end + if type(config.surfaces) ~= 'table' then return nil, 'static wired provider requires surfaces' end + return setmetatable({ + id = opts.provider_id, + mode = config.mode or 'read_only', + surfaces = copy(config.surfaces), + topology = copy(config.topology or {}), + meta = copy(config.meta or {}), + }, Provider), nil +end + +function Provider:snapshot_op(_req) + return op.always({ + ok = true, + provider_id = self.id, + mode = self.mode, + writable = false, + status = { state = 'available', available = true, mode = self.mode }, + surfaces = copy(self.surfaces), + topology = copy(self.topology), + meta = copy(self.meta), + }) +end + +function Provider:watch_op(req) return self:snapshot_op(req) end + +function Provider:observe_groups_op(req) + req = req or {} + local groups = req.groups or {} + if type(groups) ~= 'table' then return op.always({ ok = false, provider_id = self.id, status = { state = 'unavailable', available = false, err = 'groups must be an array' } }) end + for i = 1, #groups do + if groups[i] ~= 'snapshot' then + return op.always({ ok = false, provider_id = self.id, group = groups[i], status = { state = 'unavailable', available = false, err = 'unsupported static poll group: ' .. tostring(groups[i]) } }) + end + end + return self:snapshot_op(req) +end + +function Provider:apply_attachments_op(_req) return op.always(contract.read_only('apply_attachments')) end +function Provider:set_poe_op(_req) return op.always(contract.read_only('set_poe')) end +function Provider:bounce_op(_req) return op.always(contract.read_only('bounce')) end +function Provider:terminate(_reason) return true end + +return M diff --git a/src/services/hal/dependencies.lua b/src/services/hal/dependencies.lua new file mode 100644 index 00000000..59ecc5c3 --- /dev/null +++ b/src/services/hal/dependencies.lua @@ -0,0 +1,37 @@ +-- services/hal/dependencies.lua +-- Declared cross-service dependency ports for the HAL service root. + +local capdeps = require 'services.support.capdeps' +local http_sdk = require 'services.http.sdk' + +local M = {} + +function M.declarations() + local http_client = assert(capdeps.capability({ + sdk = http_sdk, + default_cap_id = 'main', + methods = { + 'status_op', + 'open_exchange_op', + 'exchange_op', + }, + })) + return { + http_client = http_client, + } +end + +function M.resolver(conn) + return capdeps.new(conn, M.declarations()) +end + +function M.manager_options(manager_name, resolver, base) + local out = {} + for k, v in pairs(base or {}) do out[k] = v end + if manager_name == 'wired' and resolver then + out.http_client_for = resolver:factory('http_client') + end + return out +end + +return M diff --git a/src/services/hal/drivers/artifact_store.lua b/src/services/hal/drivers/artifact_store.lua new file mode 100644 index 00000000..6e33b2e1 --- /dev/null +++ b/src/services/hal/drivers/artifact_store.lua @@ -0,0 +1,1105 @@ +---@module 'services.hal.drivers.artifact_store' + +local fibers = require 'fibers' +local op = require 'fibers.op' +local file = require 'fibers.io.file' +local exec = require 'fibers.io.exec' +local channel = require 'fibers.channel' +local pulse_mod = require 'fibers.pulse' +local cjson = require 'cjson.safe' +local uuid = require 'uuid' + +local checksum = require 'shared.hash.xxhash32' +local blob_source = require 'devicecode.blob_source' +local resource = require 'devicecode.support.resource' +local tablex = require 'shared.table' + +local M = {} + +---@class ArtifactStore +---@field transient_root string +---@field durable_root string +---@field durable_enabled boolean +---@field import_root string +---@field logger table|nil +local Store = {} +Store.__index = Store + +---@class ArtifactHandle +---@field _store ArtifactStore +---@field _rec table +---@field _blob_path string +local Artifact = {} +Artifact.__index = Artifact + +---@class ArtifactSink +---@field _cmd_ch Channel +---@field _closed Pulse +---@field _artifact_ref string +---@field _phase string +---@field _terminal boolean +---@field _artifact ArtifactHandle|nil +---@field _last_error string|nil +local ArtifactSink = {} +ArtifactSink.__index = ArtifactSink + +---------------------------------------------------------------------- +-- Plain helpers +---------------------------------------------------------------------- + + +local function join_path(...) + return table.concat({ ... }, '/') +end + +local function dirname(path) + if path == '/' then + return '/' + end + + local d = tostring(path or ''):match('^(.*)/[^/]+$') + if d == nil or d == '' then + return '.' + end + return d +end + +local deep_copy = tablex.deep_copy + +local function valid_ref(ref) + return type(ref) == 'string' + and ref ~= '' + and ref:match('^[A-Za-z0-9_.-]+$') ~= nil +end + +local function is_not_found_err(err) + if err == nil then + return false + end + local s = tostring(err):lower() + return s:find('no such file', 1, true) ~= nil + or s:find('not found', 1, true) ~= nil + or s:find('enoent', 1, true) ~= nil +end + +local function unwrap_attempt(st, rep, ok, value_or_err) + if st ~= 'ok' then + return false, tostring(value_or_err or rep) + end + return ok, value_or_err +end + +local function attempt_op(fn) + return fibers.run_scope_op(fn):wrap(unwrap_attempt) +end + +---------------------------------------------------------------------- +-- Durability and paths +---------------------------------------------------------------------- + +local function choose_durability(self, policy) + policy = policy or 'transient_only' + + if policy == 'require_durable' then + if not self.durable_enabled then + return nil, 'durable_disabled' + end + return 'durable' + end + + if policy == 'prefer_durable' then + if self.durable_enabled then + return 'durable' + end + return 'transient' + end + + if policy == 'transient_only' then + return 'transient' + end + + return nil, 'invalid_policy' +end + +local function root_for(self, durability) + if durability == 'durable' then + return self.durable_root + end + return self.transient_root +end + +local function staging_root(self, durability) + return join_path(root_for(self, durability), '.staging') +end + +local function staging_dir(self, ref, durability) + return join_path(staging_root(self, durability), ref) +end + +local function object_dir(self, ref, durability) + return join_path(root_for(self, durability), ref) +end + +local function data_path(dir) + return join_path(dir, 'blob.bin') +end + +local function final_meta_path(dir) + return join_path(dir, 'meta.json') +end + +local function partial_meta_path(dir) + return join_path(dir, 'meta.json.partial') +end + +---------------------------------------------------------------------- +-- Filesystem / process-backed ops +---------------------------------------------------------------------- + +-- Keep recursive removal boxed here until the file layer grows an rmdir/remove-tree +-- primitive. This is now used only where directory removal is genuinely required. +local function rm_rf_op(path) + return (exec.command { 'rm', '-rf', path, stdin = 'null', stdout = 'null', stderr = 'null' }):run_op():wrap(function (st, code, _sig, err) + if st == 'exited' and code == 0 then + return true, nil + end + if st == 'signalled' then + return false, 'rm_signalled' + end + return false, tostring(err or ('rm_failed:' .. tostring(code))) + end) +end + +local function mkdir_p_op(path, perms) + return attempt_op(function () + local ok, err = file.mkdir_p(path, perms) + if not ok then + return false, tostring(err) + end + return true, nil + end) +end + +local function rename_path_op(src, dst) + return attempt_op(function () + local ok, err = file.rename(src, dst) + if not ok then + return false, tostring(err) + end + return true, nil + end) +end + +local function unlink_path_op(path, tolerate_missing) + return attempt_op(function () + local ok, err = file.unlink(path) + if ok then + return true, nil + end + if tolerate_missing and is_not_found_err(err) then + return true, nil + end + return false, tostring(err) + end) +end + +local function unlink_known_artifact_files_op(dir) + return attempt_op(function () + local ok, err + + ok, err = fibers.perform(unlink_path_op(data_path(dir), true)) + if not ok then + return false, err + end + + ok, err = fibers.perform(unlink_path_op(partial_meta_path(dir), true)) + if not ok then + return false, err + end + + ok, err = fibers.perform(unlink_path_op(final_meta_path(dir), true)) + if not ok then + return false, err + end + + return true, nil + end) +end + +local function remove_artifact_dir_op(dir) + return attempt_op(function () + local ok, err = fibers.perform(unlink_known_artifact_files_op(dir)) + if not ok then + return false, err + end + + local ok_rm, rm_err = fibers.perform(rm_rf_op(dir)) + if not ok_rm then + return false, rm_err + end + + return true, nil + end) +end + +local function terminate_stream(stream, reason) + if stream == nil then + return true, nil + end + + return resource.terminate(stream, reason or 'artifact stream terminated') +end + +local function open_stream_now(path, mode) + local stream, err = file.open(path, mode) + if not stream then + return nil, tostring(err) + end + return stream, nil +end + +local function read_text_op(path) + return attempt_op(function (scope) + local stream, err = open_stream_now(path, 'r') + if not stream then + return false, err + end + + scope:finally(function (_, status, primary) + resource.terminate_checked(stream, primary or status or 'artifact stream closed', 'artifact stream cleanup failed') + end) + + local body, rerr = fibers.perform(stream:read_all_op()) + if rerr ~= nil then + return false, tostring(rerr) + end + + return true, body or '' + end) +end + +local function write_atomic_text_op(path, data) + return attempt_op(function (scope) + local dir = dirname(path) + local ok_dir, dir_err = fibers.perform(mkdir_p_op(dir)) + if not ok_dir then + return false, dir_err + end + + local stream, err = file.tmpfile(384, dir) + if not stream then + return false, tostring(err) + end + + scope:finally(function (_, status, primary) + resource.terminate_checked(stream, primary or status or 'artifact stream closed', 'artifact stream cleanup failed') + end) + + local n, werr = fibers.perform(stream:write_op(data)) + if n == nil then + return false, tostring(werr or 'write_failed') + end + + local okf, ferr = fibers.perform(stream:flush_op()) + if okf == nil then + return false, tostring(ferr or 'flush_failed') + end + + local okr, rerr = stream:rename(path) + if not okr then + return false, tostring(rerr or 'rename_failed') + end + + return true, nil + end) +end + +local function read_json_op(path) + return read_text_op(path):wrap(function (ok, body_or_err) + if not ok then + return false, body_or_err + end + + local obj, derr = cjson.decode(body_or_err) + if obj == nil then + return false, tostring(derr or 'decode_failed') + end + + return true, obj + end) +end + +local function write_json_op(path, obj) + local raw, err = cjson.encode(obj) + if not raw then + return op.always(false, tostring(err or 'encode_failed')) + end + return write_atomic_text_op(path, raw) +end + +---------------------------------------------------------------------- +-- Metadata helpers +---------------------------------------------------------------------- + +local function new_ready_record(ref, durability, size, checksum_hex, meta, created_at) + local now = os.time() + return { + version = 2, + artifact_ref = ref, + state = 'ready', + durability = durability, + size = size, + checksum = checksum_hex, + created_at = created_at or now, + updated_at = now, + meta = deep_copy(meta or {}), + } +end + + +local function locate_artifact_op(self, ref) + return attempt_op(function () + for _, durability in ipairs({ 'transient', 'durable' }) do + local dir = object_dir(self, ref, durability) + local mp = final_meta_path(dir) + local ok, rec_or_err = fibers.perform(read_json_op(mp)) + if ok then + return true, { + rec = rec_or_err, + durability = durability, + dir = dir, + meta_path = mp, + blob_path = data_path(dir), + } + end + if not is_not_found_err(rec_or_err) then + return false, rec_or_err + end + end + return false, 'not_found' + end) +end + +---------------------------------------------------------------------- +-- Artifact ready handle +---------------------------------------------------------------------- + +function Artifact:ref() + return self._rec.artifact_ref +end + +function Artifact:describe() + return deep_copy(self._rec) +end + +function Artifact:local_path() + return self._blob_path +end + +function Artifact:open_source_op() + return attempt_op(function () + local stream, err = open_stream_now(self._blob_path, 'r') + if not stream then + return false, err + end + return true, blob_source.from_stream(stream) + end) +end + +---------------------------------------------------------------------- +-- Sink session internals +---------------------------------------------------------------------- + +local function sink_terminal_err(phase) + if phase == 'committed' then + return 'committed' + end + if phase == 'aborted' then + return 'aborted' + end + if phase == 'closed' then + return 'closed' + end + if phase == 'sealed' then + return 'sealed' + end + return 'closed' +end + +local function make_sink_status_snapshot(session) + return { + artifact_ref = session.artifact_ref, + state = session.phase, + terminal = session.terminal == true, + bytes = session.bytes, + checksum = checksum.digest_hex_state(session.checksum_state), + last_error = session.last_error, + } +end + +local function sink_reply(ch, ok, value, err, terminal, phase) + return ch:put_op({ + ok = ok, + value = value, + err = err, + terminal = terminal and true or false, + phase = phase, + }) +end + +local function request_reply_op(ch, req, closed) + return attempt_op(function () + local reply_ch = channel.new(1) + req.reply_ch = reply_ch + + local which, _, close_reason = fibers.perform(fibers.named_choice{ + sent = ch:put_op(req), + closed = closed:next_op(), + }) + + if which == 'closed' then + return false, tostring(close_reason or 'session_closed') + end + + local which2, a, b = fibers.perform(fibers.named_choice{ + reply = reply_ch:get_op(), + closed = closed:next_op(), + }) + + if which2 == 'closed' then + return false, tostring(b or 'session_closed') + end + + local reply = a + if not reply then + return false, tostring(b or 'reply_missing') + end + + if reply.ok then + return true, reply.value + end + return false, tostring(reply.err or 'request_failed') + end) +end + +local function commit_sealed_op(session) + return attempt_op(function () + local object_parent = root_for(session.store, session.durability) + local object_path = object_dir(session.store, session.artifact_ref, session.durability) + local meta = new_ready_record( + session.artifact_ref, + session.durability, + session.bytes, + checksum.digest_hex_state(session.checksum_state), + session.meta, + session.created_at + ) + + local ok_parent, perr = fibers.perform(mkdir_p_op(object_parent)) + if not ok_parent then + return false, perr + end + + local ok_meta, merr = fibers.perform(write_json_op(final_meta_path(session.staging_dir), meta)) + if not ok_meta then + return false, merr + end + + local ok_mv, mverr = fibers.perform(rename_path_op(session.staging_dir, object_path)) + if not ok_mv then + return false, mverr + end + + session.artifact = setmetatable({ + _store = session.store, + _rec = meta, + _blob_path = data_path(object_path), + }, Artifact) + session.phase = 'committed' + session.terminal = true + return true, session.artifact + end) +end + +local function session_append_op(session, chunk) + if session.phase ~= 'open' then + return op.always(false, sink_terminal_err(session.phase)) + end + if type(chunk) ~= 'string' then + return op.always(false, 'chunk must be a string') + end + if session.stream == nil then + session.phase = 'aborted' + session.terminal = true + return op.always(false, 'closed') + end + + return session.stream:write_op(chunk):wrap(function (n, err) + if n == nil then + session.phase = 'aborted' + session.terminal = true + session.last_error = tostring(err or 'write_failed') + return false, session.last_error + end + checksum.update(session.checksum_state, chunk) + session.bytes = session.bytes + #chunk + return true, nil + end) +end + +local function session_abort_op(session, reason) + if session.phase == 'committed' then + return op.always(false, 'committed') + end + if session.phase == 'aborted' or session.phase == 'closed' then + session.terminal = true + return op.always(true, nil) + end + + return attempt_op(function () + if session.stream ~= nil then + local ok_term, term_err = terminate_stream(session.stream, reason or 'artifact sink aborted') + session.stream = nil + if ok_term ~= true then + return false, term_err or 'artifact stream termination failed' + end + end + + local ok_rm, rm_err = fibers.perform(remove_artifact_dir_op(session.staging_dir)) + if not ok_rm then + return false, rm_err + end + + session.last_error = reason and tostring(reason) or session.last_error + session.phase = 'aborted' + session.terminal = true + return true, nil + end) +end + +local function session_commit_op(session) + if session.phase == 'committed' then + return op.always(true, session.artifact) + end + if session.phase == 'aborted' or session.phase == 'closed' then + return op.always(false, sink_terminal_err(session.phase)) + end + + return fibers.run_scope_op(function (scope) + local committed = false + scope:finally(function (_, status, primary) + if session.stream ~= nil then + resource.terminate_checked(session.stream, primary or status or 'artifact sink session closed', 'artifact sink session stream cleanup failed') + session.stream = nil + end + if not committed and session.phase ~= 'committed' and session.phase ~= 'aborted' and session.phase ~= 'closed' then + session.phase = 'sealed' + if status ~= 'ok' then + session.last_error = session.last_error or tostring(primary or status) + end + end + end) + + if session.phase == 'open' or (session.phase == 'sealed' and session.stream ~= nil) then + local okf, ferr = fibers.perform(session.stream:flush_op()) + if okf == nil then + session.last_error = tostring(ferr or 'flush_failed') + session.phase = 'sealed' + return false, session.last_error + end + + local okc, cerr = fibers.perform(session.stream:close_op()) + if okc == nil then + session.last_error = tostring(cerr or 'close_failed') + session.phase = 'sealed' + return false, session.last_error + end + + session.stream = nil + session.phase = 'sealed' + end + + local ok_commit, art_or_err = fibers.perform(commit_sealed_op(session)) + if not ok_commit then + session.last_error = tostring(art_or_err) + return false, session.last_error + end + + committed = true + return true, art_or_err + end):wrap(unwrap_attempt) +end + +local function handle_session_request_op(session, req) + return op.guard(function () + if req.verb == 'append' then + return session_append_op(session, req.chunk) + elseif req.verb == 'commit' then + return session_commit_op(session) + elseif req.verb == 'abort' then + return session_abort_op(session, req.reason) + elseif req.verb == 'close' then + if session.phase == 'committed' then + session.terminal = true + return op.always(true, nil) + end + return session_abort_op(session, req.reason or 'closed without commit') + elseif req.verb == 'status' then + return op.always(true, make_sink_status_snapshot(session)) + end + return op.always(false, 'unsupported_sink_verb:' .. tostring(req.verb)) + end) +end + +local function sink_session_main(session) + local scope = assert(session.scope, 'artifact sink session missing scope') + + scope:finally(function (_, status, primary) + if session.stream ~= nil then + resource.terminate_checked(session.stream, primary or status or 'artifact sink closed', 'artifact sink stream cleanup failed') + session.stream = nil + end + -- Staging directory removal may require process/filesystem work and must + -- not be performed from a finaliser. Orderly abort/close paths remove it; + -- cancellation may leave a staging directory for later scavenging. + if session.phase ~= 'committed' then + session.phase = 'aborted' + session.terminal = true + session.last_error = session.last_error or ((status ~= 'ok') and tostring(primary or status) or 'abandoned_staging_dir') + end + session.closed:close(session.phase) + end) + + while true do + local req = fibers.perform(session.cmd_ch:get_op()) + if not req then + if session.phase ~= 'committed' then + session.phase = 'aborted' + session.terminal = true + end + return + end + + local ok, value_or_err = fibers.perform(handle_session_request_op(session, req)) + local terminal = session.terminal == true + local err = ok and nil or value_or_err + local value = ok and value_or_err or nil + local _ = fibers.perform(sink_reply(req.reply_ch, ok, value, err, terminal, session.phase)) + + if terminal then + return + end + end +end + +---------------------------------------------------------------------- +-- Artifact sink client handle +---------------------------------------------------------------------- + +local function sink_status_snapshot_from_handle(self) + return { + artifact_ref = self._artifact_ref, + state = self._phase, + terminal = self._terminal, + last_error = self._last_error, + } +end + +local function sink_request_op(self, verb, payload) + return op.guard(function () + if self._terminal then + if verb == 'status' then + return op.always(true, sink_status_snapshot_from_handle(self)) + end + if verb == 'commit' then + if self._phase == 'committed' then + return op.always(true, self._artifact) + end + return op.always(false, sink_terminal_err(self._phase)) + end + if verb == 'close' then + return op.always(true, nil) + end + if verb == 'abort' then + if self._phase == 'aborted' or self._phase == 'closed' then + return op.always(true, nil) + end + return op.always(false, sink_terminal_err(self._phase)) + end + return op.always(false, sink_terminal_err(self._phase)) + end + + return request_reply_op(self._cmd_ch, payload, self._closed):wrap(function (ok, value_or_err) + if ok then + if verb == 'commit' and type(value_or_err) == 'table' and getmetatable(value_or_err) == Artifact then + self._artifact = value_or_err + self._phase = 'committed' + self._terminal = true + elseif verb == 'close' or verb == 'abort' then + self._phase = 'aborted' + self._terminal = true + elseif verb == 'status' and type(value_or_err) == 'table' then + self._phase = value_or_err.state or self._phase + self._last_error = value_or_err.last_error or self._last_error + end + return true, value_or_err + end + + if verb == 'commit' then + self._phase = 'sealed' + self._last_error = value_or_err + elseif verb == 'abort' or verb == 'close' then + self._phase = 'aborted' + self._terminal = true + self._last_error = value_or_err + end + return false, value_or_err + end) + end) +end + +function ArtifactSink:append_op(chunk) + return sink_request_op(self, 'append', { + verb = 'append', + chunk = chunk, + }) +end + +function ArtifactSink:write_chunk_op(chunk) + return self:append_op(chunk) +end + +function ArtifactSink:commit_op(opts) + return sink_request_op(self, 'commit', { + verb = 'commit', + opts = opts, + }):wrap(function (ok, value_or_err) + if ok == true then return value_or_err, nil end + return nil, value_or_err or 'artifact_sink_commit_failed' + end) +end + +function ArtifactSink:abort_op(reason) + return sink_request_op(self, 'abort', { + verb = 'abort', + reason = reason, + }) +end + +function ArtifactSink:close_op() + return sink_request_op(self, 'close', { + verb = 'close', + }) +end + +function ArtifactSink:terminate(reason) + if self._terminal == true then + return true, nil + end + self._terminal = true + if self._cmd_ch and type(self._cmd_ch.close) == 'function' then + self._cmd_ch:close(reason or 'terminated') + end + return true, nil +end + +function ArtifactSink:status_op() + return sink_request_op(self, 'status', { + verb = 'status', + }) +end + +function ArtifactSink:status() + return sink_status_snapshot_from_handle(self) +end + +---------------------------------------------------------------------- +-- Store API +---------------------------------------------------------------------- + +function Store:status_op() + return op.always(true, { + kind = 'artifact-store', + transient_root = self.transient_root, + durable_root = self.durable_root, + durable_enabled = self.durable_enabled, + import_root = self.import_root, + }) +end + +function Store:create_sink_op(meta, opts) + opts = opts or {} + return op.guard(function () + local owner_scope = fibers.current_scope() + if owner_scope == nil then + return op.always(false, 'create_sink_op must be called from inside a fiber') + end + + local durability, derr = choose_durability(self, opts.policy) + if not durability then + return op.always(false, derr) + end + + return attempt_op(function (scope) + local ref = tostring(uuid.new()) + local sroot = staging_root(self, durability) + local sdir = staging_dir(self, ref, durability) + local dpath = data_path(sdir) + local pmeta = partial_meta_path(sdir) + local created_at = os.time() + local handed_off = false + local stream = nil + local session_scope = nil + local serr = nil + + scope:finally(function (_, status, primary) + if not handed_off then + if stream ~= nil then + resource.terminate_checked(stream, primary or status or 'sink admission failed', 'artifact sink admission stream cleanup failed') + end + if session_scope ~= nil then + session_scope:cancel(tostring(primary or status or 'sink_admission_failed')) + end + -- Do not remove directories from a finaliser; this may need a subprocess. + -- The abandoned staging directory is safe to scavenge later. + end + end) + + local ok_root, root_err = fibers.perform(mkdir_p_op(sroot)) + if not ok_root then + return false, root_err + end + + local ok_dir, dir_err = fibers.perform(mkdir_p_op(sdir)) + if not ok_dir then + return false, dir_err + end + + stream, dir_err = open_stream_now(dpath, 'w') + if not stream then + return false, dir_err + end + + session_scope, serr = owner_scope:child() + if not session_scope then + return false, tostring(serr) + end + + local session = { + store = self, + scope = session_scope, + cmd_ch = channel.new(8), + closed = pulse_mod.new(), + artifact_ref = ref, + durability = durability, + staging_dir = sdir, + data_path = dpath, + partial_meta_path = pmeta, + stream = stream, + phase = 'open', + terminal = false, + bytes = 0, + checksum_state = checksum.new(), + meta = deep_copy(meta or {}), + created_at = created_at, + artifact = nil, + last_error = nil, + } + + local ok_spawn, spawn_err = session_scope:spawn(function () + return sink_session_main(session) + end) + if not ok_spawn then + session_scope:cancel(tostring(spawn_err or 'sink_session_spawn_failed')) + return false, tostring(spawn_err) + end + + handed_off = true + stream = nil + + return true, setmetatable({ + _cmd_ch = session.cmd_ch, + _closed = session.closed, + _artifact_ref = ref, + _phase = 'open', + _terminal = false, + _artifact = nil, + _last_error = nil, + }, ArtifactSink) + end) + end) +end + +function Store:open_op(ref) + return op.guard(function () + if not valid_ref(ref) then + return op.always(false, 'invalid_artifact_ref') + end + return locate_artifact_op(self, ref):wrap(function (ok, located_or_err) + if not ok then + return false, located_or_err + end + local rec = located_or_err.rec + if rec.state ~= 'ready' then + return false, 'artifact_not_ready' + end + return true, setmetatable({ + _store = self, + _rec = rec, + _blob_path = located_or_err.blob_path, + }, Artifact) + end) + end) +end + +function Store:resolve_local_op(ref) + return op.guard(function () + if not valid_ref(ref) then + return op.always(false, 'invalid_artifact_ref') + end + return locate_artifact_op(self, ref):wrap(function (ok, located_or_err) + if not ok then + return false, located_or_err + end + local rec = located_or_err.rec + if rec.state ~= 'ready' then + return false, 'artifact_not_ready' + end + return true, { + artifact_ref = rec.artifact_ref, + durability = rec.durability, + path = located_or_err.blob_path, + meta = deep_copy(rec.meta), + size = rec.size, + checksum = rec.checksum, + state = rec.state, + } + end) + end) +end + +local function import_source_attempt_op(self, source, meta, opts) + opts = opts or {} + return attempt_op(function (scope) + local ok_sink, sink_or_err = fibers.perform(self:create_sink_op(meta, opts)) + if not ok_sink then + return false, sink_or_err + end + local sink = sink_or_err + + scope:finally(function (_, status, primary) + local reason = primary or status or 'artifact import closed' + resource.terminate_checked(source, reason, 'artifact import source cleanup failed') + resource.terminate_checked(sink, reason, 'artifact import sink cleanup failed') + end) + + local st, rep, bytes_or_primary = fibers.perform(blob_source.copy_op(source, sink, { + close_source = false, + close_sink = false, + })) + if st ~= 'ok' then + return false, tostring(bytes_or_primary or rep) + end + + local artifact, commit_err = fibers.perform(sink:commit_op()) + if artifact == nil then + return false, tostring(commit_err or 'artifact_sink_commit_failed') + end + + return true, artifact + end) +end + +function Store:import_source_op(source, meta, opts) + return op.guard(function () + if type(source) ~= 'table' or type(source.read_chunk_op) ~= 'function' then + return op.always(false, 'invalid_source') + end + return import_source_attempt_op(self, source, meta, opts) + end) +end + +function Store:import_path_op(path, meta, opts) + return op.guard(function () + if type(path) ~= 'string' or path == '' then + return op.always(false, 'invalid_path') + end + + local resolved = path + if resolved:sub(1, 1) ~= '/' then + if resolved:find('..', 1, true) or resolved:find('\\', 1, true) then + return op.always(false, 'invalid_path') + end + resolved = join_path(self.import_root, resolved) + end + + return attempt_op(function () + local stream, err = open_stream_now(resolved, 'r') + if not stream then + return false, err + end + local source = blob_source.from_stream(stream) + return fibers.perform(import_source_attempt_op(self, source, meta, opts)) + end) + end) +end + +function Store:delete_op(ref) + return op.guard(function () + if not valid_ref(ref) then + return op.always(false, 'invalid_artifact_ref') + end + return attempt_op(function () + local ok, located_or_err = fibers.perform(locate_artifact_op(self, ref)) + if not ok then + if located_or_err == 'not_found' then + return true, nil + end + return false, located_or_err + end + return fibers.perform(remove_artifact_dir_op(located_or_err.dir)) + end) + end) +end + +---------------------------------------------------------------------- +-- Constructor +---------------------------------------------------------------------- + +function M.new(opts, logger) + opts = opts or {} + + local transient_root = opts.transient_root + or os.getenv('DEVICECODE_ARTIFACT_TRANSIENT_ROOT') + or '/run/devicecode/artifacts' + + local durable_root = opts.durable_root + or os.getenv('DEVICECODE_ARTIFACT_DURABLE_ROOT') + or '/data/devicecode/artifacts' + + local import_root = opts.import_root + or os.getenv('DEVICECODE_IMPORT_ARTIFACT_ROOT') + or os.getenv('DEVICECODE_ARTIFACT_DIR') + or durable_root + or transient_root + + local durable_enabled = opts.durable_enabled + if durable_enabled == nil then + local env = os.getenv('DEVICECODE_DURABLE_ARTIFACTS') + durable_enabled = not (env == '0' or env == 'false' or env == 'FALSE') + end + + return setmetatable({ + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = durable_enabled and true or false, + import_root = import_root, + logger = logger, + }, Store) +end + +M.Store = Store +M.Artifact = Artifact +M.ArtifactSink = ArtifactSink + +return M diff --git a/src/services/hal/drivers/artifact_store_provider.lua b/src/services/hal/drivers/artifact_store_provider.lua new file mode 100644 index 00000000..56570754 --- /dev/null +++ b/src/services/hal/drivers/artifact_store_provider.lua @@ -0,0 +1,275 @@ +---@module 'services.hal.drivers.artifact_store_provider' + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local channel = require 'fibers.channel' + +local hal_types = require 'services.hal.types.core' +local cap_types = require 'services.hal.types.capabilities' +local cap_args = require 'services.hal.types.capability_args' +local control_loop = require 'services.hal.support.control_loop' +local tablex = require 'shared.table' + +local backend_mod = require 'services.hal.drivers.artifact_store' + +local M = {} + +local CONTROL_Q_LEN = 8 +local DEFAULT_STOP_TIMEOUT = 5.0 + +---@class ArtifactStoreProvider +---@field id string +---@field opts table +---@field logger table|nil +---@field scope Scope|nil +---@field control_ch Channel +---@field emit_ch Channel|nil +---@field backend any +---@field started boolean +---@field caps_applied boolean +local Driver = {} +Driver.__index = Driver + +local deep_copy = tablex.deep_copy + + +local function finalise_shell_scope(self, shell_scope) + if self.scope ~= shell_scope then + return + end + self.started = false + self.scope = nil +end + +local function emit_op(emit_ch, class, id, mode, key, data) + return op.guard(function () + local payload, err = hal_types.new.Emit(class, id, mode, key, data) + if not payload then + return op.always(false, tostring(err)) + end + + return emit_ch:put_op(payload):wrap(function () + return true, nil + end) + end) +end + +local function methods_for(self) + return { + status = function (opts, _request) + if opts ~= nil and getmetatable(opts) ~= cap_args.ArtifactStoreStatusOpts then + return op.always(false, 'invalid status opts') + end + return self.backend:status_op() + end, + + ['create-sink'] = function (opts, _request) + if type(opts) ~= 'table' or getmetatable(opts) ~= cap_args.ArtifactStoreCreateSinkOpts then + return op.always(false, 'invalid create-sink opts') + end + return self.backend:create_sink_op(opts.meta, { + policy = opts.policy, + }) + end, + + ['import-path'] = function (opts, _request) + if type(opts) ~= 'table' or getmetatable(opts) ~= cap_args.ArtifactStoreImportPathOpts then + return op.always(false, 'invalid import-path opts') + end + return self.backend:import_path_op(opts.path, opts.meta, { + policy = opts.policy, + }) + end, + + ['import-source'] = function (opts, _request) + if type(opts) ~= 'table' or getmetatable(opts) ~= cap_args.ArtifactStoreImportSourceOpts then + return op.always(false, 'invalid import-source opts') + end + return self.backend:import_source_op(opts.source, opts.meta, { + policy = opts.policy, + }) + end, + + open = function (opts, _request) + if type(opts) ~= 'table' or getmetatable(opts) ~= cap_args.ArtifactStoreOpenOpts then + return op.always(false, 'invalid open opts') + end + return self.backend:open_op(opts.artifact_ref) + end, + + delete = function (opts, _request) + if type(opts) ~= 'table' or getmetatable(opts) ~= cap_args.ArtifactStoreDeleteOpts then + return op.always(false, 'invalid delete opts') + end + return self.backend:delete_op(opts.artifact_ref) + end, + } +end + +local function shell_main(self) + local shell_scope = assert(self.scope, 'artifact_store shell without scope') + assert(self.emit_ch, 'artifact_store shell without emit channel') + + shell_scope:finally(function () + finalise_shell_scope(self, shell_scope) + end) + + local ok_meta, meta_err = fibers.perform(emit_op( + self.emit_ch, + 'artifact-store', + self.id, + 'meta', + 'info', + { + provider = 'hal.artifact_store', + version = 2, + transient_root = self.backend.transient_root, + durable_root = self.backend.durable_root, + durable_enabled = self.backend.durable_enabled, + import_root = self.backend.import_root, + } + )) + if ok_meta ~= true then + error(tostring(meta_err or 'initial meta emit failed'), 0) + end + + local ok_state, state_err = fibers.perform(emit_op( + self.emit_ch, + 'artifact-store', + self.id, + 'state', + 'status', + { state = 'available' } + )) + if ok_state ~= true then + error(tostring(state_err or 'initial state emit failed'), 0) + end + + control_loop.run_request_loop( + self.control_ch, + methods_for(self), + self.logger, + 'artifact-store' + ) +end + +function Driver:capabilities_op(emit_ch) + return op.guard(function () + if self.caps_applied then + return op.always(false, 'capabilities already applied') + end + + self.emit_ch = emit_ch + + local cap, err = cap_types.new.ArtifactStoreCapability(self.id, self.control_ch) + if not cap then + return op.always(false, tostring(err)) + end + + self.caps_applied = true + return op.always(true, { cap }) + end) +end + +function Driver:start_op(owner_scope) + assert(owner_scope ~= nil, 'artifact_store provider start_op: owner_scope is required') + + return op.guard(function () + if self.started then + return op.always(false, 'already started') + end + if not self.caps_applied then + return op.always(false, 'capabilities not applied') + end + if not self.emit_ch then + return op.always(false, 'missing emit channel') + end + + local shell_scope, err = owner_scope:child() + if not shell_scope then + return op.always(false, tostring(err)) + end + + self.scope = shell_scope + + local ok, serr = shell_scope:spawn(function () + return shell_main(self) + end) + if not ok then + self.scope = nil + shell_scope:cancel(tostring(serr or 'shell spawn failed')) + return op.always(false, tostring(serr)) + end + + self.started = true + return op.always(true, nil) + end) +end + +function Driver:terminate(reason) + if self.scope then + self.scope:cancel(reason or 'artifact_store provider terminated') + end + self.started = false + self.scope = nil + return true, nil +end + +function Driver:shutdown_op(timeout) + timeout = timeout or DEFAULT_STOP_TIMEOUT + + return op.guard(function () + if not self.started or not self.scope then + return op.always(true, nil) + end + + local shell_scope = self.scope + shell_scope:cancel('artifact_store provider stopped') + + return fibers.boolean_choice( + shell_scope:join_op():wrap(function () + finalise_shell_scope(self, shell_scope) + return true, nil + end), + sleep.sleep_op(timeout):wrap(function () + return false, 'artifact_store provider stop timeout' + end) + ):wrap(function (completed, _a, b) + if completed then + return true, nil + end + return false, b + end) + end) +end + +function Driver:fault_op() + if self.scope and self.started then + return self.scope:fault_op() + end + return op.never() +end + +function M.new(id, opts, logger) + assert(type(id) == 'string' and id ~= '', 'artifact_store_provider.new: invalid id') + + local raw_opts = opts or {} + local injected_backend = raw_opts.backend + opts = deep_copy(raw_opts) + + return setmetatable({ + id = id, + opts = opts, + logger = logger, + scope = nil, + control_ch = channel.new(CONTROL_Q_LEN), + emit_ch = nil, + backend = injected_backend or backend_mod.new(opts, logger), + started = false, + caps_applied = false, + }, Driver) +end + +M.Driver = Driver +return M diff --git a/src/services/hal/drivers/band.lua b/src/services/hal/drivers/band.lua index 79ade3a4..11f95c0c 100644 --- a/src/services/hal/drivers/band.lua +++ b/src/services/hal/drivers/band.lua @@ -1,617 +1,550 @@ -local queue = require "fibers.queue" -local op = require "fibers.op" -local fiber = require "fibers.fiber" -local log = require "services.log" -local hal_capabilities = require "services.hal.hal_capabilities" -local utils = require "services.hal.utils" -local service = require "service" -local new_msg = require "bus".new_msg -local unpack = table.unpack or unpack +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" +local provider = require "services.hal.backends.band.provider" +local cap_args = require "services.hal.types.capability_args" -local BandDriver = {} -BandDriver.__index = BandDriver +local fibers = require "fibers" +local channel = require "fibers.channel" -local BAND_MAPPING = { - ["2G"] = "802_11g", - ["5G"] = "802_11a" -} -local KICK_MODES = { - none = 0, - compare = 1, - absolute = 2, - both = 3 -} -local SUPPORT_OPTIONS = { - 'ht', 'vht' -} -local VALID_UPDATE_KEYS = { - 'client', 'chan_util', 'hostapd', 'beacon_reports', 'tcp_con' -} -local VALID_TIMEOUTS = { - 'probe', 'client', 'ap' -} -local VALID_RRM_MODES = { - 'PAT' -} -local VALID_LEGACY_OPTIONS = { +local CONTROL_Q_LEN = 16 + +local VALID_KICK_MODES = { 'none', 'compare', 'absolute', 'both' } +local VALID_RRM_MODES = { 'PAT' } +local VALID_LEGACY_KEYS = { 'eval_probe_req', 'eval_assoc_req', 'eval_auth_req', - 'min_probe_count', 'deny_assoc_reason', 'deny_auth_reason' -} -local VALID_NETWORKING_METHODS = { - 'broadcast', 'tcp+umdns', 'tcp', - broadcast = 0, - ['tcp+umdns'] = 2, - ['multicast'] = 2, - tcp = 3 + 'min_probe_count', 'deny_assoc_reason', 'deny_auth_reason', } - -local sections = { - { name = 'global', type = 'metric' }, - { name = '802_11g', type = 'metric' }, - { name = '802_11a', type = 'metric' }, - { name = 'gbltime', type = 'times' }, - { name = 'gblnet', type = 'network'}, - { name = 'localcfg', type = 'local' } +local VALID_BAND_KICKING_OPTS = { + 'rssi_center', 'rssi_reward_threshold', 'rssi_reward', + 'rssi_penalty_threshold', 'rssi_penalty', 'rssi_weight', + 'channel_util_reward_threshold', 'channel_util_reward', + 'channel_util_penalty_threshold', 'channel_util_penalty', } - -------------------------------------------------------------------------- ---- BandCapabilities ---------------------------------------------------- - -function BandDriver:set_log_level(ctx, level) - if type(level) ~= "number" or level < 0 then - return nil, "Invalid log level" - end - - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'localcfg', 'log_level', level } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - - return true, nil +local VALID_UPDATE_KEYS = { 'client', 'chan_util', 'hostapd', 'beacon_reports', 'tcp_con' } +local VALID_CLEANUP_KEYS = { 'probe', 'client', 'ap' } +local VALID_NETWORKING_METHODS = { 'broadcast', 'tcp+umdns', 'multicast', 'tcp' } +local VALID_NETWORKING_OPTS = { 'ip', 'port', 'broadcast_port', 'enable_encryption' } +local VALID_BANDS = { '2G', '5G' } +local VALID_SUPPORTS = { 'ht', 'vht' } + +local function is_in(value, list) + for _, v in ipairs(list) do + if v == value then return true end + end + return false end -function BandDriver:set_kicking(ctx, - mode, - bandwidth_threshold, - kicking_threshold, - evals_before_kick -) - if not KICK_MODES[mode] then - return nil, "Invalid kick mode" - end - if type(bandwidth_threshold) ~= "number" or bandwidth_threshold < 0 then - return nil, "Invalid bandwidth threshold" - end - if type(kicking_threshold) ~= "number" or kicking_threshold < 0 then - return nil, "Invalid kicking threshold" - end - if type(evals_before_kick) ~= "number" or evals_before_kick < 0 then - return nil, "Invalid evaluations before kick" - end - - local reqs = {} - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'global', 'kicking', KICK_MODES[mode] } - )) - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'global', 'bandwidth_threshold', bandwidth_threshold } - )) - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'global', 'kicking_threshold', kicking_threshold } - )) - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'global', 'min_number_to_kick', evals_before_kick } - )) - - for _, req in ipairs(reqs) do - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - end +---@class BandDriver +---@field id string Always '1' +---@field scope Scope +---@field control_ch Channel +---@field cap_emit_ch Channel? +---@field staged table In-memory staged band config +---@field initialised boolean +---@field caps_applied boolean +---@field log table +---@field backend table +local BandDriver = {} +BandDriver.__index = BandDriver - return true, nil +------------------------------------------------------------------------ +-- RPC handler methods +------------------------------------------------------------------------ + +---@param opts BandSetLogLevelOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_log_level(opts) + if getmetatable(opts) ~= cap_args.BandSetLogLevelOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetLogLevelOpts(opts.level) + if not casted then return false, err end + opts = casted + end + local level = opts.level + if type(level) ~= 'number' or level < 0 then + return false, 'level must be a non-negative number' + end + self.staged.log_level = level + return true end -function BandDriver:set_station_counting(ctx, use_station_count, max_station_diff) - if type(use_station_count) ~= "boolean" then - return nil, "Invalid use_station_count" - end - if type(max_station_diff) ~= "number" or max_station_diff < 0 then - return nil, "Invalid max_station_diff" - end +---@param opts BandSetKickingOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_kicking(opts) + if getmetatable(opts) ~= cap_args.BandSetKickingOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetKickingOpts( + opts.mode, opts.bandwidth_threshold, opts.kicking_threshold, opts.evals_before_kick) + if not casted then return false, err end + opts = casted + end + if not is_in(opts.mode, VALID_KICK_MODES) then + return false, 'mode must be one of: ' .. table.concat(VALID_KICK_MODES, ', ') + end + if type(opts.bandwidth_threshold) ~= 'number' or opts.bandwidth_threshold < 0 then + return false, 'bandwidth_threshold must be a non-negative number' + end + if type(opts.kicking_threshold) ~= 'number' or opts.kicking_threshold < 0 then + return false, 'kicking_threshold must be a non-negative number' + end + if type(opts.evals_before_kick) ~= 'number' or opts.evals_before_kick < 0 then + return false, 'evals_before_kick must be a non-negative integer' + end + self.staged.kicking = { + mode = opts.mode, + bandwidth_threshold = opts.bandwidth_threshold, + kicking_threshold = opts.kicking_threshold, + evals_before_kick = opts.evals_before_kick, + } + return true +end - local reqs = {} +---@param opts BandSetStationCountingOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_station_counting(opts) + if getmetatable(opts) ~= cap_args.BandSetStationCountingOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetStationCountingOpts(opts.use_station_count, opts.max_station_diff) + if not casted then return false, err end + opts = casted + end + if type(opts.use_station_count) ~= 'boolean' then + return false, 'use_station_count must be a boolean' + end + if type(opts.max_station_diff) ~= 'number' or opts.max_station_diff < 0 then + return false, 'max_station_diff must be a non-negative integer' + end + self.staged.station_counting = { + use_station_count = opts.use_station_count, + max_station_diff = opts.max_station_diff, + } + return true +end - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'global', 'use_station_count', use_station_count} - )) +---@param opts BandSetRrmModeOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_rrm_mode(opts) + if getmetatable(opts) ~= cap_args.BandSetRrmModeOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetRrmModeOpts(opts.mode) + if not casted then return false, err end + opts = casted + end + if not is_in(opts.mode, VALID_RRM_MODES) then + return false, 'mode must be one of: ' .. table.concat(VALID_RRM_MODES, ', ') + end + self.staged.rrm_mode = opts.mode + return true +end - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'global', 'max_station_diff', max_station_diff } - )) +---@param opts BandSetNeighbourReportsOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_neighbour_reports(opts) + if getmetatable(opts) ~= cap_args.BandSetNeighbourReportsOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetNeighbourReportsOpts(opts.dyn_report_num, opts.disassoc_report_len) + if not casted then return false, err end + opts = casted + end + local dyn = tonumber(opts.dyn_report_num) + local dis = tonumber(opts.disassoc_report_len) + if not dyn or dyn < 0 then + return false, 'dyn_report_num must be a non-negative integer' + end + if not dis or dis < 0 then + return false, 'disassoc_report_len must be a non-negative integer' + end + self.staged.neighbour_reports = { + dyn_report_num = dyn, + disassoc_report_len = dis, + } + return true +end - for _, req in ipairs(reqs) do - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err +---@param opts BandSetLegacyOptionsOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_legacy_options(opts) + if getmetatable(opts) ~= cap_args.BandSetLegacyOptionsOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetLegacyOptionsOpts(opts.opts) + if not casted then return false, err end + opts = casted + end + local legacy_opts = opts.opts + if type(legacy_opts) ~= 'table' then + return false, 'opts.opts must be a table' + end + if not self.staged.legacy then self.staged.legacy = {} end + for key, value in pairs(legacy_opts) do + if not is_in(key, VALID_LEGACY_KEYS) then + return false, 'unknown legacy option key: ' .. tostring(key) + end + if value == nil then + return false, 'nil value for legacy option: ' .. key end + self.staged.legacy[key] = value end - - return true, nil + return true end -function BandDriver:set_rrm_mode(ctx, mode) - if not utils.is_in(mode, VALID_RRM_MODES) then - return nil, "Invalid RRM mode" - end - - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'global', 'rrm_mode', mode } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - - return true, nil +---@param opts BandSetBandPriorityOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_band_priority(opts) + if getmetatable(opts) ~= cap_args.BandSetBandPriorityOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetBandPriorityOpts(opts.band, opts.priority) + if not casted then return false, err end + opts = casted + end + local band = type(opts.band) == 'string' and opts.band:upper() or '' + if not is_in(band, VALID_BANDS) then + return false, 'band must be "2G" or "5G"' + end + if type(opts.priority) ~= 'number' or opts.priority < 0 then + return false, 'priority must be a non-negative number' + end + if not self.staged.band_priorities then self.staged.band_priorities = {} end + self.staged.band_priorities[band] = { initial_score = opts.priority } + return true end -function BandDriver:set_neighbour_reports(ctx, dyn_report_num, disassoc_report_len) - dyn_report_num = tonumber(dyn_report_num) - disassoc_report_len = tonumber(disassoc_report_len) - - if type(dyn_report_num) ~= "number" or dyn_report_num < 0 then - return nil, "Invalid dyn_report_num" - end - if type(disassoc_report_len) ~= "number" or disassoc_report_len < 0 then - return nil, "Invalid disassoc_report_len" - end - - local reqs = {} - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'global', 'set_hostapd_nr', dyn_report_num } - )) - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'global', 'disassoc_nr_length', disassoc_report_len } - )) - - for _, req in ipairs(reqs) do - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err +---@param opts BandSetBandKickingOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_band_kicking(opts) + if getmetatable(opts) ~= cap_args.BandSetBandKickingOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetBandKickingOpts(opts.band, opts.options) + if not casted then return false, err end + opts = casted + end + local band = type(opts.band) == 'string' and opts.band:upper() or '' + if not is_in(band, VALID_BANDS) then + return false, 'band must be "2G" or "5G"' + end + if type(opts.options) ~= 'table' then + return false, 'options must be a table' + end + if not self.staged.band_kicking then self.staged.band_kicking = {} end + if not self.staged.band_kicking[band] then self.staged.band_kicking[band] = {} end + for key, value in pairs(opts.options) do + if not is_in(key, VALID_BAND_KICKING_OPTS) then + return false, 'unknown band kicking option: ' .. tostring(key) end + local n = tonumber(value) + if n == nil then + return false, 'value for ' .. key .. ' must be a number' + end + self.staged.band_kicking[band][key] = n end - - return true, nil + return true end -function BandDriver:set_legacy_options(ctx, opts) - if type(opts) ~= "table" then - return nil, "Options must be a table" - end +---@param opts BandSetSupportBonusOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_support_bonus(opts) + if getmetatable(opts) ~= cap_args.BandSetSupportBonusOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetSupportBonusOpts(opts.band, opts.support, opts.reward) + if not casted then return false, err end + opts = casted + end + local band = type(opts.band) == 'string' and opts.band:upper() or '' + if not is_in(band, VALID_BANDS) then + return false, 'band must be "2G" or "5G"' + end + if not is_in(opts.support, VALID_SUPPORTS) then + return false, 'support must be "ht" or "vht"' + end + if type(opts.reward) ~= 'number' then + return false, 'reward must be a number' + end + if not self.staged.support_bonus then self.staged.support_bonus = {} end + if not self.staged.support_bonus[band] then self.staged.support_bonus[band] = {} end + self.staged.support_bonus[band][opts.support] = opts.reward + return true +end - for key, value in pairs(opts) do - if not utils.is_in(key, VALID_LEGACY_OPTIONS) then - return false, "No entry associated with " .. key +---@param opts BandSetUpdateFreqOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_update_freq(opts) + if getmetatable(opts) ~= cap_args.BandSetUpdateFreqOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetUpdateFreqOpts(opts.updates) + if not casted then return false, err end + opts = casted + end + if type(opts.updates) ~= 'table' then + return false, 'updates must be a table' + end + if not self.staged.update_freq then self.staged.update_freq = {} end + for key, value in pairs(opts.updates) do + if not is_in(key, VALID_UPDATE_KEYS) then + return false, 'unknown update key: ' .. tostring(key) end - if type(value) == "nil" then - return false, "Invalid value for " .. key - end - - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'global', key, value } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err + if type(value) ~= 'number' or value < 0 then + return false, 'value for ' .. key .. ' must be a non-negative number' end + self.staged.update_freq[key] = value end - - return true, nil + return true end -function BandDriver:set_band_priority(ctx, band, priority) - band = band:upper() - if type(priority) ~= "number" or priority < 0 then - return nil, "Invalid priority" - end - local full_band = BAND_MAPPING[band] - if not full_band then - return nil, "Invalid band" - end - - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', full_band, 'initial_score', priority } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - - return true, nil +---@param opts BandSetClientInactiveKickoffOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_client_inactive_kickoff(opts) + if getmetatable(opts) ~= cap_args.BandSetClientInactiveKickoffOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetClientInactiveKickoffOpts(opts.timeout) + if not casted then return false, err end + opts = casted + end + local timeout = tonumber(opts.timeout) + if not timeout or timeout < 0 then + return false, 'timeout must be a non-negative integer' + end + self.staged.con_timeout = timeout + return true end -function BandDriver:set_band_kicking(ctx, - band, - options) - band = band:upper() - local full_band = BAND_MAPPING[band] - if not full_band then - return nil, "Invalid band" - end - - local configs = { - rssi_center = {type = "number", entry = { 'dawn', full_band, 'rssi_center' }}, - rssi_reward_threshold = {type = "number", entry = { 'dawn', full_band, 'rssi_val' }}, - rssi_reward = {type = "number", entry = { 'dawn', full_band, 'rssi' }}, - rssi_penalty_threshold = {type = "number", entry = { 'dawn', full_band, 'low_rssi_val' }}, - rssi_penalty = {type = "number", entry = { 'dawn', full_band, 'low_rssi' }}, - rssi_weight = {type = "number", entry = { 'dawn', full_band, 'rssi_weight' }}, - channel_util_reward_threshold = {type = "number", entry = { 'dawn', full_band, 'chan_util_val' }}, - channel_util_reward = {type = "number", entry = { 'dawn', full_band, 'chan_util' }}, - channel_util_penalty_threshold = {type = "number", entry = { 'dawn', full_band, 'max_chan_util_val' }}, - channel_util_penalty = {type = "number", entry = { 'dawn', full_band, 'max_chan_util' }} - } - - local reqs = {} - for key, value in pairs(options) do - local config = configs[key] - if not config then - return false, "No entry associated with " .. key - end - if config.type == "number" then - value = tonumber(value) +---@param opts BandSetCleanupOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_cleanup(opts) + if getmetatable(opts) ~= cap_args.BandSetCleanupOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetCleanupOpts(opts.timeouts) + if not casted then return false, err end + opts = casted + end + if type(opts.timeouts) ~= 'table' then + return false, 'timeouts must be a table' + end + if not self.staged.cleanup then self.staged.cleanup = {} end + for key, value in pairs(opts.timeouts) do + if not is_in(key, VALID_CLEANUP_KEYS) then + return false, 'unknown cleanup key: ' .. tostring(key) end - if type(value) ~= config.type then - return false, "Invalid type for " .. key - end - - local config_entry = {unpack(config.entry)} - table.insert(config_entry, value) - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - config_entry - )) - end - - for _, req in ipairs(reqs) do - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err + if type(value) ~= 'number' or value < 0 then + return false, 'value for cleanup.' .. key .. ' must be a non-negative number' end + self.staged.cleanup[key] = value end - - return true, nil + return true end -function BandDriver:set_support_bonus(ctx, band, support, reward) - band = band:upper() - if not BAND_MAPPING[band] then - return nil, "Invalid band" - end - if not utils.is_in(support, SUPPORT_OPTIONS) then - return nil, "Invalid support option" +---@param opts BandSetNetworkingOpts +---@return boolean ok +---@return string? reason +function BandDriver:set_networking(opts) + if getmetatable(opts) ~= cap_args.BandSetNetworkingOpts then + opts = opts or {} + local casted, err = cap_args.new.BandSetNetworkingOpts(opts.method, opts.options) + if not casted then return false, err end + opts = casted end - if type(reward) ~= "number" then - return nil, "Invalid reward" - end - - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', BAND_MAPPING[band], support .. '_support', reward } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err + if not is_in(opts.method, VALID_NETWORKING_METHODS) then + return false, 'method must be one of: ' .. table.concat(VALID_NETWORKING_METHODS, ', ') end - - return true, nil -end - -function BandDriver:set_update_freq(ctx, updates) - if type(updates) ~= "table" then - return nil, "Updates must be a table" + if type(opts.options) ~= 'table' then + return false, 'options must be a table' end - local reqs = {} - for key, freq in pairs(updates) do - if not utils.is_in(key, VALID_UPDATE_KEYS) then - return nil, "Invalid update key: " .. key + local net = { method = opts.method } + for key, value in pairs(opts.options) do + if not is_in(key, VALID_NETWORKING_OPTS) then + return false, 'unknown networking option: ' .. tostring(key) end - if type(freq) ~= "number" or freq < 0 then - return nil, "Invalid frequency for " .. key + if key == 'ip' and type(value) ~= 'string' then + return false, 'networking.ip must be a string' end - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'gbltime', 'update_' .. key, freq } - )) - end - - for _, req in ipairs(reqs) do - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err + if (key == 'port' or key == 'broadcast_port') and type(value) ~= 'number' then + return false, 'networking.' .. key .. ' must be a number' end + if key == 'enable_encryption' and type(value) ~= 'boolean' then + return false, 'networking.enable_encryption must be a boolean' + end + net[key] = value end - - return true, nil + self.staged.networking = net + return true end -function BandDriver:set_client_inactive_kickoff(ctx, timeout) - timeout = tonumber(timeout) - if type(timeout) ~= "number" or timeout < 0 then - return nil, "Invalid timeout" +---@return boolean ok +---@return string? reason +function BandDriver:apply() + local ok, err = pcall(function() self.backend:apply(self.staged) end) + if not ok then + return false, tostring(err) end - - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'gbltime', 'con_timeout', timeout } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - - return true, nil + return true end -function BandDriver:set_cleanup(ctx, timeouts) - if type(timeouts) ~= "table" then - return nil, "Timeouts must be a table" - end - - local reqs = {} - for key, timeout in pairs(timeouts) do - if not utils.is_in(key, VALID_TIMEOUTS) then - return nil, "Invalid timeout key: " .. key - end - if type(timeout) ~= "number" or timeout < 0 then - return nil, "Invalid timeout for " .. key - end - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'gbltime', 'remove_' .. key, timeout } - )) - end - - for _, req in ipairs(reqs) do - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end +---@return boolean ok +---@return string? reason +function BandDriver:clear() + local ok, err = self.backend:clear() + if not ok then + return false, err end + self.staged = {} + return true +end - return true, nil +---@return boolean ok +function BandDriver:rollback() + self.staged = {} + return true end -function BandDriver:set_networking(ctx, method, options) - local configs = { - ip = {type = "string", entry = { 'dawn', 'gblnet', 'tcp_ip' }}, - port = {type = "number", entry = { 'dawn', 'gblnet', 'tcp_port' }}, - broadcast_port = {type = "number", entry = { 'dawn', 'gblnet', 'broadcast_port' }}, - enable_encryption = {type = "boolean", entry = { 'dawn', 'gblnet', 'use_symm_enc' }}, - } +------------------------------------------------------------------------ +-- Control manager fiber +------------------------------------------------------------------------ - local method_id = VALID_NETWORKING_METHODS[method] - if not method_id then - return nil, "Invalid networking method: " .. tostring(method) - end +function BandDriver:control_manager() + fibers.current_scope():finally(function() + self.log:debug({ what = 'band_driver_stopped', id = self.id }) + end) - local reqs = {} - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', 'gblnet', 'method', method_id } - )) - for key, value in pairs(options) do - local config = configs[key] - if not config then - return false, "No entry associated with " .. key - end - if type(value) ~= config.type then - return false, "Invalid type for " .. key + while true do + local name, request = fibers.perform(fibers.named_choice({ + rpc = self.control_ch:get_op(), + cancel = fibers.current_scope():cancel_op(), + })) + + if name == 'cancel' then break end + + local fn = self[request.verb] + local ok, reason + if type(fn) ~= 'function' then + ok, reason = false, 'unknown verb: ' .. tostring(request.verb) + else + local call_ok, r1, r2 = pcall(fn, self, request.opts) + if not call_ok then + ok, reason = false, tostring(r1) + else + ok, reason = r1, r2 + end end - local config_entry = {unpack(config.entry)} - table.insert(config_entry, value) - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - config_entry - )) - end - - for _, req in ipairs(reqs) do - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err + local reply = hal_types.new.Reply(ok, reason) + if reply then + request.reply_ch:put(reply) end end - return true, nil end -function BandDriver:apply(ctx) - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'commit' }, - { 'dawn' } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end +------------------------------------------------------------------------ +-- Public driver interface +------------------------------------------------------------------------ - return true, nil +---@return string err empty string on success +function BandDriver:init() + local ok, err = self.backend:clear() + if not ok then + return "band backend clear failed: " .. tostring(err) + end + self.initialised = true + return "" end -------------------------------------------------------------------------- - ---- Apply band capabilities ---- @param capability_info_q Queue ---- @return table ---- @return string? -function BandDriver:apply_capabilities(capability_info_q) - self.info_q = capability_info_q - local capabilities = { - band = { - control = hal_capabilities.new_band_capability(self.command_q), - id = "1" +---@param emit_ch Channel +---@return Capability[]? caps +---@return string err +function BandDriver:capabilities(emit_ch) + if not self.initialised then + return nil, "driver not initialised" + end + if self.caps_applied then + return nil, "capabilities already applied" + end + self.cap_emit_ch = emit_ch + + local cap, cap_err = cap_types.new.Capability( + 'band', + '1', + self.control_ch, + { + 'set_log_level', + 'set_kicking', + 'set_station_counting', + 'set_rrm_mode', + 'set_neighbour_reports', + 'set_legacy_options', + 'set_band_priority', + 'set_band_kicking', + 'set_support_bonus', + 'set_update_freq', + 'set_client_inactive_kickoff', + 'set_cleanup', + 'set_networking', + 'apply', + 'clear', + 'rollback', } - } - return capabilities, nil -end - ---- Handle a capability request ---- @param ctx Context ---- @param request table -function BandDriver:handle_capability(ctx, request) - local command = request.command - local args = request.args or {} - local ret_ch = request.return_channel - - if type(ret_ch) == 'nil' then return end - - if type(command) == "nil" then - ret_ch:put({ - result = nil, - err = 'No command was provided' - }) - return + ) + if not cap then + return nil, cap_err end - local func = self[command] - if type(func) ~= "function" then - ret_ch:put({ - result = nil, - err = "Command does not exist" - }) - return - end - - fiber.spawn(function() - local result, err = func(self, ctx, unpack(args)) - - ret_ch:put({ - result = result, - err = err - }) - end) + self.caps_applied = true + return { cap }, "" end ---- Main driver loop ---- @param ctx Context -function BandDriver:_main(ctx) - log.info(string.format( - "%s - %s: Starting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - - -- Main event loop - while not ctx:err() do - op.choice( - self.command_q:get_op():wrap(function(req) - self:handle_capability(ctx, req) - end), - ctx:done_op() - ):perform() - end - - log.info(string.format( - "%s - %s: Exiting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) -end - ---- Spawn driver fiber ---- @param conn Connection The bus connection -function BandDriver:spawn(conn) - self.conn = conn - service.spawn_fiber(string.format("Band Driver (%s)", self.band), conn, self.ctx, function(fctx) - self:_main(fctx) - end) -end - -function BandDriver:init(ctx, conn) - local req = conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'foreach' }, - { 'dawn', nil, function(cursor, section) - if section[".type"] ~= "hostapd" then - cursor:delete('dawn', section[".name"]) - end - end } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return resp.payload.result, resp.payload.err or ctx_err - end - - for _, section in ipairs(sections) do - local req = conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'dawn', section.name, section.type } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return resp.payload.result, resp.payload.err or ctx_err - end +---@return boolean ok +---@return string err +function BandDriver:start() + if not self.initialised then + return false, "driver not initialised" end - - local commit_req = conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'commit' }, - { 'dawn' } - )) - - local resp, ctx_err = commit_req:next_msg_with_context(ctx) - commit_req:unsubscribe() - if resp.payload then - return resp.payload.result, resp.payload.err or ctx_err + if not self.caps_applied then + return false, "capabilities not applied" end - return nil, ctx_err or "No response from UCI" + self.scope:spawn(function() self:control_manager() end) + return true, "" end -function BandDriver.new(ctx) - local self = { - ctx = ctx, - command_q = queue.new(10) - } - return setmetatable(self, BandDriver) +---Create a new BandDriver instance. +---@param logger table +---@return BandDriver? driver +---@return string err +local function new(logger) + local bknd, berr = provider.new() + if not bknd then + return nil, "no band backend: " .. tostring(berr) + end + + local scope, serr = fibers.current_scope():child() + if not scope then + return nil, "failed to create child scope: " .. tostring(serr) + end + + local driver = setmetatable({ + id = '1', + scope = scope, + control_ch = channel.new(CONTROL_Q_LEN), + cap_emit_ch = nil, + staged = {}, + initialised = false, + caps_applied = false, + log = logger, + backend = bknd, + }, BandDriver) + + return driver, "" end -return { new = BandDriver.new } +return { + new = new, + Driver = BandDriver, +} diff --git a/src/services/hal/drivers/control_store.lua b/src/services/hal/drivers/control_store.lua new file mode 100644 index 00000000..508e4125 --- /dev/null +++ b/src/services/hal/drivers/control_store.lua @@ -0,0 +1,249 @@ +---@module 'services.hal.drivers.control_store' + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local channel = require 'fibers.channel' + +local hal_types = require 'services.hal.types.core' +local cap_types = require 'services.hal.types.capabilities' +local cap_args = require 'services.hal.types.capability_args' +local control_loop = require 'services.hal.support.control_loop' + +local provider_mod = require 'services.hal.drivers.control_store_provider' + +local M = {} + +local CONTROL_Q_LEN = 8 +local DEFAULT_STOP_TIMEOUT = 5.0 + +---@class ControlStoreDriver +---@field id string +---@field root string +---@field scope Scope|nil +---@field control_ch Channel +---@field emit_ch Channel|nil +---@field logger table|nil +---@field provider ControlStoreProvider +---@field started boolean +---@field caps_applied boolean +local Driver = {} +Driver.__index = Driver + + +local function finalise_shell_scope(self, shell_scope) + if self.scope ~= shell_scope then + return + end + self.started = false + self.scope = nil +end + +local function emit_op(emit_ch, class, id, mode, key, data) + return op.guard(function () + local payload, err = hal_types.new.Emit(class, id, mode, key, data) + if not payload then + return op.always(false, tostring(err)) + end + + return emit_ch:put_op(payload):wrap(function () + return true, nil + end) + end) +end + +local function methods_for(self) + return { + status = function (opts, request) + return self.provider:status_op() + end, + + get = function (opts, request) + if type(opts) ~= 'table' or getmetatable(opts) ~= cap_args.ControlStoreGetOpts then + return op.always(false, 'invalid get opts') + end + return self.provider:get_op(opts) + end, + + put = function (opts, request) + if type(opts) ~= 'table' or getmetatable(opts) ~= cap_args.ControlStorePutOpts then + return op.always(false, 'invalid put opts') + end + return self.provider:put_op(opts) + end, + + delete = function (opts, request) + if type(opts) ~= 'table' or getmetatable(opts) ~= cap_args.ControlStoreDeleteOpts then + return op.always(false, 'invalid delete opts') + end + return self.provider:delete_op(opts) + end, + + list = function (opts, request) + if opts ~= nil and (type(opts) ~= 'table' or getmetatable(opts) ~= cap_args.ControlStoreListOpts) then + return op.always(false, 'invalid list opts') + end + return self.provider:list_op(opts) + end, + } +end + +local function shell_main(self) + local shell_scope = assert(self.scope, 'control_store shell without scope') + assert(self.emit_ch, 'control_store shell without emit channel') + + shell_scope:finally(function () + finalise_shell_scope(self, shell_scope) + end) + + local eok1, eerr1 = fibers.perform(emit_op(self.emit_ch, 'control-store', self.id, 'meta', 'details', { + root = self.root, + kind = 'control-store', + })) + if eok1 ~= true then + error(tostring(eerr1 or 'initial meta emit failed'), 0) + end + + local eok2, eerr2 = fibers.perform(emit_op(self.emit_ch, 'control-store', self.id, 'state', 'status', { + state = 'available', + })) + if eok2 ~= true then + error(tostring(eerr2 or 'initial state emit failed'), 0) + end + + control_loop.run_request_loop(self.control_ch, methods_for(self), self.logger, 'control-store') +end + +function Driver:capabilities_op(emit_ch) + return op.guard(function () + if self.caps_applied then + return op.always(false, 'capabilities already applied') + end + + self.emit_ch = emit_ch + + local cap, err = cap_types.new.ControlStoreCapability(self.id, self.control_ch) + if not cap then + return op.always(false, tostring(err)) + end + + self.caps_applied = true + return op.always(true, { cap }) + end) +end + +---@param owner_scope Scope +function Driver:start_op(owner_scope) + assert(owner_scope ~= nil, 'control_store driver start_op: owner_scope is required') + + return op.guard(function () + if self.started then + return op.always(false, 'already started') + end + if not self.caps_applied then + return op.always(false, 'capabilities not applied') + end + if not self.emit_ch then + return op.always(false, 'missing emit channel') + end + + return fibers.run_scope_op(function () + local pok, perr = fibers.perform(self.provider:prepare_op()) + if not pok then + return false, tostring(perr or 'control_store provider prepare failed') + end + + local shell_scope, serr = owner_scope:child() + if not shell_scope then + return false, tostring(serr) + end + + self.scope = shell_scope + + local ok, err = shell_scope:spawn(function () + return shell_main(self) + end) + if not ok then + self.scope = nil + shell_scope:cancel(tostring(err or 'shell spawn failed')) + return false, tostring(err) + end + + self.started = true + return true, nil + end):wrap(function (status, reason, ok, err) + if status ~= 'ok' then + return false, tostring(reason or err or 'control_store driver start failed') + end + return ok, err + end) + end) +end + +function Driver:terminate(reason) + if self.scope then + self.scope:cancel(reason or 'control_store driver terminated') + end + self.started = false + self.scope = nil + return true, nil +end + +function Driver:shutdown_op(timeout) + timeout = timeout or DEFAULT_STOP_TIMEOUT + + return op.guard(function () + if not self.started or not self.scope then + return op.always(true, nil) + end + + local shell_scope = self.scope + shell_scope:cancel() + + return fibers.boolean_choice( + shell_scope:join_op():wrap(function () + finalise_shell_scope(self, shell_scope) + return true, nil + end), + sleep.sleep_op(timeout):wrap(function () + return false, 'control_store driver stop timeout' + end) + ):wrap(function (completed, a, b) + if completed then + return true, nil + end + return false, b + end) + end) +end + +function Driver:fault_op() + if self.scope and self.started then + return self.scope:fault_op() + end + return op.never() +end + +---@param id string +---@param root string +---@param logger table|nil +---@return ControlStoreDriver +function M.new(id, root, logger) + assert(type(id) == 'string' and id ~= '', 'control_store.new: invalid id') + assert(type(root) == 'string' and root ~= '', 'control_store.new: invalid root') + + return setmetatable({ + id = id, + root = root, + scope = nil, + control_ch = channel.new(CONTROL_Q_LEN), + emit_ch = nil, + logger = logger, + provider = provider_mod.new(root, logger), + started = false, + caps_applied = false, + }, Driver) +end + +M.Driver = Driver +return M diff --git a/src/services/hal/drivers/control_store_provider.lua b/src/services/hal/drivers/control_store_provider.lua new file mode 100644 index 00000000..8fcabe0c --- /dev/null +++ b/src/services/hal/drivers/control_store_provider.lua @@ -0,0 +1,303 @@ +---@module 'services.hal.drivers.control_store_provider' + +local fibers = require 'fibers' +local op = require 'fibers.op' +local file = require 'fibers.io.file' +local resource = require 'devicecode.support.resource' + +local M = {} + +---@class ControlStoreProvider +---@field root string +---@field logger table|nil +local Provider = {} +Provider.__index = Provider + +local INDEX_FILE = '.control_store_index' + +local function valid_key(key) + return type(key) == 'string' + and key ~= '' + and not key:find('[/\\]', 1) + and key:match('^[%w%._%-]+$') ~= nil +end + +local function path_for(root, key) + return root .. '/' .. key +end + +local function index_path(root) + return root .. '/' .. INDEX_FILE +end + +local function split_lines(s) + local out = {} + if s == '' then + return out + end + for line in (s .. '\n'):gmatch('(.-)\n') do + if line ~= '' then + out[#out + 1] = line + end + end + return out +end + +local function join_lines(xs) + return table.concat(xs, '\n') +end + +local function sort_unique(xs) + local seen, out = {}, {} + for _, x in ipairs(xs) do + if not seen[x] then + seen[x] = true + out[#out + 1] = x + end + end + table.sort(out) + return out +end + +local function is_not_found_err(err) + if err == nil then + return false + end + local s = tostring(err):lower() + return s:find('no such file', 1, true) ~= nil + or s:find('not found', 1, true) ~= nil + or s:find('enoent', 1, true) ~= nil +end + +-- Box the currently synchronous file.open(...) impurity inside a single +-- operation-owned subtree. The caller still gets a proper Op. +local function with_open_file_op(path, mode, body_fn) + return fibers.run_scope_op(function (scope) + local f, err = file.open(path, mode) + if not f then + return false, tostring(err) + end + + scope:finally(function (_, status, primary) + resource.terminate_checked(f, primary or status or 'control store file closed', 'control store file cleanup failed') + end) + + return body_fn(f) + end):wrap(function (st, rep, ok, value_or_err) + if st ~= 'ok' then + return false, tostring(value_or_err or rep) + end + return ok, value_or_err + end) +end + +local function read_file_required_op(path) + return with_open_file_op(path, 'r', function (f) + local body, rerr = fibers.perform(f:read_all_op()) + if rerr ~= nil then + return false, tostring(rerr) + end + return true, body or '' + end) +end + +local function read_index_op(root) + local path = index_path(root) + + return fibers.run_scope_op(function () + local ok, body_or_err = fibers.perform(read_file_required_op(path)) + if ok then + return true, sort_unique(split_lines(body_or_err)) + end + + if is_not_found_err(body_or_err) then + return true, {} + end + + return false, body_or_err + end):wrap(function (st, rep, ok, value_or_err) + if st ~= 'ok' then + return false, tostring(value_or_err or rep) + end + return ok, value_or_err + end) +end + +local function write_file_op(path, data) + return with_open_file_op(path, 'w', function (f) + local n, werr = fibers.perform(f:write_op(data)) + if n == nil then + return false, tostring(werr) + end + return true, nil + end) +end + +local function write_index_op(root, keys) + return write_file_op(index_path(root), join_lines(sort_unique(keys))) +end + +function Provider:prepare_op() + return op.guard(function () + local ok, err = file.mkdir_p(self.root) + if ok then return op.always(true, nil) end + return op.always(false, tostring(err or 'control_store_root_create_failed')) + end) +end + +function Provider:status_op() + return op.always(true, { + root = self.root, + kind = 'control-store', + }) +end + +function Provider:get_op(opts) + return op.guard(function () + if type(opts) ~= 'table' or not valid_key(opts.key) then + return op.always(false, 'invalid key') + end + + return fibers.run_scope_op(function () + local ok_index, keys_or_err = fibers.perform(read_index_op(self.root)) + if not ok_index then + return false, keys_or_err + end + + local present = false + for _, k in ipairs(keys_or_err) do + if k == opts.key then + present = true + break + end + end + if not present then + return false, 'not found' + end + + return fibers.perform(read_file_required_op(path_for(self.root, opts.key))) + end):wrap(function (st, rep, ok, value_or_err) + if st ~= 'ok' then + return false, tostring(value_or_err or rep) + end + return ok, value_or_err + end) + end) +end + +function Provider:put_op(opts) + return op.guard(function () + if type(opts) ~= 'table' or not valid_key(opts.key) then + return op.always(false, 'invalid key') + end + if type(opts.data) ~= 'string' then + return op.always(false, 'data must be a string') + end + + return fibers.run_scope_op(function () + local okw, werr = fibers.perform(write_file_op(path_for(self.root, opts.key), opts.data)) + if not okw then + return false, werr + end + + local ok_index, keys_or_err = fibers.perform(read_index_op(self.root)) + if not ok_index then + return false, keys_or_err + end + + keys_or_err[#keys_or_err + 1] = opts.key + return fibers.perform(write_index_op(self.root, keys_or_err)) + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) + end) +end + +function Provider:delete_op(opts) + return op.guard(function () + if type(opts) ~= 'table' or not valid_key(opts.key) then + return op.always(false, 'invalid key') + end + + return fibers.run_scope_op(function () + local ok_index, keys_or_err = fibers.perform(read_index_op(self.root)) + if not ok_index then + return false, keys_or_err + end + + local kept, present = {}, false + for _, k in ipairs(keys_or_err) do + if k == opts.key then + present = true + else + kept[#kept + 1] = k + end + end + if not present then + return false, 'not found' + end + + -- Temporary best-effort tombstone/truncate until the lower file layer + -- gives us an unlink_op(). + local okw, werr = fibers.perform(write_file_op(path_for(self.root, opts.key), '')) + if not okw then + return false, werr + end + + return fibers.perform(write_index_op(self.root, kept)) + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) + end) +end + +function Provider:list_op(opts) + return op.guard(function () + if opts ~= nil and type(opts) ~= 'table' then + return op.always(false, 'invalid options') + end + + local prefix = opts and opts.prefix or nil + if prefix ~= nil and type(prefix) ~= 'string' then + return op.always(false, 'invalid prefix') + end + + return read_index_op(self.root):wrap(function (ok, keys_or_err) + if not ok then + return false, keys_or_err + end + + if prefix == nil or prefix == '' then + return true, keys_or_err + end + + local out = {} + for _, k in ipairs(keys_or_err) do + if k:sub(1, #prefix) == prefix then + out[#out + 1] = k + end + end + return true, out + end) + end) +end + +---@param root string +---@param logger table|nil +---@return ControlStoreProvider +function M.new(root, logger) + assert(type(root) == 'string' and root ~= '', 'control_store_provider.new: invalid root') + return setmetatable({ + root = root, + logger = logger, + }, Provider) +end + +M.Provider = Provider +return M diff --git a/src/services/hal/drivers/cpu.lua b/src/services/hal/drivers/cpu.lua new file mode 100644 index 00000000..d99be6e2 --- /dev/null +++ b/src/services/hal/drivers/cpu.lua @@ -0,0 +1,343 @@ +-- services/hal/drivers/cpu.lua +-- +-- CPU HAL driver. +-- Exposes a single 'cpu' capability with a 'get' RPC offering. +-- Reads utilisation from /proc/stat (double-sample, 1-second gap) and +-- frequency from /sys/devices/system/cpu/cpuN/cpufreq/scaling_cur_freq. +-- CPU model and core count are read once from /proc/cpuinfo at driver +-- creation and included in meta. + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local file = require "fibers.io.file" +local channel = require "fibers.channel" + +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" +local cap_args = require "services.hal.types.capability_args" +local cache_mod = require "shared.cache" + +local perform = fibers.perform + +local CONTROL_Q_LEN = 8 +local FREQ_SYSFS_FMT = '/sys/devices/system/cpu/cpu%d/cpufreq/scaling_cur_freq' + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +---@class CpuDriver +---@field scope Scope +---@field control_ch Channel +---@field cap_emit_ch Channel? +---@field cache Cache +---@field model string +---@field core_count number +---@field logger Logger? +local CpuDriver = {} +CpuDriver.__index = CpuDriver + +---- helpers ---- + +---@param path string +---@return string? content +---@return string err +local function read_file(path) + local f, open_err = file.open(path, 'r') + if not f then + return nil, tostring(open_err) + end + local content, read_err = f:read_all() + f:close() + if not content then + return nil, tostring(read_err) + end + return content, "" +end + +--- Parse /proc/cpuinfo for the first model name and core count. +---@param logger Logger? +---@return string model +---@return number core_count +local function read_cpuinfo(logger) + local content, err = read_file('/proc/cpuinfo') + if not content then + dlog(logger, 'warn', { what = 'cpuinfo_read_failed', err = tostring(err) }) + return "", 1 + end + local model = content:match("model name%s*:%s*([^\n]+)") or "" + local count = 0 + for _ in content:gmatch("processor%s*:") do + count = count + 1 + end + return model:match("^%s*(.-)%s*$") or model, math.max(count, 1) +end + +--- Parse /proc/stat cpu lines into {user, nice, system, idle, ...} tables. +---@param content string +---@return table +local function parse_stat(content) + local result = {} + for line in content:gmatch("[^\n]+") do + local name, rest = line:match("^(cpu%w*)%s+(.+)$") + if name then + local fields = {} + for n in rest:gmatch("%d+") do + fields[#fields + 1] = tonumber(n) + end + result[name] = fields + end + end + return result +end + +--- Compute utilisation (%) from two successive /proc/stat snapshots. +---@param s1 table +---@param s2 table +---@return number util percentage 0-100 +local function compute_util(s1, s2) + local total1, total2, idle1, idle2 = 0, 0, (s1[4] or 0), (s2[4] or 0) + for _, v in ipairs(s1) do total1 = total1 + v end + for _, v in ipairs(s2) do total2 = total2 + v end + local delta_total = total2 - total1 + local delta_idle = idle2 - idle1 + if delta_total == 0 then return 0 end + return math.max(0, math.min(100, (1 - delta_idle / delta_total) * 100)) +end + +---- capability verbs ---- + +local VALID_FIELDS = { utilisation = true, core_utilisations = true, frequency = true, core_frequencies = true } + +---@param opts CpuGetOpts +---@return boolean ok +---@return any value_or_err +function CpuDriver:get(opts) + if opts == nil or getmetatable(opts) ~= cap_args.CpuGetOpts then + return false, "invalid opts" + end + local field = opts.field + local max_age = opts.max_age + + if not VALID_FIELDS[field] then + return false, "unsupported field: " .. tostring(field) + end + + local cached = self.cache:get(field, max_age) + if cached ~= nil then + return true, cached + end + + -- Utilisation pair: double-sample /proc/stat with 1-second sleep. + if field == 'utilisation' or field == 'core_utilisations' then + local s1_raw, err1 = read_file('/proc/stat') + if not s1_raw then + return false, "failed to read /proc/stat: " .. err1 + end + local s1 = parse_stat(s1_raw) + perform(sleep.sleep_op(1)) + local s2_raw, err2 = read_file('/proc/stat') + if not s2_raw then + return false, "failed to read /proc/stat (2nd sample): " .. err2 + end + local s2 = parse_stat(s2_raw) + + local overall = compute_util(s1['cpu'] or {}, s2['cpu'] or {}) + local per_core = {} + for i = 0, self.core_count - 1 do + local key = 'cpu' .. i + per_core[key] = compute_util(s1[key] or {}, s2[key] or {}) + end + + self.cache:set('utilisation', overall) + self.cache:set('core_utilisations', per_core) + if field == 'utilisation' then + return true, overall + end + return true, per_core + end + + -- Frequency pair: read scaling_cur_freq for each core. + if field == 'frequency' or field == 'core_frequencies' then + local per_core = {} + local total = 0 + local count = 0 + for i = 0, self.core_count - 1 do + local path = FREQ_SYSFS_FMT:format(i) + local raw, ferr = read_file(path) + if raw then + local khz = tonumber(raw:match("%d+")) + if khz then + local key = 'cpu' .. i + per_core[key] = khz + total = total + khz + count = count + 1 + end + else + dlog(self.logger, 'debug', { what = 'frequency_read_skipped', path = path, err = tostring(ferr) }) + end + end + local avg = (count > 0) and (total / count) or 0 + + self.cache:set('frequency', avg) + self.cache:set('core_frequencies', per_core) + if field == 'frequency' then + return true, avg + end + return true, per_core + end + + return false, "unreachable" +end + +---- control manager ---- + +function CpuDriver:control_manager() + fibers.current_scope():finally(function() + dlog(self.logger, 'debug', { what = 'control_manager_exiting' }) + end) + + while true do + local request, req_err = self.control_ch:get() + if not request then + dlog(self.logger, 'debug', { what = 'control_ch_closed', err = tostring(req_err) }) + break + end + ---@cast request ControlRequest + + local fn = self[request.verb] + local ok, value_or_err + if type(fn) ~= 'function' then + ok, value_or_err = false, "unsupported verb: " .. tostring(request.verb) + else + local st, _, r1, r2 = fibers.run_scope(function() + return fn(self, request.opts) + end) + if st ~= 'ok' then + ok, value_or_err = false, "internal error: " .. tostring(r1) + else + ok, value_or_err = r1, r2 + end + end + + local reply = hal_types.new.Reply(ok, value_or_err) + if reply then + request.reply_ch:put(reply) + end + end +end + +---- emit helpers ---- + +---@param mode EmitMode +---@param key string +---@param data any +function CpuDriver:_emit(mode, key, data) + if not self.cap_emit_ch then return end + local payload, err = hal_types.new.Emit('cpu', '1', mode, key, data) + if not payload then + dlog(self.logger, 'debug', { what = 'emit_failed', key = key, err = tostring(err) }) + return + end + self.cap_emit_ch:put(payload) +end + +---- public interface ---- + +---@return string error +function CpuDriver:init() + if self.initialised then + return "already initialised" + end + self.initialised = true + return "" +end + +---@param emit_ch Channel +---@return Capability[] +---@return string error +function CpuDriver:capabilities(emit_ch) + if not self.initialised then + return {}, "cpu driver not initialised" + end + self.cap_emit_ch = emit_ch + local cap, err = cap_types.new.CpuCapability('1', self.control_ch) + if not cap then + return {}, err + end + return { cap }, "" +end + +---@return boolean ok +---@return string error +function CpuDriver:start() + if not self.initialised then + return false, "cpu driver not initialised" + end + self:_emit('meta', 'info', { + provider = 'hal', + version = 1, + model = self.model, + core_count = self.core_count, + }) + + local ok, err = self.scope:spawn(function() + self:control_manager() + end) + if not ok then + return false, "failed to spawn control_manager: " .. tostring(err) + end + return true, "" +end + +---@param timeout number? +---@return boolean ok +---@return string error +function CpuDriver:stop(timeout) + timeout = timeout or 5 + self.scope:cancel('cpu driver stopped') + local source = perform(op.named_choice { + join = self.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + if source == 'timeout' then + return false, "cpu driver stop timeout" + end + return true, "" +end + +---@param logger Logger? +---@return CpuDriver? +---@return string error +local function new(logger) + local scope, err = fibers.current_scope():child() + if not scope then + return nil, "failed to create child scope: " .. tostring(err) + end + + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + dlog(logger, 'error', { what = 'scope_failed', err = tostring(primary), status = st }) + end + dlog(logger, 'debug', { what = 'stopped' }) + end) + + local model, core_count = read_cpuinfo(logger) + + return setmetatable({ + scope = scope, + control_ch = channel.new(CONTROL_Q_LEN), + cap_emit_ch = nil, + cache = cache_mod.new(), + model = model, + core_count = core_count, + logger = logger, + initialised = false, + }, CpuDriver), "" +end + +return { new = new } diff --git a/src/services/hal/drivers/filesystem.lua b/src/services/hal/drivers/filesystem.lua new file mode 100644 index 00000000..05ddda7b --- /dev/null +++ b/src/services/hal/drivers/filesystem.lua @@ -0,0 +1,399 @@ +-- HAL modules +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" +local cap_args = require "services.hal.types.capability_args" + +-- Fibers modules +local fibers = require "fibers" +local sleep = require "fibers.sleep" +local op = require "fibers.op" +local channel = require "fibers.channel" +local file = require "fibers.io.file" + +---@class FSDriver +---@field scope Scope +---@field roots table +---@field control_chs table +---@field cap_emit_ch Channel +---@field logger table? +---@field initialised boolean +---@field caps_applied boolean +local FSDriver = {} +FSDriver.__index = FSDriver + +---- Constant Definitions ---- + +local DEFAULT_STOP_TIMEOUT = 5 +local CONTROL_Q_LEN = 8 + +local function dlog(self, level, payload) + if self.logger and self.logger[level] then + self.logger[level](self.logger, payload) + end +end + +---- Utility Functions ---- + +--- Return a ControlError +---@param err string? +---@param code integer? +---@return boolean ok +---@return string reason +---@return integer? code +local function return_error(err, code) + if err == nil then + err = "unknown error" + end + return false, err, code +end + +--- Emit from the filesystem capability +---@param emit_ch Channel +---@param root_name string +---@param mode EmitMode +---@param key string +---@param data any +---@return boolean ok +---@return string? error +local function emit(emit_ch, root_name, mode, key, data) + local payload, err = hal_types.new.Emit( + 'fs', + root_name, + mode, + key, + data + ) + if not payload then + return false, err + end + emit_ch:put(payload) + return true +end + +---- Filesystem Capabilities ---- + +--- Read a file from a root +---@param root_name string +---@param opts FilesystemReadOpts? +---@return boolean ok +---@return string reason_or_content +---@return integer? code +function FSDriver:read(root_name, opts) + if opts == nil or getmetatable(opts) ~= cap_args.FilesystemReadOpts then + return return_error("invalid options", 1) + end + + local filename = opts.filename + if filename == nil then + return return_error("missing filename", 1) + end + + local root_path = self.roots[root_name] + if not root_path then + return return_error("unknown root: " .. tostring(root_name), 1) + end + + local full_path = root_path .. "/" .. filename + + -- Open file using fibers stream + local f, open_err = file.open(full_path, "r") + if not f then + return return_error("failed to open file " .. full_path .. ": " .. tostring(open_err), 1) + end + + local content, read_err = f:read_all() + local _, close_err = f:close() + + if not content then + return return_error("failed to read file " .. full_path .. ": " .. tostring(read_err), 1) + end + + if close_err then + dlog(self, 'warn', { + what = 'read_close_warning', + root = root_name, + filename = filename, + err = tostring(close_err), + }) + end + + -- Emit success event + local ok, emit_err = emit(self.cap_emit_ch, root_name, 'event', 'read_success', { + filename = filename + }) + if not ok then + dlog(self, 'warn', { + what = 'read_success_emit_failed', + root = root_name, + filename = filename, + err = tostring(emit_err), + }) + end + + return true, content +end + +--- Write a file to a root +---@param root_name string +---@param opts FilesystemWriteOpts? +---@return boolean ok +---@return string? reason +---@return integer? code +function FSDriver:write(root_name, opts) + if opts == nil or getmetatable(opts) ~= cap_args.FilesystemWriteOpts then + return return_error("invalid options", 1) + end + + local filename = opts.filename + if filename == nil then + return return_error("missing filename", 1) + end + + local root_path = self.roots[root_name] + if not root_path then + return return_error("unknown root: " .. tostring(root_name), 1) + end + + local full_path = root_path .. "/" .. filename + + -- Open file using fibers stream + local f, open_err = file.open(full_path, "w") + if not f then + return return_error("failed to open file for writing: " .. tostring(open_err), 1) + end + + local ok, write_err = f:write(opts.data) + local _, close_err = f:close() + + if not ok then + return return_error("failed to write file: " .. tostring(write_err), 1) + end + + if close_err then + dlog(self, 'warn', { + what = 'write_close_warning', + root = root_name, + filename = filename, + err = tostring(close_err), + }) + end + + -- Emit success event + local ok_emit, emit_err = emit(self.cap_emit_ch, root_name, 'event', 'write_success', { + filename = filename + }) + if not ok_emit then + dlog(self, 'warn', { + what = 'write_success_emit_failed', + root = root_name, + filename = filename, + err = tostring(emit_err), + }) + end + + return true +end + +--- Validate that a function is implemented +---@param fn any +---@return boolean is_valid +---@return string? error +local function validate_fn(fn) + if fn == nil then + return false, tostring(fn) .. " is unimplemented" + end + if type(fn) ~= "function" then + return false, tostring(fn) .. " is not a function" + end + return true +end + +---- Long Running Fibers ---- + +function FSDriver:control_manager() + if self.cap_emit_ch == nil then + dlog(self, 'error', { what = 'control_manager_missing_cap_emit_channel' }) + return + end + + dlog(self, 'debug', { what = 'control_manager_started' }) + + fibers.current_scope():finally(function() + dlog(self, 'debug', { what = 'control_manager_exiting' }) + end) + + while true do + -- Build named choice over all root control channels + local choice_arms = {} + for root_name, control_ch in pairs(self.control_chs) do + choice_arms[root_name] = control_ch:get_op() + end + + -- Wait for a request from any root's control channel + local root_name, request, req_err = fibers.perform(op.named_choice(choice_arms)) + + if not request then + dlog(self, 'error', { what = 'control_channel_get_failed', err = tostring(req_err) }) + break + end + + ---@cast request ControlRequest + + local ok, reason, code + + local fn = self[request.verb] + local valid, validation_err = validate_fn(fn) + if not valid then + ok = false + reason = "no function exists for verb: " .. tostring(validation_err) + else + local call_ok, fn_ok, fn_reason, fn_code = pcall(fn, self, root_name, request.opts) + if not call_ok then + ok = false + reason = "internal error: " .. tostring(fn_ok) + code = 1 + else + ok = fn_ok + reason = fn_reason + code = fn_code + end + end + + local reply, reply_err = hal_types.new.Reply(ok, reason, code) + if not reply then + dlog(self, 'error', { what = 'reply_create_failed', err = tostring(reply_err) }) + else + request.reply_ch:put(reply) + end + end +end + +---- Driver Functions ---- + +--- Spawn filesystem driver services +---@return boolean ok +---@return string error +function FSDriver:start() + if not self.initialised then + return false, "filesystem not initialised" + end + if not self.caps_applied then + return false, "capabilities not applied" + end + + self.scope:spawn(function() self:control_manager() end) + + return true, "" +end + +--- Closes down the filesystem driver +---@param timeout number? Timeout in seconds +---@return boolean ok +---@return string error +function FSDriver:stop(timeout) + timeout = timeout or DEFAULT_STOP_TIMEOUT + self.scope:cancel() + + local source = fibers.perform(op.named_choice { + join = self.scope:join_op(), + timeout = sleep.sleep_op(timeout) + }) + + if source == "timeout" then + return false, "filesystem stop timeout" + end + return true, "" +end + +--- Apply capabilities to HAL and start monitoring state +--- Filesystem must be initialised first +---@param emit_ch Channel +---@return Capability[]? capabilities +---@return string error +function FSDriver:capabilities(emit_ch) + if not self.initialised then + return nil, "filesystem not initialised" + end + if self.caps_applied then + return nil, "capabilities already applied" + end + + self.cap_emit_ch = emit_ch + + local caps = {} + + for cap_name, _ in pairs(self.roots) do + local control_ch = self.control_chs[cap_name] + local cap, cap_err = cap_types.new.FilesystemCapability(cap_name, control_ch) + if not cap then + return nil, "failed to create capability for root " .. cap_name .. ": " .. tostring(cap_err) + end + + table.insert(caps, cap) + end + + self.caps_applied = true + + return caps, "" +end + +--- Initialize the filesystem driver +--- Creates missing root directories and validates configuration +---@return string error +function FSDriver:init() + if self.initialised then + return "already initialised" + end + + -- Validate roots + if type(self.roots) ~= 'table' or next(self.roots) == nil then + return "roots must be a non-empty table" + end + + -- Create control channels for each root and create missing directories + for root_name, root_path in pairs(self.roots) do + if type(root_path) ~= 'string' or root_path == '' then + return "root path for " .. tostring(root_name) .. " must be a non-empty string" + end + + -- Create control channel for this root + self.control_chs[root_name] = channel.new(CONTROL_Q_LEN) + + -- Create missing root directory + local ok, mkdir_err = file.mkdir_p(root_path) + if not ok then + local err_msg = "failed to create root directory " .. tostring(root_name) .. + " at " .. tostring(root_path) .. ": " .. tostring(mkdir_err) + return err_msg + end + + dlog(self, 'debug', { what = 'root_ready', root = root_name, path = root_path }) + end + + self.initialised = true + return "" +end + +--- Create a new filesystem driver +---@param roots table A mapping of root names to their paths +---@param logger table? +---@return FSDriver? +---@return string error +local function new(roots, logger) + local self = setmetatable({}, FSDriver) + local scope, sc_err = fibers.current_scope():child() + if not scope then return nil, sc_err end + + self.scope = scope + self.roots = roots or {} + self.control_chs = {} + self.logger = logger + self.initialised = false + self.caps_applied = false + + return self, "" +end + +return { + new = new, +} diff --git a/src/services/hal/drivers/memory.lua b/src/services/hal/drivers/memory.lua new file mode 100644 index 00000000..59a8f049 --- /dev/null +++ b/src/services/hal/drivers/memory.lua @@ -0,0 +1,249 @@ +-- services/hal/drivers/memory.lua +-- +-- Memory HAL driver. +-- Exposes a single 'memory' capability with a 'get' RPC offering. +-- Reads from /proc/meminfo on each cache miss, computing all four +-- fields at once (total, used, free, util). + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local file = require "fibers.io.file" +local channel = require "fibers.channel" + +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" +local cap_args = require "services.hal.types.capability_args" +local cache_mod = require "shared.cache" + +local perform = fibers.perform + +local CONTROL_Q_LEN = 8 + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +---@class MemoryDriver +---@field scope Scope +---@field control_ch Channel +---@field cap_emit_ch Channel? +---@field cache Cache +---@field logger Logger? +local MemoryDriver = {} +MemoryDriver.__index = MemoryDriver + +---- helpers ---- + +---@param path string +---@return string? content +---@return string error +local function read_file(path) + local f, open_err = file.open(path, 'r') + if not f then + return nil, tostring(open_err) + end + local content, read_err = f:read_all() + f:close() + if not content then + return nil, tostring(read_err) + end + return content, "" +end + +--- Parse /proc/meminfo and compute total, free, used, util. +---@return table? result +---@return string error +local function read_meminfo() + local content, err = read_file('/proc/meminfo') + if not content then + return nil, "failed to read /proc/meminfo: " .. err + end + + local function kv(key) + local v = content:match(key .. "%s*:%s*(%d+)") + return v and tonumber(v) or 0 + end + + local mem_total = kv("MemTotal") + local mem_free = kv("MemFree") + local buffers = kv("Buffers") + local cached = kv("Cached") + + local effective_free = mem_free + buffers + cached + local used = mem_total - effective_free + local util = (mem_total > 0) and (used / mem_total * 100) or 0 + + return { total = mem_total, used = used, free = effective_free, util = util }, "" +end + +---- capability verbs ---- + +local VALID_FIELDS = { total = true, used = true, free = true, util = true } + +---@param opts MemoryGetOpts +---@return boolean ok +---@return any value_or_err +function MemoryDriver:get(opts) + if opts == nil or getmetatable(opts) ~= cap_args.MemoryGetOpts then + return false, "invalid opts" + end + local field = opts.field + local max_age = opts.max_age + + if not VALID_FIELDS[field] then + return false, "unsupported field: " .. tostring(field) + end + + local cached = self.cache:get(field, max_age) + if cached ~= nil then + return true, cached + end + + local info, err = read_meminfo() + if not info then + return false, err + end + + -- Populate all four cache entries at once. + self.cache:set('total', info.total) + self.cache:set('used', info.used) + self.cache:set('free', info.free) + self.cache:set('util', info.util) + + return true, info[field] +end + +---- control manager ---- + +function MemoryDriver:control_manager() + fibers.current_scope():finally(function() + dlog(self.logger, 'debug', { what = 'control_manager_exiting' }) + end) + + while true do + local request, req_err = self.control_ch:get() + if not request then + dlog(self.logger, 'debug', { what = 'control_ch_closed', err = tostring(req_err) }) + break + end + + local fn = self[request.verb] + local ok, value_or_err + if type(fn) ~= 'function' then + ok, value_or_err = false, "unsupported verb: " .. tostring(request.verb) + else + local st, _, r1, r2 = fibers.run_scope(function() + return fn(self, request.opts) + end) + if st ~= 'ok' then + ok, value_or_err = false, "internal error: " .. tostring(r1) + else + ok, value_or_err = r1, r2 + end + end + + local reply = hal_types.new.Reply(ok, value_or_err) + if reply then + request.reply_ch:put(reply) + end + end +end + +---- public interface ---- + +---@return string error +function MemoryDriver:init() + if self.initialised then + return "already initialised" + end + self.initialised = true + return "" +end + +---@param emit_ch Channel +---@return Capability[]? +---@return string error +function MemoryDriver:capabilities(emit_ch) + if not self.initialised then + return nil, "memory driver not initialised" + end + self.cap_emit_ch = emit_ch + local cap, err = cap_types.new.MemoryCapability('1', self.control_ch) + if not cap then + return {}, err + end + return { cap }, "" +end + +---@return boolean ok +---@return string error +function MemoryDriver:start() + if not self.initialised then + return false, "memory driver not initialised" + end + local payload, err = hal_types.new.Emit('memory', '1', 'meta', 'info', { + provider = 'hal', + version = 1, + }) + if payload and self.cap_emit_ch then + self.cap_emit_ch:put(payload) + elseif not payload then + dlog(self.logger, 'debug', { what = 'meta_emit_failed', err = tostring(err) }) + end + + local ok, spawn_err = self.scope:spawn(function() + self:control_manager() + end) + if not ok then + return false, "failed to spawn control_manager: " .. tostring(spawn_err) + end + return true, "" +end + +---@param timeout number? +---@return boolean ok +---@return string error +function MemoryDriver:stop(timeout) + timeout = timeout or 5 + self.scope:cancel('memory driver stopped') + local source = perform(op.named_choice { + join = self.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + if source == 'timeout' then + return false, "memory driver stop timeout" + end + return true, "" +end + +---@param logger Logger? +---@return MemoryDriver? +---@return string error +local function new(logger) + local scope, err = fibers.current_scope():child() + if not scope then + return nil, "failed to create child scope: " .. tostring(err) + end + + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + dlog(logger, 'error', { what = 'scope_failed', err = tostring(primary), status = st }) + end + dlog(logger, 'debug', { what = 'stopped' }) + end) + + return setmetatable({ + scope = scope, + control_ch = channel.new(CONTROL_Q_LEN), + cap_emit_ch = nil, + cache = cache_mod.new(), + logger = logger, + initialised = false, + }, MemoryDriver), "" +end + +return { new = new } diff --git a/src/services/hal/drivers/modem.lua b/src/services/hal/drivers/modem.lua index c42bf11e..5c2db0ef 100644 --- a/src/services/hal/drivers/modem.lua +++ b/src/services/hal/drivers/modem.lua @@ -1,808 +1,884 @@ --- driver.lua -local fiber = require "fibers.fiber" +-- Modem modules +local modem_backend_provider = require "services.hal.backends.modem.provider" + +-- HAL modules +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" +local capability_args = require "services.hal.types.capability_args" +local cache_mod = require "shared.cache" + +-- Service modules +-- (logger injected via new()) + +-- Fibers modules +local fibers = require "fibers" local op = require "fibers.op" -local queue = require "fibers.queue" -local context = require "fibers.context" local channel = require "fibers.channel" local sleep = require "fibers.sleep" -local sc = require "fibers.utils.syscall" -local service = require "service" -local at = require "services.hal.drivers.modem.at" -local mmcli = require "services.hal.drivers.modem.mmcli" -local utils = require "services.hal.utils" -local hal_capabilities = require "services.hal.hal_capabilities" -local mode_overrides = require "services.hal.drivers.modem.mode" -local model_overrides = require "services.hal.drivers.modem.model" -local json = require "cjson.safe" -local log = require "services.log" -local wraperr = require "wraperr" - -local unpack = table.unpack or unpack -local CMD_TIMEOUT = 3 - ----@class Driver ----@field ctx Context ----@field address string ----@field command_q Queue ----@field refresh_rate_channel Channel -local Driver = {} -Driver.__index = Driver - -local model_info = { - quectel = { - -- these are ordered, as eg25gl should match before eg25g - { mod_string = "UNKNOWN", rev_string = "eg25gl", model = "eg25", model_variant = "gl" }, - { mod_string = "UNKNOWN", rev_string = "eg25g", model = "eg25", model_variant = "g" }, - { mod_string = "UNKNOWN", rev_string = "ec25e", model = "ec25", model_variant = "e" }, - { mod_string = "em06-e", rev_string = "em06e", model = "em06", model_variant = "e" }, - { mod_string = "rm520n-gl", rev_string = "rm520ngl", model = "rm520n", model_variant = "gl" } - -- more quectel models here +local cond = require "fibers.cond" +local pulse = require "fibers.pulse" + +---@class Modem +---@field address ModemAddress +---@field cache Cache +---@field control_ch Channel +---@field cap_emit_ch Channel +---@field scope Scope +---@field model string +---@field model_variant string +---@field mode string +---@field initialised boolean +---@field caps_applied boolean +---@field state_pulse Pulse +---@field sim_inserted_pulse Pulse +---@field sim_state_ch Channel +---@field log Logger +local Modem = {} +Modem.__index = Modem + +local function list_to_map(list) + local map = {} + for _, value in ipairs(list) do + map[value] = true + end + return map +end + +---- Constant Definitions ---- +local D_LOG_EMITTER = false + +local DEFAULT_STOP_TIMEOUT = 5 +local LISTEN_TRIGGER_INTERVAL = 1 + +local CONTROL_Q_LEN = 8 + +local GROUP_FIELDS = { + identity = { + "imei", + "drivers", + "plugin", + "model", + "revision", + "firmware", + }, + ports = { + "device", + "primary_port", + "at_ports", + "qmi_ports", + "net_ports", + }, + sim = { + "sim", + "iccid", + "imsi", + "gid1", + "sim_lock", + "sim_lock_retries", + "modem_state", + }, + network = { + "access_techs", + "operator", + "mcc", + "mnc", + "active_band_class", + }, + signal = { + "signal", + }, + traffic = { + "rx_bytes", + "tx_bytes", }, - fibocom = {} } -local function array_to_table(arr) - local t = {} - for _, v in ipairs(arr) do - t[v] = true +local FIELD_TO_GROUP = {} +for group_name, fields in pairs(GROUP_FIELDS) do + for _, field in ipairs(fields) do + FIELD_TO_GROUP[field] = group_name end - return t end -local function get_ports(ports) - local port_list = {} +local GROUP_FETCHERS = { + identity = "read_identity", + ports = "read_ports", + sim = "read_sim_info", + network = "read_network_info", + signal = "read_signal", + traffic = "read_traffic", +} - -- ports is now a comma-separated string - if type(ports) == "string" then - for port in ports:gmatch("[^,]+") do - port = port:match("^%s*(.-)%s*$") -- trim whitespace - local port_name, port_type = string.match(port, "^([%w%-]+)%s*%(([%w%-]+)%)") - if port_name and port_type then - if port_list[port_type] == nil then - port_list[port_type] = { port_name } - else - table.insert(port_list[port_type], port_name) - end - end - end - end +local VALID_GROUPS = list_to_map { + "identity", + "ports", + "sim", + "network", + "signal", + "traffic", +} - return port_list -end - ----returns a list of control and info capabilities for the modem ----@return table -function Driver:apply_capabilities(capability_info_q) - self.info_q = capability_info_q - local capabilities = {} - capabilities.modem = { - control = hal_capabilities.new_modem_capability(self.command_q), - id = self.imei - } - return capabilities -end - ----continuously polls the modem for modem and sim information ----omg I hate this, when mvp is done and theres no major deadline this is the first thing to go -function Driver:poll_info() - local poll_freq = 10 - local poll_ctx = context.with_timeout(self.ctx, poll_freq) - local send_ctx = context.with_cancel(self.ctx) - while not self.ctx:err() do - local infos = {} - local modem_info, modem_err = self:get_modem_info() - if modem_err then - log.error(string.format("Modem - %s: Failed to get modem info: %s", self.imei, modem_err)) - else - infos.modem = modem_info - end - local band_info, band_err = self.nas_get_rf_band_info() - if not band_err then - infos.band = band_info - end +---- Modem Utility Functions ---- + +--- Emit from the modem capability +---@param emit_ch Channel +---@param imei string +---@param mode EmitMode +---@param key string +---@param data any +---@return boolean ok +---@return string? error +local function emit(emit_ch, imei, mode, key, data) + local payload, err = hal_types.new.Emit( + 'modem', + imei, + mode, + key, + data + ) + if not payload then + return false, err + end + emit_ch:put(payload) + return true +end - if infos.modem and infos.modem.generic.sim ~= '--' then - local sim_info, sim_err = self:get_sim_info(infos.modem.generic.sim) - if sim_err then - log.debug(string.format("Sim - %s: Failed to get sim info: %s", - self.imei, - sim_err)) - else - infos.sim = sim_info - end - local signal, signal_err = self:get_signal() - if not signal_err then - infos.modem.signal = signal - end - end +--- Utility function to return a ControlError +---@param err string? +---@param code integer? +---@return boolean ok +---@return string reason +---@return integer? code +local function return_error(err, code) + if err == nil then + err = "unknown error" + end + return false, err, code +end - if infos.modem and infos.modem["3gpp"]["registration-state"] ~= '--' then - local nas_info, nas_err = self.get_nas_info() - if nas_err then - log.debug("MCC MNC failed retrieval", nas_err) - else - infos.nas = nas_info - end - end +--- Emit an event. +---@param key string +---@param data any +---@return boolean ok +---@return string? error +function Modem:_emit_event(key, data) + return emit(self.cap_emit_ch, self.imei, 'event', key, data) +end - if infos.modem and infos.modem.generic.state ~= 'failed' then - local gids, gid_err = self.uim_get_gids() - if gid_err then - log.debug(gid_err) - else - infos.gids = gids - end - end +--- Emit a state. +---@param key string +---@param data any +---@return boolean ok +---@return string? error +function Modem:_emit_state(key, data) + return emit(self.cap_emit_ch, self.imei, 'state', key, data) +end - if self.at_port then - local response, fw_err = at.send_with_context( - context.with_timeout(self.ctx, 10), - self.at_port, - "AT+QGMR" - ) - if not fw_err then - for _, line in ipairs(response or {}) do - local firmware_version = string.match(line, "([%w]+_[%w]+%.[%w]+%.[%w]+%.[%w]+)") - if firmware_version then - infos.modem = infos.modem or {} - infos.modem.firmware = firmware_version - break - end - end - end - end +--- Emit meta information. +---@param key string +---@param data any +---@return boolean ok +---@return string? error +function Modem:_emit_meta(key, data) + return emit(self.cap_emit_ch, self.imei, 'meta', key, data) +end - if infos.modem and infos.modem.generic then - local ports = get_ports(infos.modem.generic.ports or {}) - local net_port = ports.net and ports.net[1] or nil - if net_port then - local path = string.format('/sys/class/net/%s/statistics/', net_port) - local rx_bytes = utils.read_file(path .. 'rx_bytes') - local tx_bytes = utils.read_file(path .. 'tx_bytes') - infos.modem.net = { - rx_bytes = tonumber(rx_bytes) or 0, - tx_bytes = tonumber(tx_bytes) or 0 - } - end - end +--- Validate that a function is implemented +---@param fn any +---@param verb string +---@return boolean is_valid +---@return string? error +local function validate_fn(fn, verb) + if fn == nil then + return false, tostring(verb) .. " is unimplemented" + end + if type(fn) ~= "function" then + return false, tostring(verb) .. " is not a function" + end + return true +end - send_ctx:cancel('new infos available') - send_ctx = context.with_cancel(self.ctx) - if next(infos) ~= nil then - fiber.spawn(function() - op.choice( - self.info_q:put_op({ - type = "modem", - id = self.imei, - sub_topic = {}, - endpoints = "multiple", - info = infos - }), - send_ctx:done_op() - ):perform() - end) - end - local poll_freq_update = op.choice( - poll_ctx:done_op(), - self.refresh_rate_channel:get_op() - ):perform() - if poll_freq_update then poll_freq = poll_freq_update end +--- Trim the traceback from an error message +---@param err string +---@return string trimmed_error +local function trim_error(err) + local traceback_start = err:find("\nstack traceback:") + if traceback_start then + return err:sub(1, traceback_start - 1) + end + return err +end - -- Make sure to not loop until we have hit our deadline - poll_ctx:done_op():perform() - poll_ctx = context.with_timeout(self.ctx, poll_freq) +---@param snapshot any +---@param field string +---@return any +local function extract_group_field(snapshot, field) + if field == "signal" then + return snapshot.values end + return snapshot[field] +end - log.trace(string.format("Modem - %s: Polling info stopped", self.imei)) +---@param group string +---@return boolean +local function group_valid(group) + return VALID_GROUPS[group] == true end -local function is_array(tbl) - if type(tbl) ~= "table" then - return false - end - local count = 0 - for k, _ in pairs(tbl) do - count = count + 1 - if type(k) ~= "number" or k < 1 or k > count then - return false - end - end - return count > 0 +---@param group string +---@param value any +function Modem:_cache_group(group, value) + self.cache:set(group, value) end -local function format_arrays(tbl) - if type(tbl) ~= "table" then - return tbl +---@param group string +function Modem:_invalidate_group(group) + if not group_valid(group) then return end + if self.cache and type(self.cache.delete) == 'function' then + self.cache:delete(group) + elseif self.cache and self.cache.store then + self.cache.store[group] = nil end +end - -- Check if this table is an array - if is_array(tbl) then - -- Convert array to comma-separated string - local str_parts = {} - for i = 1, #tbl do - table.insert(str_parts, tostring(tbl[i])) - end - return table.concat(str_parts, ",") - end - - -- Not an array, so recursively process nested tables - local result = {} - for k, v in pairs(tbl) do - result[k] = format_arrays(v) - end - return result -end - ----Reads mmcli modem output into a table structure ----@return table? ----@return table? error -function Driver:get_modem_info() - local new_ctx = context.with_timeout(self.ctx, CMD_TIMEOUT) - local cmd = mmcli.information(new_ctx, self.address) - local out, err = cmd:combined_output() - if err then return nil, wraperr.new(err) end - - local info, _, err = json.decode(out) - if err then return nil, wraperr.new(err) end - - -- format any array fields to be a string of format [val1,val2,...] - return format_arrays(info.modem), nil -end - ----Reads mmcli sim output into a table structure ----@param sim_address string ----@return table? ----@return table? error -function Driver:get_sim_info(sim_address) - local new_ctx = context.with_timeout(self.ctx, CMD_TIMEOUT) - local cmd = mmcli.sim_information(new_ctx, sim_address) - local out, err = cmd:combined_output() - if err then return nil, wraperr.new(err) end - - local info, _, err = json.decode(out) - if err then return nil, wraperr.new(err) end - - return info.sim, nil -end - -local SIGNAL_TECHNOLOGIES = array_to_table({ - '5g', - 'cdma1x', - 'evdo', - 'gsm', - 'lte', - 'umts' -}) -local IGNORE_FIELDS = array_to_table({ - 'error-rate' -}) - -function Driver:get_signal() - local cmd = mmcli.signal_get(context.with_timeout(self.ctx, CMD_TIMEOUT), self.address) - - local out, err = cmd:combined_output() - if err then return nil, wraperr.new(err) end - - local info, _, err = json.decode(out) - if err then return nil, wraperr.new(err) end - - for signal_tech, signals in pairs(info.modem.signal) do - if SIGNAL_TECHNOLOGIES[signal_tech] then - local valid_signal = false - local filtered_signals = {} - for signal_name, signal_value in pairs(signals) do - if not IGNORE_FIELDS[signal_name] and signal_value ~= '--' then - filtered_signals[signal_name] = signal_value - valid_signal = true - end - end - if valid_signal then - return filtered_signals, nil - end - end +---@param groups string[] +function Modem:invalidate_groups(groups) + for _, group in ipairs(groups or {}) do + self:_invalidate_group(group) end - return nil, wraperr.new("No valid signals") end ----Gets initial modem information and binds protocol specific functions to the driver -function Driver:init() - local info, err = self:get_modem_info() - if info == nil or err then return err end +function Modem:_invalidate_dynamic() + self:invalidate_groups { 'sim', 'network', 'signal', 'traffic' } +end + +---@param group string +---@param timescale number? +---@return any snapshot +---@return string error +function Modem:_get_group(group, timescale) + if not group_valid(group) then + return nil, "unsupported group: " .. tostring(group) + end - -- let's get the driver mode - local drivers = info.generic.drivers + local cached = self.cache:get(group, timescale) + if cached ~= nil then + return cached, "" + end - -- drivers is a comma-separated string - if drivers:match("qmi_wwan") then - self.mode = "qmi" - elseif drivers:match("cdc_mbim") then - self.mode = "mbim" + local fetcher_name = GROUP_FETCHERS[group] + local fetcher = self.backend and self.backend[fetcher_name] or nil + if type(fetcher) ~= "function" then + return nil, "group " .. tostring(group) .. " is not implemented by backend" end - -- -- now let's enrich the driver with mode specific functions/overrides - assert(mode_overrides.add_mode_funcs(self)) + local snapshot, err = fetcher(self.backend) + if err ~= "" then + return nil, err + end - -- let's now determine the manufacturer, model and variant - local plugin = info.generic.plugin + self:_cache_group(group, snapshot) + return snapshot, "" +end - local model = info.generic.model +--- Checks if an error string indicates a closed command +---@param err string +---@return boolean is_closed +local function is_command_closed(err) + return err == "Command closed" or err == "Stream closed" +end - local revision = info.generic.revision +---- Modem Capabilities ---- - for man, mods in pairs(model_info) do - if string.match(plugin:lower(), man) then - self.manufacturer = man - for _, details in ipairs(mods) do - if details.mod_string == model:lower() or utils.starts_with(revision, details.rev_string) then - log.info(man, details.model, details.model_variant, "detected at:", self.address) - self.model = details.model - self.model_variant = details.model_variant - end - end - break - end +--- Get a modem attribute. +---@param opts ModemGetOpts? +---@return boolean ok +---@return any reason_or_value +---@return integer? code +function Modem:get(opts) + if opts == nil or getmetatable(opts) ~= capability_args.ModemGetOpts then + return return_error("invalid options", 1) end + local field = opts.field + local timescale = opts.timescale - self.imei = info.generic['equipment-identifier'] - self.device = info.generic.device + -- Check that the field is supported + local group = FIELD_TO_GROUP[field] + if not group then + return return_error("unsupported field: " .. tostring(field), 1) + end - self.primary_port = string.format('/dev/%s', info.generic["primary-port"]) - local ports = get_ports(info.generic.ports or {}) - self.at_port = ports.at and ports.at[1] and string.format("/dev/%s", ports.at[1]) or nil - if self.at_port == nil then - log.warn( - string.format("%s - %s: Could not find at port", - self.ctx:value("service_name"), - self.ctx:value("fiber_name") - ) - ) + local snapshot, err = self:_get_group(group, timescale) + if err ~= "" then + return return_error("error getting field " .. tostring(field) .. ": " .. tostring(err), 1) end - -- -- we add any make/model specific functions/overrides - model_overrides.add_model_funcs(self) -end ----Starts modem information, monitor and command manager fibers ----@param bus_conn Connection -function Driver:spawn(bus_conn) - service.spawn_fiber('Modem Info Poll - ' .. self.imei, bus_conn, self.ctx, function() - self:poll_info() - end) - service.spawn_fiber('Modem State Monitor - ' .. self.imei, bus_conn, self.ctx, function(monitor_ctx) - self:state_monitor(monitor_ctx) - end) - service.spawn_fiber('Modem Command Manager - ' .. self.imei, bus_conn, self.ctx, function() - self:command_manager() - end) -end + local value = extract_group_field(snapshot, field) + if value == nil then + return return_error("field unavailable: " .. tostring(field), 1) + end --- Base methods can be defined here -function Driver.set_power_low(ctx) - local cmd_ctx = context.with_timeout(ctx, 0.3) - return at.send_with_context(cmd_ctx, "AT+CFUN=0") + return true, value end -function Driver.set_power_high(ctx) - local cmd_ctx = context.with_timeout(ctx, 0.3) - return at.send_with_context(cmd_ctx, "AT+CFUN=1") +--- Enable the modem +---@return boolean ok +---@return string? reason +---@return integer? code +function Modem:enable() + local ok, err = self.backend:enable() + if not ok then + return return_error(err, 1) + end + self:_invalidate_dynamic() + return true end -function Driver:set_func_flight() - local cmd_ctx = context.with_timeout(self.ctx, 0.3) - return at.send_with_context(cmd_ctx, "AT+CFUN=4") +--- Disable the modem +---@return boolean ok +---@return string? reason +---@return integer? code +function Modem:disable() + local ok, err = self.backend:disable() + if not ok then + return return_error(err, 1) + end + self:_invalidate_dynamic() + return true end -function Driver:disable() - local cmd_ctx = context.with_timeout(self.ctx, CMD_TIMEOUT) - local cmd = mmcli.disable(cmd_ctx, self.address) - return cmd:run() +--- Reset the modem +---@return boolean ok +---@return string? reason +---@return integer? code +function Modem:reset() + local ok, err = self.backend:reset() + if not ok then + return return_error(err, 1) + end + self:_invalidate_dynamic() + return true end -function Driver:enable() - local cmd_ctx = context.with_timeout(self.ctx, CMD_TIMEOUT) - local cmd = mmcli.enable(cmd_ctx, self.address) - return cmd:run() +--- Connect the modem +---@param opts ModemConnectOpts? +---@return boolean ok +---@return string? reason +---@return integer? code +function Modem:connect(opts) + if opts == nil or getmetatable(opts) ~= capability_args.ModemConnectOpts then + return return_error("invalid options", 1) + end + local ok, err = self.backend:connect(opts.connection_string) + if not ok then + return return_error(err, 1) + end + self:invalidate_groups { 'network', 'signal', 'traffic' } + return true end -function Driver:reset() - local cmd_ctx = context.with_timeout(self.ctx, CMD_TIMEOUT) - local cmd = mmcli.reset(cmd_ctx, self.address) - return cmd:run() +--- Disconnect the modem +---@return boolean ok +---@return string? reason +---@return integer? code +function Modem:disconnect() + local ok, err = self.backend:disconnect() + if not ok then + return return_error(err, 1) + end + self:invalidate_groups { 'network', 'signal', 'traffic' } + return true end -function Driver:connect(connection_string) - local new_ctx = context.with_timeout(self.ctx, CMD_TIMEOUT) - local cmd = mmcli.connect(new_ctx, self.address, connection_string) - local out, err = cmd:combined_output() - return out, err +--- Inhibit the modem +---@return boolean ok +---@return string? reason +---@return integer? code +function Modem:inhibit() + local done_ch = channel.new() + + local ok, err = self.scope:spawn(function() + local result_ok, result_err = self.backend:inhibit() + done_ch:put({ ok = result_ok, err = result_err }) + end) + + if not ok then + return return_error("failed to spawn inhibit fiber: " .. tostring(err), 1) + end + + local source, msg, primary = fibers.perform(op.named_choice { + done = done_ch:get_op(), + failed = self.scope:fault_op(), + }) + + if source == "done" then + if not msg.ok then + return return_error(msg.err, 1) + end + return true + elseif source == "failed" then + return return_error("modem inhibit failed: " .. tostring(primary), 1) + end + return return_error("unexpected error during modem inhibit", 1) end -function Driver:disconnect() - local cmd_ctx = context.with_timeout(self.ctx, CMD_TIMEOUT) - local cmd = mmcli.disconnect(cmd_ctx, self.address) - return cmd:run() +--- Uninhibit the modem +---@return boolean ok +---@return string? reason +---@return integer? code +function Modem:uninhibit() + local ok, err = self.backend:uninhibit() + if not ok then + return return_error(err, 1) + end + return true end -function Driver:inhibit() - if self.inhibit_cmd then return true end - self.inhibit_cmd = mmcli.inhibit(self.address) - self.inhibit_cmd:setprdeathsig(sc.SIGKILL) - local err = self.inhibit_cmd:start() - if err then - log.trace(string.format("Modem inhibit failed, reason: %s", err)) - return false +--- Start listening for a sim insertion +---@return boolean ok +---@return string? reason +---@return integer? code +function Modem:listen_for_sim() + if self.listening_for_sim then + return true + end + self.listening_for_sim = true + local ok, err = fibers.current_scope():spawn(function() + self:_emit_state("sim_listener", "open") + + fibers.current_scope():finally(function() + self.listening_for_sim = false + self:_emit_state("sim_listener", "closed") + end) + + -- Capture pulse version before reading state to close the insertion race window. + local last_seen = self.sim_inserted_pulse:version() + local sim_present = self.sim_state_ch:get() + + while sim_present ~= true do + local source, _, primary = fibers.perform(op.named_choice { + inserted = self.sim_inserted_pulse:changed_op(last_seen), + trigger = sleep.sleep_op(LISTEN_TRIGGER_INTERVAL), + failed = self.scope:fault_op(), + }) + if source == "inserted" then + break + elseif source == "trigger" then + local trigger_ok, check_err = self.backend:trigger_sim_presence_check() + if not trigger_ok then + self.log:error({ what = 'trigger_sim_check_failed', imei = self.imei, err = tostring(check_err) }) + end + elseif source == "failed" then + self.log:error({ what = 'listen_for_sim_scope_faulted', imei = self.imei, err = tostring(primary) }) + break + end + end + end) + if not ok then + return return_error("listen_for_sim spawn failed: " .. tostring(err), 1) end return true end -function Driver:uninhibit() - if not self.inhibit_cmd then return true end - self.inhibit_cmd:kill() - self.inhibit_cmd:wait() - self.inhibit_cmd = nil +--- Set the signal update period +---@param opts ModemSignalUpdateOpts +---@return boolean ok +---@return string? reason +---@return integer? code +function Modem:set_signal_update_freq(opts) + if opts == nil or getmetatable(opts) ~= capability_args.ModemSignalUpdateOpts then + return return_error("invalid options", 1) + end + local ok, err = self.backend:set_signal_update_interval(opts.frequency) + if not ok then + return return_error(err, 1) + end return true end -function Driver:wait_for_sim() - if self.waiting_for_sim then return end - self.waiting_for_sim = true - local warm_swap_ctx = context.with_cancel(self.ctx) - fiber.spawn(function() - local connected = false - - local sim_monitor_cmd = self.monitor_slot_status() - sim_monitor_cmd:setprdeathsig(sc.SIGKILL) - local sim_stdout = assert(sim_monitor_cmd:stdout_pipe()) - local sim_cmd_err = sim_monitor_cmd:start() - if sim_cmd_err then - sim_monitor_cmd:kill() - sim_monitor_cmd:wait() - sim_stdout:close() - log.error(string.format( - "%s - %s: Failed to start SIM monitor for %s, reason: %s", - warm_swap_ctx:value("service_name"), - warm_swap_ctx:value("fiber_name"), - self.imei, - sim_cmd_err - )) - return - end +function Modem:emitter() + local timeout_buffer = 0.1 + self.log:debug({ what = 'emitter_started', imei = self.imei }) + + fibers.current_scope():finally(function() + self.log:debug({ what = 'emitter_exiting', imei = self.imei }) + end) - log.trace(string.format( - "%s - %s: Waiting for SIM for %s", - warm_swap_ctx:value("service_name"), - warm_swap_ctx:value("fiber_name"), - self.imei - )) - local continue = true - local sim_monitor_output = "" - while not connected and continue do - local state, ctx_err = op.choice( - sim_stdout:read_line_op():wrap(function(line) - if line == nil then - continue = false - return + local seen_version = 0 + while true do + self.log:debug({ what = 'emitter_waiting', imei = self.imei, seen_version = seen_version }) + local new_version = self.state_pulse:changed(seen_version) + if not new_version then + -- Pulse was closed + break + end + seen_version = new_version + sleep.sleep(timeout_buffer) -- we want to put some buffer time in to invalidate any cache + self.log:debug({ what = 'emitter_dispatching', imei = self.imei }) + + for group_name, fields in pairs(GROUP_FIELDS) do + local snapshot, group_err = self:_get_group(group_name, timeout_buffer) + if group_err ~= "" then + if D_LOG_EMITTER then + self.log:warn({ + what = 'emitter_group_failed', + imei = self.imei, + group = group_name, + err = tostring(trim_error(group_err)) + }) + end + else + for _, field in ipairs(fields) do + local value = extract_group_field(snapshot, field) + if value ~= nil then + local emit_ok, emit_err = self:_emit_meta(field, value) + if not emit_ok and D_LOG_EMITTER then + self.log:warn({ + what = 'emitter_emit_failed', + imei = self.imei, + field = tostring(field), + err = tostring(emit_err) + }) + end end - -- we need to accumulate the sim monitor output as - -- it can be a variable number of lines - sim_monitor_output = sim_monitor_output .. line - local sim_state, err = utils.parse_slot_monitor(sim_monitor_output) - if err then return end - sim_monitor_output = "" - return sim_state == 'present' - end), - warm_swap_ctx:done_op():wrap(function() - sim_monitor_cmd:kill() - return nil, warm_swap_ctx:err() - end) - ):perform() - if ctx_err then break end - if state ~= nil then - connected = state + end end end - if connected then - log.info(string.format( - "%s - %s: SIM detected for %s", - warm_swap_ctx:value("service_name"), - warm_swap_ctx:value("fiber_name"), - self.imei - )) + end +end + +--- Handles both modem card state changes and SIM presence lifecycle in a single fiber. +--- card_state is always current when SIM removal decisions are made, eliminating the +--- need for backend-level guards about whether to reset on SIM absent. +function Modem:modem_lifecycle_monitor() + local function on_card_change(state_update) + ---@cast state_update ModemStateEvent + self.log:debug({ what = 'state_change', imei = self.imei, from = state_update.from, to = state_update.to }) + local to_state = state_update and state_update.to or nil + if to_state == 'failed' or to_state == 'disabled' then + self:_invalidate_dynamic() + elseif to_state == 'locked' then + self:invalidate_groups { 'network', 'signal', 'traffic' } else - log.error(string.format( - "%s - %s: SIM not detected for %s, exiting", - warm_swap_ctx:value("service_name"), - warm_swap_ctx:value("fiber_name"), - self.imei - )) + self:invalidate_groups { 'network', 'signal', 'traffic' } end - warm_swap_ctx:cancel() - sim_monitor_cmd:wait() - sim_stdout:close() - end) + self:_emit_state('card', state_update) - sleep.sleep(0.1) - - log.trace(string.format( - "%s - %s: Power cycling for modem %s", - warm_swap_ctx:value("service_name"), - warm_swap_ctx:value("fiber_name"), - self.imei - )) - local high_power = true - local out, err - while not warm_swap_ctx:err() do - -- this is going to really hammer the modem - -- without a courtesy sleep - if high_power then - out, err = self.set_power_low(warm_swap_ctx) - if err then - log.debug(string.format( - "Setting low power failed: %s (%s)", - out, - err - )) - high_power = false - else - high_power = false + self.state_pulse:signal() + end + + local function on_sim_change(present, current_card_state) + local sim_state = present == true and "present" or "absent" + + self.log:debug({ what = 'sim_' .. sim_state, imei = self.imei }) + self:_emit_state("sim_state", sim_state) + + if present == true then + self:_invalidate_dynamic() + self.sim_inserted_pulse:signal() + self.state_pulse:signal() + elseif present == false then + self:_invalidate_dynamic() + if current_card_state ~= "failed" then + -- Only reset if the card is not already in a failed state. + -- If failed, a SIM-absent report is expected and resetting would cause a boot loop. + self:reset() + self.scope:cancel("modem restarting") end end - sleep.sleep(1) - if not high_power then - out, err = self.set_power_high(warm_swap_ctx) - if err then - log.debug(string.format( - "Setting high power failed: %s (%s) for %s", - out, - err, self.imei - )) + end + + self.log:debug({ what = 'lifecycle_monitor_started', imei = self.imei }) + + fibers.current_scope():finally(function() + self.log:debug({ what = 'lifecycle_monitor_exiting', imei = self.imei }) + end) + + local init_card_state, state_err = fibers.perform(self.backend:monitor_state_op()) + if state_err ~= "" then + self.log:error({ what = 'state_monitor_init_failed', imei = self.imei, err = tostring(state_err) }) + return + end + ---@cast init_card_state ModemStateEvent + on_card_change(init_card_state) + local card_state = init_card_state and init_card_state.to + local sim_present = nil + while true do + local source, v1, v2 = fibers.perform(op.named_choice { + card_change = self.backend:monitor_state_op(), -- outputs when modem state changes + sim_change = self.backend:wait_for_sim_present_op(), -- outputs when sim state changes + send = self.sim_state_ch:put_op(sim_present), + }) + + if is_command_closed(v2) then + local command_name = source == "card_change" and "state monitor" or "sim monitor" + self.log:error({ what = command_name .. '_closed', imei = self.imei, err = tostring(v2) }) + break + end + + if source == "card_change" then + local state_update, err = v1, v2 + ---@cast state_update ModemStateEvent + if err ~= "" then + self.log:error({ what = 'state_monitor_error', imei = self.imei, err = tostring(err) }) + elseif state_update then + card_state = state_update.to + on_card_change(state_update) + end + elseif source == "sim_change" then + local present, err = v1, v2 + if err ~= "" then + self.log:error({ what = 'sim_poll_error', imei = self.imei, err = tostring(err) }) else - high_power = true + if sim_present ~= present then + on_sim_change(present, card_state) + end + sim_present = present end end - sleep.sleep(1) + -- source == "send": listener consumed sim_present, loop to re-offer + end + self.log:trace({ what = 'lifecycle_monitor_exiting', imei = self.imei }) +end + +function Modem:control_manager() + if self.cap_emit_ch == nil then + self.log:error({ what = 'control_no_emit_ch', imei = self.imei }) + return + end + if self.control_ch == nil then + self.log:error({ what = 'control_no_control_ch', imei = self.imei }) + return end - -- we must attempt to put modem into high power state even if disconnected - -- as we could otherwise get stuck in a failed state boot-loop - if not high_power then - for _ = 1, 3 do - out, err = self.set_power_high(context.with_timeout(context.background(), CMD_TIMEOUT)) - if err then - sleep.sleep(1) + self.log:debug({ what = 'control_manager_started', imei = self.imei }) + + fibers.current_scope():finally(function() + self.log:debug({ what = 'control_manager_exiting', imei = self.imei }) + end) + + while true do + local request, req_err = self.control_ch:get() + if not request then + self.log:error({ what = 'control_ch_error', imei = self.imei, err = tostring(req_err) }) + break + end + + ---@cast request ControlRequest + + local ok, reason, code + + local fn = self[request.verb] + local valid, validation_err = validate_fn(fn, request.verb) + if not valid then + ok = false + reason = validation_err + else + local call_ok, fn_ok, fn_reason, fn_code = pcall(fn, self, request.opts) + if not call_ok then + ok = false + reason = "internal error: " .. tostring(fn_ok) + code = 1 else - high_power = true - break + ok = fn_ok + reason = fn_reason + code = fn_code end end - if not high_power then - log.error(string.format( - '%s: Failed to set modem power high "%s" (%s) for %s', - self.ctx:value("service_name"), - out, - err, - self.imei - )) + + local reply, reply_err = hal_types.new.Reply(ok, reason, code) + if not reply then + self.log:error({ what = 'reply_create_failed', imei = self.imei, err = tostring(reply_err) }) + else + request.reply_ch:put(reply) end end - self.waiting_for_sim = false end -function Driver:sim_detect() - fiber.spawn(function() - self:wait_for_sim() - end) - return true, nil +---- Driver Functions ---- + +--- Spawn driver services +---@return boolean ok +---@return string error +function Modem:start() + if not self.initialised then + return false, "modem not initialised" + end + if not self.caps_applied then + return false, "capabilities not applied" + end + + self.scope:spawn(function() self:modem_lifecycle_monitor() end) + self.scope:spawn(function() self:control_manager() end) + self.scope:spawn(function() self:emitter() end) + + -- Signal initial pulse so emitter emits the initial state + self.state_pulse:signal() + + return true, "" end -function Driver:fix_failure() - fiber.spawn(function() - self:wait_for_sim() - self:inhibit() - self:uninhibit() - end) - return true, nil -end - -function Driver:set_signal_update_freq(seconds) - local cmd = mmcli.signal_setup(self.ctx, self.address, seconds) - local cmd_err = cmd:run() - self.refresh_rate_channel:put(seconds) - return (cmd_err == nil), cmd_err -end - -local function modem_states_equal(state1, state2) - return state1.curr_state == state2.curr_state and - state1.prev_state == state2.prev_state and - state1.reason == state2.reason -end - -function Driver:state_monitor(ctx) - if ctx:err() then return end - log.trace(string.format( - "%s - %s: Started for %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - self.imei - )) - - -- setup the modem monitor - local state_monitor_cmd = mmcli.monitor_state(self.address) - state_monitor_cmd:setprdeathsig(sc.SIGKILL) - local state_stdout = assert(state_monitor_cmd:stdout_pipe()) - local cmd_err = state_monitor_cmd:start() - if cmd_err then - state_monitor_cmd:kill() - state_monitor_cmd:wait() - state_stdout:close() - log.error(string.format( - "%s - %s: Failed to start for %s, reason: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - self.imei, - cmd_err - )) - return +--- Closes down the modem driver +---@param timeout number? Timeout in seconds +---@return boolean ok +---@return string error +function Modem:stop(timeout) + timeout = timeout or DEFAULT_STOP_TIMEOUT + + if self.backend and self.backend.shutdown_op then + local ok, err = fibers.perform(self.backend:shutdown_op(1.0)) + if not ok then + self.log:warn({ + what = "backend_shutdown_failed", + address = self.address, + imei = self.imei, + err = tostring(err), + }) + end end - local sim_monitor_cmd = self.monitor_slot_status() - sim_monitor_cmd:setprdeathsig(sc.SIGKILL) - local sim_stdout = assert(sim_monitor_cmd:stdout_pipe()) - local sim_cmd_err = sim_monitor_cmd:start() - if sim_cmd_err then - state_monitor_cmd:kill() - state_monitor_cmd:wait() - state_stdout:close() - sim_monitor_cmd:kill() - sim_monitor_cmd:wait() - sim_stdout:close() - log.error(string.format( - "%s - %s: Failed to start SIM monitor for %s, reason: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - self.imei, - sim_cmd_err - )) - return + self.scope:cancel("modem stopping") + + local source = fibers.perform(op.named_choice { + join = self.scope:join_op(), + timeout = sleep.sleep_op(timeout) + }) + + if source == "timeout" then + return false, "modem stop timeout" end + return true, "" +end - local prev_modem_state = {} - local curr_modem_state - local curr_sim_state = self.is_sim_inserted() - local continue = true - local sim_monitor_output = "" - local enabled_sleep_op = nil -- This op is only created in the case of a modem stuck in enabled - while not ctx:err() and continue do - local modem_state, sim_state, err = op.choice( - state_stdout:read_line_op():wrap(function(line) - if line == nil then - sim_monitor_cmd:kill() - continue = false - return - end - local state, err = utils.parse_modem_monitor(line) - if err then return end - return state - end), - sim_stdout:read_line_op():wrap(function(line) - if line == nil then - state_monitor_cmd:kill() - continue = false - return - end - -- we need to accumulate the sim monitor output as - -- it can be a variable number of lines - sim_monitor_output = sim_monitor_output .. line - local sim_state, err = utils.parse_slot_monitor(sim_monitor_output) - if err then return end - sim_monitor_output = "" - return nil, sim_state == 'present' - end), - ctx:done_op():wrap(function() - state_monitor_cmd:kill() - sim_monitor_cmd:kill() - return nil, nil, ctx:err() - end), - enabled_sleep_op -- if this is nil to op the list will simply look like state, sim, ctx with no sleep - ):perform() - enabled_sleep_op = nil - if err then break end - - if modem_state then - curr_modem_state = modem_state +--- Apply capabilities to HAL and start monitoring state +--- Modem must be initialised first +---@param emit_ch Channel +---@return Capability[]? capabilities +---@return string? error +function Modem:capabilities(emit_ch) + if not self.initialised then + return nil, "modem not initialised" + end + if self.caps_applied then + return nil, "capabilities already applied" + end + + self.cap_emit_ch = emit_ch + + local modem_cap, mod_cap_err = cap_types.new.ModemCapability( + self.imei, + self.control_ch + ) + if not modem_cap then + return nil, "failed to create modem capability: " .. tostring(mod_cap_err) + end + + self.caps_applied = true + + return { modem_cap } +end + +--- Setup modem overrides and long running fibers +---@return string error +function Modem:init() + if self.initialised then + return "already initialised" + end + + local backend_built_sig = cond.new() + + local ok, err = self.scope:spawn(function() + self.backend = modem_backend_provider.new(self.address) + + self.scope:finally(function() + if self.backend and self.backend.terminate then + self.backend:terminate("modem_driver_scope_exit") + end + end) + + local sim_ok, sim_err = self.backend:start_sim_presence_monitor() + if not sim_ok then + error("failed to start SIM presence monitor: " .. tostring(sim_err)) end - if sim_state ~= nil then - curr_sim_state = sim_state + + local state_ok, state_err = self.backend:start_state_monitor() + if not state_ok then + error("failed to start modem state monitor: " .. tostring(state_err)) end - if curr_modem_state then - -- we want to preserve the current modem seperate from the sim changes - -- as when a sim is next inserted we want to remember the original curr_state - local merged_state = curr_modem_state - if curr_sim_state == false and curr_modem_state.curr_state ~= 'failed' then - merged_state = { - type = curr_modem_state.type, - prev_state = curr_modem_state.prev_state, - curr_state = 'no_sim', - reason = curr_modem_state.reason - } - end - if prev_modem_state.curr_state ~= merged_state.curr_state then - self.info_q:put({ - type = "modem", - id = self.imei, - sub_topic = { "state" }, - endpoints = "single", - info = merged_state - }) - prev_modem_state = merged_state - end - if merged_state.curr_state == 'enabled' then - enabled_sleep_op = sleep.sleep_op(60):wrap(function() - if not self.is_sim_inserted() then - self:reset() - end - end) - end + local identity_info, identity_err = self.backend:read_identity() + if not identity_info then + error("failed to get modem identity info: " .. tostring(identity_err)) end + + self:_cache_group("identity", identity_info) + self.imei = identity_info.imei + self.model = identity_info.model + self.mode = identity_info.mode + + backend_built_sig:signal() + end) + + if not ok then + return "failed to spawn modem backend fiber: " .. tostring(err) end - state_monitor_cmd:wait() - state_stdout:close() - sim_monitor_cmd:wait() - sim_stdout:close() - log.trace(string.format( - "%s - %s: Closed for %s, reason: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - self.imei, - ctx:err() - )) -end - ----Listens for commands from HAL and executes them -function Driver:command_manager() - log.trace(string.format("Modem - %s: Command Manager started", self.imei)) - while not self.ctx:err() do - local cmd_msg = op.choice( - self.command_q:get_op(), - self.ctx:done_op() - ):perform() - - - if cmd_msg ~= nil then - local cmd = self[cmd_msg.command] - local ret, err = nil, 'command does not exist' - if cmd ~= nil then - local args = cmd_msg.args or {} - log.trace(string.format( - "%s - %s: Executing command: '%s'", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - cmd_msg.command - )) - ret, err = cmd(self, unpack(args)) - end - fiber.spawn(function() - op.choice( - cmd_msg.return_channel:put_op({ result = ret, err = err }), - self.ctx:done_op() - ):perform() - end) - end + local source, _, primary = fibers.perform(op.named_choice { + backend_ready = backend_built_sig:wait_op(), + failed = self.scope:fault_op() + }) + + if source == "backend_ready" then + self.initialised = true + return "" + elseif source == "failed" then + return "modem init failed: " .. tostring(primary) -- primary is the error from the faulted fiber + else + return "unexpected error during modem init" end - log.trace(string.format("Modem - %s: Command Manager stopped (%s)", self.imei, self.ctx:err())) end -local function new(ctx, address) - local self = setmetatable({}, Driver) - self.ctx = ctx - self.address = address - self.command_q = queue.new(10) +--- Create a new Modem driver. +---@param address ModemAddress +---@param logger Logger +---@return Modem? modem +---@return string error +local function new(address, logger) + if type(address) ~= 'string' or address == '' then + return nil, "invalid address" + end + + local control_ch = channel.new(CONTROL_Q_LEN) + + local scope, err = fibers.current_scope():child() + if not scope then + return nil, "failed to create child scope: " .. tostring(err) + end + + -- Print out driver stack trace if scope closes on a failure + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + logger:error({ what = 'scope_error', address = address, err = tostring(primary) }) + logger:debug({ what = 'scope_exit', address = address, status = st }) + end + logger:debug({ what = 'stopped', address = address }) + end) - self.refresh_rate_channel = channel.new() - -- Other initial properties - return self + return setmetatable({ + scope = scope, + log = logger, + address = address, + cache = cache_mod.new(math.huge, fibers.now, '.'), + initialised = false, -- modem cannot apply capabilities until initialised + caps_applied = false, -- modem cannot start until capabilities applied + listening_for_sim = false, + state_pulse = pulse.new(), + sim_inserted_pulse = pulse.new(), + sim_state_ch = channel.new(), + control_ch = control_ch + }, Modem), "" end return { diff --git a/src/services/hal/drivers/modem/at.lua b/src/services/hal/drivers/modem/at.lua deleted file mode 100644 index df54eb8b..00000000 --- a/src/services/hal/drivers/modem/at.lua +++ /dev/null @@ -1,71 +0,0 @@ -package.path = '/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua;' .. package.path - -local file = require 'fibers.stream.file' -local op = require 'fibers.op' - -local function trim(input) - -- Pattern matches non-printable characters and spaces at the start and end of the string - -- %c matches control characters, %s matches all whitespace characters - -- %z matches the character with representation 0x00 (NUL byte) - return (input:gsub("^[%c%s%z]+", ""):gsub("[%c%s%z]+$", "")) -end - -local function send_with_context(ctx, port, command) - local reader, err = file.open(port, "r") - if not reader then return nil, "error opening AT read port: "..err end - - local writer = assert(file.open(port, "w")) - if not writer then return nil, "error opening AT write port: "..err end - - -- file write - op.choice( - writer:write_chars_op(command..'\r'), - ctx:done_op() - ):perform() - - writer:close() - - if ctx:err() then reader:close() return nil, ctx:err() end - - local res = {} - - while true do - local line = op.choice( - reader:read_line_op(), - ctx:done_op() - ):perform() - - if ctx:err() then reader:close() return nil, ctx:err() end - if not line then reader:close() return nil, 'unknown error' end - - line = trim(line) - - -- check for non-descriptive success/fail - if line:find("^OK$") then - reader:close() - return res, nil - elseif line:find("^ERROR$") then - reader:close() - return res, 'error' - else - -- check for descriptive fail - local error_code - error_code = line:match("^%+CME ERROR: (%d+)$") - if error_code then - reader:close() - return res, error_code - end - error_code = line:match("^%+CMS ERROR: (%d+)$") - if error_code then - reader:close() - return res, error_code - end - end - - if #line > 0 then table.insert(res, line) end - end -end - -return { - send_with_context = send_with_context -} \ No newline at end of file diff --git a/src/services/hal/drivers/modem/mmcli.lua b/src/services/hal/drivers/modem/mmcli.lua deleted file mode 100644 index f127961f..00000000 --- a/src/services/hal/drivers/modem/mmcli.lua +++ /dev/null @@ -1,75 +0,0 @@ -local exec = require "fibers.exec" - -local function monitor_modems() - return exec.command('mmcli', '-M') -end - -local function inhibit(device) - return exec.command('mmcli', '-m', device, '--inhibit') -end - -local function connect(ctx, device, connection_string) - connection_string = string.format("--simple-connect=%s", connection_string) - return exec.command_context(ctx, 'mmcli', '-m', device, connection_string) -end - -local function disconnect(ctx, device) - return exec.command_context(ctx, 'mmcli', '-m', device, '--simple-disconnect') -end - -local function reset(ctx, device) - return exec.command_context(ctx, 'mmcli', '-m', device, '-r') -end - -local function enable(ctx, device) - return exec.command_context(ctx, 'mmcli', '-m', device, '-e') -end - -local function disable(ctx, device) - return exec.command_context(ctx, 'mmcli', '-m', device, '-d') -end - -local function monitor_state(device) - return exec.command('mmcli', '-m', device, '-w') -end - -local function information(ctx, device) - return exec.command_context(ctx, 'mmcli', '-J', '-m', device) -end - -local function sim_information(ctx, device) - return exec.command_context(ctx, 'mmcli', '-J', '-i', device) -end -local function location_status(ctx, device) - return exec.command_context(ctx, 'mmcli', '-J', '-m', device, '--location-status') -end - -local function signal_setup(ctx, device, rate) - return exec.command_context(ctx, 'mmcli', '-m', device, '--signal-setup=' .. tostring(rate)) -end - -local function signal_get(ctx, device) - return exec.command_context(ctx, 'mmcli', '-J', '-m', device, '--signal-get') -end - -local function three_gpp_set_initial_eps_bearer_settings(ctx, device, settings) - local settings_string = string.format("--3gpp-set-initial-eps-bearer-settings=%s", settings) - return exec.command_context(ctx, 'mmcli', '-m', device, settings_string) -end - -return { - monitor_modems = monitor_modems, - inhibit = inhibit, - connect = connect, - disconnect = disconnect, - reset = reset, - enable = enable, - disable = disable, - monitor_state = monitor_state, - information = information, - sim_information = sim_information, - location_status = location_status, - signal_setup = signal_setup, - signal_get = signal_get, - three_gpp_set_initial_eps_bearer_settings = three_gpp_set_initial_eps_bearer_settings, -} diff --git a/src/services/hal/drivers/modem/mode.lua b/src/services/hal/drivers/modem/mode.lua deleted file mode 100644 index 4f18b166..00000000 --- a/src/services/hal/drivers/modem/mode.lua +++ /dev/null @@ -1,14 +0,0 @@ --- driver/mode.lua - -local function add_mode_funcs(modem) - local mode = modem.mode:lower() - local status, mode_funcs = pcall(require, "services.hal.drivers.modem.mode." .. mode) - if not status then - return false, "Mode driver not found for mode: " .. modem.mode .. "\n\t" .. mode_funcs - end - return mode_funcs(modem) -end - -return { - add_mode_funcs = add_mode_funcs -} diff --git a/src/services/hal/drivers/modem/mode/mbim.lua b/src/services/hal/drivers/modem/mode/mbim.lua deleted file mode 100644 index f20eb7e0..00000000 --- a/src/services/hal/drivers/modem/mode/mbim.lua +++ /dev/null @@ -1,11 +0,0 @@ --- driver/mode/mbim.lua -return function(modem) - modem.example_1 = function() - print("Running MBIM example 1...") - end - modem.example_2 = function() - print("Running MBIM example 2...") - end - -- Add other MBIM-specific methods - return true -end diff --git a/src/services/hal/drivers/modem/mode/qmi.lua b/src/services/hal/drivers/modem/mode/qmi.lua deleted file mode 100644 index 6d20fd1f..00000000 --- a/src/services/hal/drivers/modem/mode/qmi.lua +++ /dev/null @@ -1,150 +0,0 @@ -local context = require 'fibers.context' -local wraperr = require "wraperr" -local qmicli = require "services.hal.drivers.modem.qmicli" -local utils = require "services.hal.utils" -local log = require "services.log" - -local CMD_TIMEOUT = 3 - --- driver/mode/qmi.lua -return function(modem) - modem.is_sim_inserted = function() - local new_ctx = context.with_timeout(modem.ctx, CMD_TIMEOUT) - local cmd = qmicli.uim_get_card_status(new_ctx, modem.primary_port) - cmd:setpgid(true) - local out, cmd_err = cmd:combined_output() - if cmd_err then return nil, cmd_err end - - local status, parse_err = utils.parse_slot_monitor(out) - if parse_err then return nil, parse_err end - return status == 'present', nil - end - - modem.set_power_low = function(ctx) - local new_ctx = context.with_timeout(ctx, CMD_TIMEOUT) - local cmd = qmicli.uim_sim_power_off(new_ctx, modem.primary_port) - cmd:setpgid(true) - return cmd:combined_output() - end - - modem.set_power_high = function(ctx) - local new_ctx = context.with_timeout(ctx, CMD_TIMEOUT) - local cmd = qmicli.uim_sim_power_on(new_ctx, modem.primary_port) - cmd:setpgid(true) - return cmd:combined_output() - end - - modem.monitor_slot_status = function() - return qmicli.uim_monitor_slot_status(modem.primary_port) - end - - modem.get_mcc_mnc = function() - local new_ctx = context.with_timeout(modem.ctx, CMD_TIMEOUT) - local cmd = qmicli.nas_get_home_network(new_ctx, modem.primary_port) - cmd:setpgid(true) - local out, err = cmd:combined_output() - if err then return nil, nil, wraperr.new(err) end - - local mcc = out:match("MCC:%s+'(%d+)'") - local mnc = out:match("MNC:%s+'(%d+)'") - return mcc, mnc, nil - end - - modem.uim_get_gids = function() - local gids = {} - - local gid1_ctx = context.with_timeout(modem.ctx, CMD_TIMEOUT) - local gid1_cmd = qmicli.uim_read_transparent(gid1_ctx, modem.primary_port, '0x3F00,0x7FFF,0x6F3E') - gid1_cmd:setpgid(true) - - local gid1_out, gid1_cmd_err = gid1_cmd:combined_output() - if gid1_cmd_err then return gids, wraperr.new(gid1_cmd_err) end - - -- Anchor parsing to the "Read result:" section and make it nil-safe. - -- Example expected format (simplified): "Read result: 12:34:56:78" - local gid1_hex = gid1_out:match("Read result:%s*([0-9A-Fa-f:]+)") - if not gid1_hex then - return gids, wraperr.new("failed to parse GID1 from qmicli output") - end - - gids.gid1 = gid1_hex:gsub(":", "") - return gids - end - - modem.nas_get_rf_band_info = function() - local key_map = { - ["Band Information"] = "band-information", - ["Radio Interface"] = "radio-interface", - ["Active Band Class"] = "active-band-class", - ["Active Channel"] = "active-channel", - ["Band Information (Extended)"] = "band-information-extended" - } - local band_info_cmd = qmicli.nas_get_rf_band_info( - context.with_timeout(modem.ctx, CMD_TIMEOUT), - modem.primary_port - ) - band_info_cmd:setpgid(true) - - local band_info_out, band_info_err = band_info_cmd:combined_output() - if band_info_err then return nil, string.format("%s (%s)", band_info_out, band_info_err) end - - local band_info, parse_err = utils.parse_qmicli_output(band_info_out, key_map) - if parse_err then return nil, parse_err end - return band_info - end - local function nas_get_home_network_parsed() - local key_map = { - ["Home network"] = "home-network", - MCC = "mcc", - MNC = "mnc", - Description = "description", - ["Network name source"] = "network-name-source" - } - local new_ctx = context.with_timeout(modem.ctx, CMD_TIMEOUT) - local cmd = qmicli.nas_get_home_network(new_ctx, modem.primary_port) - cmd:setpgid(true) - local out, err = cmd:combined_output() - if err then return nil, wraperr.new(err) end - - return utils.parse_qmicli_output(out, key_map) - end - - local function nas_get_signal_info_parsed() - local key_map = { - LTE = "lte", - RSSI = "rssi", - RSRQ = "rsrq", - RSRP = "rsrp", - SNR = "snr" - } - local new_ctx = context.with_timeout(modem.ctx, CMD_TIMEOUT) - local cmd = qmicli.nas_get_signal_info(new_ctx, modem.primary_port) - cmd:setpgid(true) - local out, err = cmd:combined_output() - if err then return nil, wraperr.new(err) end - - return utils.parse_qmicli_output(out, key_map) - end - - modem.get_nas_info = function() - local nas_infos = {} - - local home_network_info, hn_err = nas_get_home_network_parsed() - if hn_err == nil and home_network_info then - for k, v in pairs(home_network_info) do - nas_infos[k] = v - end - end - - local signal_info, sig_err = nas_get_signal_info_parsed() - if sig_err == nil and signal_info then - for k, v in pairs(signal_info) do - nas_infos[k] = v - end - end - - return nas_infos - end - -- Add other QMI-specific methods - return true -end diff --git a/src/services/hal/drivers/modem/model.lua b/src/services/hal/drivers/modem/model.lua deleted file mode 100644 index ee16a095..00000000 --- a/src/services/hal/drivers/modem/model.lua +++ /dev/null @@ -1,14 +0,0 @@ --- driver/model.lua - -local function add_model_funcs(modem) - local manufacturer = modem.manufacturer - local status, model_funcs = pcall(require, "services.hal.drivers.modem.model." .. manufacturer) - if not status then - return false, "Model functions not found for manufacturer: " .. modem.manufacturer - end - return model_funcs(modem) -end - -return { - add_model_funcs = add_model_funcs -} diff --git a/src/services/hal/drivers/modem/model/fibocom.lua b/src/services/hal/drivers/modem/model/fibocom.lua deleted file mode 100644 index 5e4ca1ea..00000000 --- a/src/services/hal/drivers/modem/model/fibocom.lua +++ /dev/null @@ -1,20 +0,0 @@ --- driver/model/fibocom.lua -local funcs = { - { - name = 'func1', - models = {l860 = true, fm350 = true}, - func = function(modem) - print("Special function for L860 and FM350") - end - }, - -- Other functions -} - -return function(modem) - for _, f in ipairs(funcs) do - if f.models[modem:model()] then - modem[f.name] = f.func - end - end - return true -end \ No newline at end of file diff --git a/src/services/hal/drivers/modem/model/quectel.lua b/src/services/hal/drivers/modem/model/quectel.lua deleted file mode 100644 index 91657a30..00000000 --- a/src/services/hal/drivers/modem/model/quectel.lua +++ /dev/null @@ -1,61 +0,0 @@ -local context = require "fibers.context" -local mmcli = require "services.hal.drivers.modem.mmcli" - -local CMD_TIMEOUT = 3 - --- driver/model/quectel.lua -local funcs = { - { - name = 'func1', - conditionals = { - function(modem) - return modem.model == 'eg25' - end, - function(modem) - return modem.model == 'em06' - end - }, - func = function(modem) - print("Special function for EG25 and EM06") - end - }, - -- This is a special case for the RM520N, it has a initial bearer which will always cause a - -- multiple PDN failure unless set to the APN we want to use OR set to empty. - { - name = 'connect', - conditionals = { - function(modem) - return modem.model == 'rm520n' - end - }, - func = function(modem, connection_string) - local clear_initial_bearer_ctx = context.with_timeout(modem.ctx, CMD_TIMEOUT) - local cmd_clear = mmcli.three_gpp_set_initial_eps_bearer_settings( - clear_initial_bearer_ctx, - modem.address, - "apn=" - ) - local out_clear, err_clear = cmd_clear:combined_output() - if err_clear then - return out_clear, err_clear - end - local connect_ctx = context.with_timeout(modem.ctx, CMD_TIMEOUT) - local cmd = mmcli.connect(connect_ctx, modem.address, connection_string) - local out, err = cmd:combined_output() - return out, err - end - } - -- Other functions -} - -return function(modem) - for _, f in ipairs(funcs) do - for _, cond in ipairs(f.conditionals) do - if cond(modem) then - modem[f.name] = f.func - break - end - end - end - return true -end diff --git a/src/services/hal/drivers/modem/qmicli.lua b/src/services/hal/drivers/modem/qmicli.lua deleted file mode 100644 index a27405df..00000000 --- a/src/services/hal/drivers/modem/qmicli.lua +++ /dev/null @@ -1,48 +0,0 @@ -local exec = require "fibers.exec" - -local function uim_get_card_status(ctx, port) - return exec.command_context(ctx, "qmicli", "-p", "-d", port, "--uim-get-card-status") -end - -local function uim_sim_power_off(ctx, port) - return exec.command_context(ctx, "qmicli", "-p", "-d", port, "--uim-sim-power-off=1") -end - -local function uim_sim_power_on(ctx, port) - return exec.command_context(ctx, "qmicli", "-p", "-d", port, "--uim-sim-power-on=1") -end - -local function uim_monitor_slot_status(port) - return exec.command('qmicli', '-p', '-d', port, '--uim-monitor-slot-status') -end - -local function uim_read_transparent(ctx, port, address_string) - local addresses = string.format('--uim-read-transparent=%s', address_string) - return exec.command_context(ctx, 'qmicli', '-p', '-d', port, addresses) -end - -local function nas_get_rf_band_info(ctx, port) - return exec.command_context(ctx, 'qmicli', '-p', '-d', port, '--nas-get-rf-band-info') -end -local function nas_get_home_network(ctx, port) - return exec.command_context(ctx, 'qmicli', '-p', '-d', port, '--nas-get-home-network') -end - -local function nas_get_serving_system(ctx, port) - return exec.command_context(ctx, 'qmicli', '-p', '-d', port, '--nas-get-serving-system') -end -local function nas_get_signal_info(ctx, port) - return exec.command_context(ctx, 'qmicli', '-p', '-d', port, '--nas-get-signal-info') -end -return { - uim_get_card_status = uim_get_card_status, - uim_sim_power_off = uim_sim_power_off, - uim_sim_power_on = uim_sim_power_on, - uim_monitor_slot_status = uim_monitor_slot_status, - uim_read_transparent = uim_read_transparent, - - nas_get_rf_band_info = nas_get_rf_band_info, - nas_get_home_network = nas_get_home_network, - nas_get_serving_system = nas_get_serving_system, - nas_get_signal_info = nas_get_signal_info -} diff --git a/src/services/hal/drivers/network.lua b/src/services/hal/drivers/network.lua new file mode 100644 index 00000000..6f047193 --- /dev/null +++ b/src/services/hal/drivers/network.lua @@ -0,0 +1,34 @@ +-- services/hal/drivers/network.lua +-- Thin semantic driver facade around a network backend provider. + +local provider_loader = require 'services.hal.backends.network.provider' + +local M = {} +local Driver = {} +Driver.__index = Driver + +function M.new(config, opts) + local provider, err = provider_loader.new(config or {}, opts or {}) + if not provider then return nil, err end + return setmetatable({ provider = provider }, Driver), nil +end + +function Driver:validate_op(req) return self.provider:validate_op(req) end +function Driver:plan_op(req) return self.provider:plan_op(req) end +function Driver:apply_op(req) return self.provider:apply_op(req) end +function Driver:snapshot_op(req) return self.provider:snapshot_op(req) end +function Driver:watch_op(req) return self.provider:watch_op(req) end +function Driver:probe_link_op(req) return self.provider:probe_link_op(req) end +function Driver:read_counters_op(req) return self.provider:read_counters_op(req) end +function Driver:apply_live_weights_op(req) return self.provider:apply_live_weights_op(req) end +function Driver:apply_shaping_op(req) return self.provider:apply_shaping_op(req) end +function Driver:speedtest_op(req) return self.provider:speedtest_op(req) end + +function Driver:terminate(reason) + if self.provider and type(self.provider.terminate) == 'function' then + return self.provider:terminate(reason) + end + return true, nil +end + +return M diff --git a/src/services/hal/drivers/platform.lua b/src/services/hal/drivers/platform.lua new file mode 100644 index 00000000..d0dc195c --- /dev/null +++ b/src/services/hal/drivers/platform.lua @@ -0,0 +1,290 @@ +-- services/hal/drivers/platform.lua +-- +-- Platform identity HAL driver. +-- Reads hw_revision, fw_version, serial number and board_revision at +-- driver creation and publishes them as a retained state/identity message. +-- Exposes a 'platform' capability with a 'get' RPC offering. +-- Only the 'uptime' field is readable at runtime (from /proc/uptime). + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local file = require "fibers.io.file" +local exec = require "fibers.io.exec" +local channel = require "fibers.channel" + +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" +local cap_args = require "services.hal.types.capability_args" +local cache_mod = require "shared.cache" + +local perform = fibers.perform + +local CONTROL_Q_LEN = 8 + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +---@class PlatformDriver +---@field scope Scope +---@field control_ch Channel +---@field cap_emit_ch Channel? +---@field cache Cache +---@field identity table hw_revision, fw_version, serial, board_revision +---@field logger Logger? +local PlatformDriver = {} +PlatformDriver.__index = PlatformDriver + +---- helpers ---- + +---@param path string +---@return string? content +---@return string error +local function read_file(path) + local f, open_err = file.open(path, 'r') + if not f then + return nil, tostring(open_err) + end + local content, read_err = f:read_all() + f:close() + if not content then + return nil, tostring(read_err) + end + return content, "" +end + +---@param s string +---@return string +local function trim(s) + return (s or ""):match("^%s*(.-)%s*$") or "" +end + +local function copy_plain(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +--- Run fw_printenv and extract a named variable. +---@param varname string +---@return string value +local function read_fw_printenv(varname) + local cmd = exec.command { 'fw_printenv', varname, stdin = 'null', stdout = 'pipe', stderr = 'stdout' } + local out, _, code = perform(cmd:output_op()) + if code ~= 0 or not out then + return "" + end + -- Output format: "varname=value\n" + return trim(out:match(varname .. "=(.+)$") or "") +end + +--- Read all identity fields once at driver creation. +---@return table identity +local function read_identity() + local function safe_read(path) + local v, _ = read_file(path) + return trim(v or "") + end + + local board_revision = read_fw_printenv('board_revision') + + return { + hw_revision = safe_read('/etc/hwrevision'), + fw_version = safe_read('/etc/fwversion'), + serial = safe_read('/data/serial'), + board_revision = board_revision, + } +end + +---- capability verbs ---- + +---@param opts PlatformGetOpts +---@return boolean ok +---@return any value_or_err +function PlatformDriver:get(opts) + if opts == nil or getmetatable(opts) ~= cap_args.PlatformGetOpts then + return false, "invalid opts" + end + local field = opts.field + local max_age = opts.max_age + + if field == 'identity' then + return true, copy_plain(self.identity) + end + + if self.identity and self.identity[field] ~= nil then + return true, self.identity[field] + end + + if field ~= 'uptime' then + return false, "unsupported field: " .. tostring(field) + end + + local cached = self.cache:get('uptime', max_age) + if cached ~= nil then + return true, cached + end + + local raw, err = read_file('/proc/uptime') + if not raw then + return false, "failed to read /proc/uptime: " .. err + end + + local uptime_s = tonumber(raw:match("([%d%.]+)")) + if not uptime_s then + return false, "failed to parse /proc/uptime" + end + + self.cache:set('uptime', uptime_s) + return true, uptime_s +end + +---- control manager ---- + +function PlatformDriver:control_manager() + fibers.current_scope():finally(function() + dlog(self.logger, 'debug', { what = 'control_manager_exiting' }) + end) + + while true do + local request, req_err = self.control_ch:get() + if not request then + dlog(self.logger, 'debug', { what = 'control_ch_closed', err = tostring(req_err) }) + break + end + + local fn = self[request.verb] + local ok, value_or_err + if type(fn) ~= 'function' then + ok, value_or_err = false, "unsupported verb: " .. tostring(request.verb) + else + local st, _, r1, r2 = fibers.run_scope(function() + return fn(self, request.opts) + end) + if st ~= 'ok' then + ok, value_or_err = false, "internal error: " .. tostring(r1) + else + ok, value_or_err = r1, r2 + end + end + + local reply = hal_types.new.Reply(ok, value_or_err) + if reply then + request.reply_ch:put(reply) + end + end +end + +---- public interface ---- + +---@return string error +function PlatformDriver:init() + if self.initialised then + return "already initialised" + end + self.initialised = true + return "" +end + +---@param emit_ch Channel +---@return Capability[]? +---@return string error +function PlatformDriver:capabilities(emit_ch) + if not self.initialised then + return nil, "platform driver not initialised" + end + self.cap_emit_ch = emit_ch + local cap, err = cap_types.new.PlatformCapability('1', self.control_ch) + if not cap then + return {}, err + end + return { cap }, "" +end + +---@return boolean ok +---@return string error +function PlatformDriver:start() + if not self.initialised then + return false, "platform driver not initialised" + end + if self.cap_emit_ch then + -- Publish identity as a retained state sub-topic (consistent with modem state/card). + local state_payload, state_err = hal_types.new.Emit( + 'platform', '1', 'state', 'identity', self.identity) + if state_payload then + self.cap_emit_ch:put(state_payload) + else + dlog(self.logger, 'debug', { what = 'state_identity_emit_failed', err = tostring(state_err) }) + end + + -- Publish meta. + local meta_payload, meta_err = hal_types.new.Emit('platform', '1', 'meta', 'info', { + provider = 'hal', + version = 1, + }) + if meta_payload then + self.cap_emit_ch:put(meta_payload) + else + dlog(self.logger, 'debug', { what = 'meta_emit_failed', err = tostring(meta_err) }) + end + end + + local ok, spawn_err = self.scope:spawn(function() + self:control_manager() + end) + if not ok then + return false, "failed to spawn control_manager: " .. tostring(spawn_err) + end + return true, "" +end + +---@param timeout number? +---@return boolean ok +---@return string error +function PlatformDriver:stop(timeout) + timeout = timeout or 5 + self.scope:cancel('platform driver stopped') + local source = perform(op.named_choice { + join = self.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + if source == 'timeout' then + return false, "platform driver stop timeout" + end + return true, "" +end + +---@param logger Logger? +---@return PlatformDriver? +---@return string error +local function new(logger) + local scope, err = fibers.current_scope():child() + if not scope then + return nil, "failed to create child scope: " .. tostring(err) + end + + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + dlog(logger, 'error', { what = 'scope_failed', err = tostring(primary), status = st }) + end + dlog(logger, 'debug', { what = 'stopped' }) + end) + + local identity = read_identity() + + return setmetatable({ + scope = scope, + control_ch = channel.new(CONTROL_Q_LEN), + cap_emit_ch = nil, + cache = cache_mod.new(), + identity = identity, + logger = logger, + initialised = false, + }, PlatformDriver), "" +end + +return { new = new } diff --git a/src/services/hal/drivers/power.lua b/src/services/hal/drivers/power.lua new file mode 100644 index 00000000..ad431b39 --- /dev/null +++ b/src/services/hal/drivers/power.lua @@ -0,0 +1,211 @@ +-- services/hal/drivers/power.lua +-- +-- Power HAL driver. +-- Exposes a 'power' capability with 'shutdown' and 'reboot' RPC offerings. +-- +-- Reply-before-exec pattern: each verb spawns the exec in a child fiber and +-- returns true immediately so the control_manager sends the reply before the +-- OS executes the power command. + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local exec = require "fibers.io.exec" +local channel = require "fibers.channel" + +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" +local cap_args = require "services.hal.types.capability_args" + +local perform = fibers.perform + +local CONTROL_Q_LEN = 8 + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +-- Delay (seconds) before executing a power command. This gives the reply +-- time to be transmitted to the caller before the system goes down. +local EXEC_DELAY = 1 + +---@class PowerDriver +---@field scope Scope +---@field control_ch Channel +---@field cap_emit_ch Channel? +---@field logger Logger? +local PowerDriver = {} +PowerDriver.__index = PowerDriver + +---- capability verbs ---- + +---@param opts PowerActionOpts? +---@return boolean ok +---@return nil reason +function PowerDriver:shutdown(opts) + if opts ~= nil and getmetatable(opts) ~= cap_args.PowerActionOpts then + return false, "invalid opts" + end + local delay = (opts and opts.delay) or EXEC_DELAY + self.scope:spawn(function() + -- Give caller time to receive the reply before the system shuts down. + perform(sleep.sleep_op(delay)) + dlog(self.logger, 'info', { what = 'executing_shutdown' }) + local cmd = exec.command { 'shutdown', '-h', 'now', stdin = 'null', stdout = 'null', stderr = 'null' } + perform(cmd:run_op()) + end) + return true, nil +end + +---@param opts PowerActionOpts? +---@return boolean ok +---@return nil reason +function PowerDriver:reboot(opts) + if opts ~= nil and getmetatable(opts) ~= cap_args.PowerActionOpts then + return false, "invalid opts" + end + local delay = (opts and opts.delay) or EXEC_DELAY + self.scope:spawn(function() + perform(sleep.sleep_op(delay)) + dlog(self.logger, 'info', { what = 'executing_reboot' }) + local cmd = exec.command { 'reboot', stdin = 'null', stdout = 'null', stderr = 'null' } + perform(cmd:run_op()) + end) + return true, nil +end + +---- control manager ---- + +function PowerDriver:control_manager() + fibers.current_scope():finally(function() + dlog(self.logger, 'debug', { what = 'control_manager_exiting' }) + end) + + while true do + local request, req_err = self.control_ch:get() + if not request then + dlog(self.logger, 'debug', { what = 'control_ch_closed', err = tostring(req_err) }) + break + end + + local fn = self[request.verb] + local ok, value_or_err + if type(fn) ~= 'function' then + ok, value_or_err = false, "unsupported verb: " .. tostring(request.verb) + else + local st, _, r1, r2 = fibers.run_scope(function() + return fn(self, request.opts) + end) + if st ~= 'ok' then + ok, value_or_err = false, "internal error: " .. tostring(r1) + else + ok, value_or_err = r1, r2 + end + end + + -- Reply is sent BEFORE the exec fiber (spawned by shutdown/reboot) runs. + local reply = hal_types.new.Reply(ok, value_or_err) + if reply then + request.reply_ch:put(reply) + end + end +end + +---- public interface ---- + +---@return string error +function PowerDriver:init() + if self.initialised then + return "already initialised" + end + self.initialised = true + return "" +end + +---@param emit_ch Channel +---@return Capability[]? +---@return string error +function PowerDriver:capabilities(emit_ch) + if not self.initialised then + return nil, "power driver not initialised" + end + self.cap_emit_ch = emit_ch + local cap, err = cap_types.new.PowerCapability('1', self.control_ch) + if not cap then + return {}, err + end + return { cap }, "" +end + +---@return boolean ok +---@return string error +function PowerDriver:start() + if not self.initialised then + return false, "power driver not initialised" + end + if self.cap_emit_ch then + local meta_payload, meta_err = hal_types.new.Emit('power', '1', 'meta', 'info', { + provider = 'hal', + version = 1, + }) + if meta_payload then + self.cap_emit_ch:put(meta_payload) + else + dlog(self.logger, 'debug', { what = 'meta_emit_failed', err = tostring(meta_err) }) + end + end + + local ok, spawn_err = self.scope:spawn(function() + self:control_manager() + end) + if not ok then + return false, "failed to spawn control_manager: " .. tostring(spawn_err) + end + return true, "" +end + +---@param timeout number? +---@return boolean ok +---@return string error +function PowerDriver:stop(timeout) + timeout = timeout or 5 + self.scope:cancel('power driver stopped') + local source = perform(op.named_choice { + join = self.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + if source == 'timeout' then + return false, "power driver stop timeout" + end + return true, "" +end + +---@param logger Logger? +---@return PowerDriver? +---@return string error +local function new(logger) + local scope, err = fibers.current_scope():child() + if not scope then + return nil, "failed to create child scope: " .. tostring(err) + end + + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + dlog(logger, 'error', { what = 'scope_failed', err = tostring(primary), status = st }) + end + dlog(logger, 'debug', { what = 'stopped' }) + end) + + return setmetatable({ + scope = scope, + control_ch = channel.new(CONTROL_Q_LEN), + cap_emit_ch = nil, + logger = logger, + initialised = false, + }, PowerDriver), "" +end + +return { new = new } diff --git a/src/services/hal/drivers/radio.lua b/src/services/hal/drivers/radio.lua new file mode 100644 index 00000000..03b30813 --- /dev/null +++ b/src/services/hal/drivers/radio.lua @@ -0,0 +1,634 @@ +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" +local provider = require "services.hal.backends.radio.provider" +local cap_args = require "services.hal.types.capability_args" + +local fibers = require "fibers" +local channel = require "fibers.channel" +local sleep = require "fibers.sleep" + +local CONTROL_Q_LEN = 16 +local DEFAULT_REPORT_PERIOD = 60 -- seconds +local INT32_MAX = 2147483647 + +local VALID_BANDS = { '2g', '5g' } +local VALID_HTMODES = { + 'HE20', 'HE40+', 'HE40-', 'HE80', 'HE160', + 'HT20', 'HT40+', 'HT40-', + 'VHT20', 'VHT40+', 'VHT40-', 'VHT80', 'VHT160', +} +local VALID_ENCRYPTIONS = { + 'none', 'wep', 'psk', 'psk2', 'psk-mixed', + 'sae', 'sae-mixed', 'owe', 'wpa', 'wpa2', 'wpa3', +} +local VALID_MODES = { 'ap', 'sta', 'adhoc', 'mesh', 'monitor' } + +local function is_in(value, list) + for _, v in ipairs(list) do + if v == value then return true end + end + return false +end + +local function is_list(t) + if type(t) ~= 'table' then return false end + local i = 1 + for k in pairs(t) do + if k ~= i then return false end + i = i + 1 + end + return true +end + +local function fix_underflow(n) + if type(n) == 'number' and n > INT32_MAX then + return n - 4294967296 + end + return n +end + +---@class RadioDriver +---@field id string UCI radio section name +---@field scope Scope +---@field control_ch Channel +---@field cap_emit_ch Channel? +---@field staged table In-memory staged config +---@field iface_update_ch Channel Interface add/remove/reset messages for stats_loop +---@field iface_counter number Counter for auto-named interfaces +---@field report_period_ch Channel +---@field initialised boolean +---@field caps_applied boolean +---@field log table +---@field backend table +local RadioDriver = {} +RadioDriver.__index = RadioDriver + +local function emit_event(emit_ch, id, key, data) + local payload = hal_types.new.Emit('radio', id, 'event', key, data) + if payload then emit_ch:put(payload) end +end + +local function emit_state(emit_ch, id, key, data) + local payload = hal_types.new.Emit('radio', id, 'state', key, data) + if payload then emit_ch:put(payload) end +end + +------------------------------------------------------------------------ +-- RPC handler methods (called by control_manager) +------------------------------------------------------------------------ + +---@param opts RadioSetChannelsOpts +---@return boolean ok +---@return string? reason +function RadioDriver:set_channels(opts) + if getmetatable(opts) ~= cap_args.RadioSetChannelsOpts then + opts = opts or {} + local casted, err = cap_args.new.RadioSetChannelsOpts(opts.band, opts.channel, opts.htmode, opts.channels) + if not casted then return false, err end + opts = casted + end + if type(opts.band) ~= 'string' or not is_in(opts.band, VALID_BANDS) then + return false, 'band must be one of: ' .. table.concat(VALID_BANDS, ', ') + end + if type(opts.htmode) ~= 'string' or not is_in(opts.htmode, VALID_HTMODES) then + return false, 'htmode must be one of: ' .. table.concat(VALID_HTMODES, ', ') + end + if opts.channel == 'auto' then + if not is_list(opts.channels) or #opts.channels == 0 then + return false, 'channels must be a non-empty list when channel is "auto"' + end + elseif type(opts.channel) ~= 'number' and type(opts.channel) ~= 'string' then + return false, 'channel must be a number, string, or "auto"' + end + + self.staged.band = opts.band + self.staged.channel = opts.channel + self.staged.htmode = opts.htmode + self.staged.channels = (opts.channel == 'auto') and opts.channels or nil + return true +end + +---@param opts RadioSetTxpowerOpts +---@return boolean ok +---@return string? reason +function RadioDriver:set_txpower(opts) + if getmetatable(opts) ~= cap_args.RadioSetTxpowerOpts then + opts = opts or {} + local casted, err = cap_args.new.RadioSetTxpowerOpts(opts.txpower) + if not casted then return false, err end + opts = casted + end + if type(opts.txpower) ~= 'number' and type(opts.txpower) ~= 'string' then + return false, 'txpower must be a number or string' + end + self.staged.txpower = opts.txpower + return true +end + +---@param opts RadioSetCountryOpts +---@return boolean ok +---@return string? reason +function RadioDriver:set_country(opts) + if getmetatable(opts) ~= cap_args.RadioSetCountryOpts then + opts = opts or {} + local casted, err = cap_args.new.RadioSetCountryOpts(opts.country) + if not casted then return false, err end + opts = casted + end + if type(opts.country) ~= 'string' or #opts.country ~= 2 then + return false, 'country must be a 2-character string' + end + self.staged.country = opts.country:upper() + return true +end + +---@param opts RadioSetEnabledOpts +---@return boolean ok +---@return string? reason +function RadioDriver:set_enabled(opts) + if getmetatable(opts) ~= cap_args.RadioSetEnabledOpts then + opts = opts or {} + local casted, err = cap_args.new.RadioSetEnabledOpts(opts.enabled) + if not casted then return false, err end + opts = casted + end + if type(opts.enabled) ~= 'boolean' then + return false, 'enabled must be a boolean' + end + -- UCI disabled flag is inverted + self.staged.disabled = opts.enabled and '0' or '1' + return true +end + +---@param opts RadioAddInterfaceOpts +---@return boolean ok +---@return string? iface_name generated interface name on success +function RadioDriver:add_interface(opts) + if getmetatable(opts) ~= cap_args.RadioAddInterfaceOpts then + opts = opts or {} + local casted, err = cap_args.new.RadioAddInterfaceOpts( + opts.ssid, opts.encryption, opts.password, opts.network, opts.mode, opts.enable_steering) + if not casted then return false, err end + opts = casted + end + if type(opts.ssid) ~= 'string' or opts.ssid == '' then + return false, 'ssid must be a non-empty string' + end + if type(opts.encryption) ~= 'string' or not is_in(opts.encryption, VALID_ENCRYPTIONS) then + return false, 'encryption must be one of: ' .. table.concat(VALID_ENCRYPTIONS, ', ') + end + if type(opts.password) ~= 'string' then + return false, 'password must be a string' + end + if type(opts.network) ~= 'string' or opts.network == '' then + return false, 'network must be a non-empty string' + end + if type(opts.mode) ~= 'string' or not is_in(opts.mode, VALID_MODES) then + return false, 'mode must be one of: ' .. table.concat(VALID_MODES, ', ') + end + if type(opts.enable_steering) ~= 'boolean' then + return false, 'enable_steering must be a boolean' + end + + local iface_name = self.id .. '_i' .. tostring(self.iface_counter) + self.iface_counter = self.iface_counter + 1 + + table.insert(self.staged.interfaces, { + name = iface_name, + ssid = opts.ssid, + encryption = opts.encryption, + password = opts.password, + network = opts.network, + mode = opts.mode, + enable_steering = opts.enable_steering, + }) + self.iface_update_ch:put({ op = 'add', name = iface_name }) + -- Reply carries the generated interface name in reason + return true, iface_name +end + +---@param opts RadioDeleteInterfaceOpts +---@return boolean ok +---@return string? reason +function RadioDriver:delete_interface(opts) + if getmetatable(opts) ~= cap_args.RadioDeleteInterfaceOpts then + opts = opts or {} + local casted, err = cap_args.new.RadioDeleteInterfaceOpts(opts.interface) + if not casted then return false, err end + opts = casted + end + if type(opts.interface) ~= 'string' or opts.interface == '' then + return false, 'interface must be a non-empty string' + end + -- Remove from staged interfaces list + for i, iface in ipairs(self.staged.interfaces) do + if iface.name == opts.interface then + table.remove(self.staged.interfaces, i) + table.insert(self.staged.deleted_interfaces, opts.interface) + self.iface_update_ch:put({ op = 'remove', name = opts.interface }) + return true + end + end + return false, 'interface not found in staged config: ' .. opts.interface +end + +---@return boolean ok +function RadioDriver:clear_radio_config() + self.staged = { + name = self.id, + path = self.staged.path, + type = self.staged.type, + interfaces = {}, + deleted_interfaces = {}, + } + self.iface_update_ch:put({ op = 'reset' }) + return true +end + +---@param opts RadioSetReportPeriodOpts +---@return boolean ok +---@return string? reason +function RadioDriver:set_report_period(opts) + if getmetatable(opts) ~= cap_args.RadioSetReportPeriodOpts then + opts = opts or {} + local casted, err = cap_args.new.RadioSetReportPeriodOpts(opts.period) + if not casted then return false, err end + opts = casted + end + local period = opts.period + if type(period) ~= 'number' or period <= 0 then + return false, 'period must be a positive number' + end + self.report_period_ch:put(period) + return true +end + +---Clear all UCI config owned by this radio, resetting it to a blank state. +---Resets staged driver state and commits + reloads wireless via the backend. +---@return boolean ok +---@return string? reason +function RadioDriver:clear() + local ok, err = pcall(function() self.backend:clear() end) + if not ok then + return false, tostring(err) + end + self.staged = { + name = self.id, + path = self.staged.path, + type = self.staged.type, + interfaces = {}, + deleted_interfaces = {}, + } + self.iface_update_ch:put({ op = 'reset' }) + self.iface_counter = 0 + return true +end + +---@return boolean ok +---@return string? reason +function RadioDriver:apply() + local ok, err = pcall(function() self.backend:apply(self.staged) end) + if not ok then + return false, tostring(err) + end + -- Reset interface counter on successful apply + self.iface_counter = 0 + return true +end + +---@return boolean ok +function RadioDriver:rollback() + self.staged = { + name = self.id, + path = self.staged.path, + type = self.staged.type, + interfaces = {}, + deleted_interfaces = {}, + } + self.iface_update_ch:put({ op = 'reset' }) + self.iface_counter = 0 + return true +end + +------------------------------------------------------------------------ +-- Control manager: RPC dispatch +------------------------------------------------------------------------ + +local function dispatch_rpc(driver, request) + local fn = driver[request.verb] + local ok, reason + if type(fn) ~= 'function' then + ok, reason = false, 'unknown verb: ' .. tostring(request.verb) + else + local call_ok, r1, r2 = pcall(fn, driver, request.opts) + if not call_ok then + ok, reason = false, tostring(r1) + else + ok, reason = r1, r2 + end + end + local reply = hal_types.new.Reply(ok, reason) + if reply then + request.reply_ch:put(reply) + end +end + +function RadioDriver:control_manager() + fibers.current_scope():finally(function() + self.log:debug({ what = 'radio_driver_stopped', id = self.id }) + end) + + while true do + local source, request = fibers.perform(fibers.named_choice({ + rpc = self.control_ch:get_op(), + cancel = fibers.current_scope():cancel_op(), + })) + + if source == 'cancel' then break end + dispatch_rpc(self, request) + end +end + +------------------------------------------------------------------------ +-- Stats loop: tick and event handlers +------------------------------------------------------------------------ + +---Emit all interface-level stats for one interface name. +local function tick_iface(emit_ch, id, backend, iface_name, connected) + local info, _ = backend:get_iface_info(iface_name) + if info then + if info.txpower ~= nil then + emit_state(emit_ch, id, 'iface_power', { + interface = iface_name, + value = fix_underflow(info.txpower), + }) + end + if info.channel ~= nil then + emit_state(emit_ch, id, 'iface_channel', { + interface = iface_name, + value = fix_underflow(info.channel), + }) + end + end + + local noise, _ = backend:get_iface_survey(iface_name) + if noise ~= nil then + emit_state(emit_ch, id, 'iface_noise', { + interface = iface_name, + value = fix_underflow(noise), + }) + end + + for _, stat in ipairs(backend.SYSFS_STATS or {}) do + local sval, _ = backend:read_sysfs_stat(iface_name, stat) + if sval ~= nil then + emit_state(emit_ch, id, 'iface_' .. stat, { + interface = iface_name, + value = fix_underflow(sval), + }) + end + end + + if connected[iface_name] then + for mac in pairs(connected[iface_name]) do + local sta, _ = backend:get_station_info(iface_name, mac) + if sta then + if sta.signal ~= nil then + emit_state(emit_ch, id, 'client_signal', { + mac = mac, + interface = iface_name, + value = fix_underflow(sta.signal), + }) + end + if sta.tx_bytes ~= nil then + emit_state(emit_ch, id, 'client_tx_bytes', { + mac = mac, + interface = iface_name, + value = fix_underflow(sta.tx_bytes), + }) + end + if sta.rx_bytes ~= nil then + emit_state(emit_ch, id, 'client_rx_bytes', { + mac = mac, + interface = iface_name, + value = fix_underflow(sta.rx_bytes), + }) + end + end + end + end +end + +---Emit stats for every currently-tracked interface. +local function on_tick(emit_ch, id, backend, interfaces_set, connected) + for iface_name in pairs(interfaces_set) do + tick_iface(emit_ch, id, backend, iface_name, connected) + end +end + +---Update connected-station bookkeeping and emit a client_event. +local function on_client_event(emit_ch, id, connected, ev) + if not ev then return end + local mac = ev.mac + local iface = ev.interface + + if not connected[iface] then connected[iface] = {} end + + if ev.added then + connected[iface][mac] = true + else + connected[iface][mac] = nil + end + + + emit_event(emit_ch, id, 'client_event', { + mac = mac, + connected = ev.added, + interface = iface, + timestamp = os.time(), + }) +end + +------------------------------------------------------------------------ +-- Stats loop fiber +------------------------------------------------------------------------ + +function RadioDriver:stats_loop() + local emit_ch = self.cap_emit_ch + local id = self.id + local report_period = DEFAULT_REPORT_PERIOD + local backend = self.backend + local connected = {} + local interfaces_set = {} + + backend:start_client_monitor() + + fibers.current_scope():finally(function() + if backend and backend.terminate then + backend:terminate('radio stats loop stopped') + end + self.log:debug({ what = 'radio_stats_loop_stopped', id = id }) + end) + + local function update_interfaces(op, name) + if op == 'add' then + interfaces_set[name] = true + -- Seed connected set with any clients already associated before startup + local macs, _ = backend:get_connected_macs(name) + for _, mac in ipairs(macs) do + on_client_event(emit_ch, id, connected, { mac = mac, interface = name, added = true }) + end + elseif op == 'remove' then + interfaces_set[name] = nil + elseif op == 'reset' then + interfaces_set = {} + end + end + + while true do + local name, val = fibers.perform(fibers.named_choice({ + client_event = backend:watch_clients_op(), + iface_update = self.iface_update_ch:get_op(), + tick = sleep.sleep_op(report_period), + new_period = self.report_period_ch:get_op(), + cancel = fibers.current_scope():cancel_op(), + })) + + if name == 'cancel' then + break + elseif name == 'new_period' then + report_period = val + elseif name == 'iface_update' then + update_interfaces(val.op, val.name) + elseif name == 'client_event' then + if val and interfaces_set[val.interface] then + on_client_event(emit_ch, id, connected, val) + end + elseif name == 'tick' then + on_tick(emit_ch, id, backend, interfaces_set, connected) + end + end + + if backend and backend.stop_client_monitor_op then + fibers.perform(backend:stop_client_monitor_op(0.2)) + end +end + +------------------------------------------------------------------------ +-- Public driver interface +------------------------------------------------------------------------ + +---@param path string hardware sysfs path for this radio +---@param rtype string driver type (e.g. "mac80211") +---@return string err empty string on success +function RadioDriver:init(path, rtype) + if not path or path == '' then + return "init failed: path is required" + end + self.staged.path = path + self.staged.type = rtype or '' + self.initialised = true + return "" +end + +---@param emit_ch Channel +---@return Capability[]? caps +---@return string err +function RadioDriver:capabilities(emit_ch) + if not self.initialised then + return nil, "driver not initialised" + end + if self.caps_applied then + return nil, "capabilities already applied" + end + self.cap_emit_ch = emit_ch + + local cap, cap_err = cap_types.new.Capability( + 'radio', + self.id, + self.control_ch, + { + 'set_channels', + 'set_txpower', + 'set_country', + 'set_enabled', + 'add_interface', + 'delete_interface', + 'clear_radio_config', + 'set_report_period', + 'clear', + 'apply', + 'rollback', + } + ) + if not cap then + return nil, cap_err + end + + self.caps_applied = true + return { cap }, "" +end + +---@return boolean ok +---@return string err +function RadioDriver:start() + if not self.initialised then + return false, "driver not initialised" + end + if not self.caps_applied then + return false, "capabilities not applied" + end + + self.scope:spawn(function() self:control_manager() end) + self.scope:spawn(function() self:stats_loop() end) + return true, "" +end + +---Create a new RadioDriver instance. +---@param name string UCI radio section name (e.g. "radio0") +---@param logger table +---@return RadioDriver? driver +---@return string err +local function new(name, logger) + if type(name) ~= 'string' or name == '' then + return nil, "invalid radio name" + end + + local bknd, berr = provider.new(name) + if not bknd then + return nil, "no radio backend: " .. tostring(berr) + end + + local scope, serr = fibers.current_scope():child() + if not scope then + return nil, "failed to create child scope: " .. tostring(serr) + end + + local driver = setmetatable({ + id = name, + scope = scope, + control_ch = channel.new(CONTROL_Q_LEN), + cap_emit_ch = nil, + iface_update_ch = channel.new(16), + report_period_ch = channel.new(1), + staged = { + name = name, + path = '', + type = '', + interfaces = {}, + deleted_interfaces = {}, + }, + iface_counter = 0, + initialised = false, + caps_applied = false, + log = logger, + backend = bknd, + }, RadioDriver) + + return driver, "" +end + +return { + new = new, + Driver = RadioDriver, +} diff --git a/src/services/hal/drivers/signature_verify_openssl.lua b/src/services/hal/drivers/signature_verify_openssl.lua new file mode 100644 index 00000000..f0965103 --- /dev/null +++ b/src/services/hal/drivers/signature_verify_openssl.lua @@ -0,0 +1,156 @@ +---@module 'services.hal.drivers.signature_verify_openssl' + +local fibers = require 'fibers' +local op = require 'fibers.op' +local file = require 'fibers.io.file' +local exec = require 'fibers.io.exec' +local resource = require 'devicecode.support.resource' + +local M = {} + +---@class SignatureVerifyOpenSSL +---@field file any +---@field exec any +---@field tmpdir string +---@field backend_name string +local Driver = {} +Driver.__index = Driver + +local function tmpfile_path(stream) + if stream and type(stream.filename) == 'function' then + return stream:filename() + end + return nil +end + + +local function classify_verify_failure(detail) + local low = tostring(detail or ''):lower() + return low:find('signature verification failure', 1, true) ~= nil + or low:find('signature verify failure', 1, true) ~= nil + or low:find('verification failure', 1, true) ~= nil +end + +function Driver:verify_ed25519_op(pubkey_pem, message, signature) + return op.guard(function () + if type(pubkey_pem) ~= 'string' or pubkey_pem == '' then + return op.always(false, 'public_key_required') + end + if type(message) ~= 'string' then + return op.always(false, 'message_required') + end + if type(signature) ~= 'string' or signature == '' then + return op.always(false, 'signature_required') + end + + return fibers.run_scope_op(function (scope) + local function register_tmp(stream) + scope:finally(function (_, status, primary) + resource.terminate_checked(stream, primary or status or 'signature verify tmpfile closed', 'signature verify tmpfile cleanup failed') + end) + end + + local function write_tmp_bytes(label, data) + -- Box the currently synchronous tmpfile creation inside one + -- operation-owned subtree. Callers still receive a real Op. + local stream, err = self.file.tmpfile(384, self.tmpdir) + if not stream then + return nil, label .. '_tmpfile_failed:' .. tostring(err) + end + + register_tmp(stream) + + local path = tmpfile_path(stream) + if type(path) ~= 'string' or path == '' then + return nil, label .. '_tmpfile_path_unavailable' + end + + local n, werr = fibers.perform(stream:write_op(data)) + if n == nil then + return nil, label .. '_write_failed:' .. tostring(werr or 'write_failed') + end + + local fok, ferr = fibers.perform(stream:flush_op()) + if fok == nil then + return nil, label .. '_flush_failed:' .. tostring(ferr or 'flush_failed') + end + + return { stream = stream, path = path }, nil + end + + local pubf, perr = write_tmp_bytes('pubkey', pubkey_pem) + if not pubf then + return false, perr + end + + local msgf, merr = write_tmp_bytes('message', message) + if not msgf then + return false, merr + end + + local sigf, serr = write_tmp_bytes('signature', signature) + if not sigf then + return false, serr + end + + local cmd = self.exec.command { + 'openssl', + 'pkeyutl', + '-verify', + '-pubin', + '-inkey', pubf.path, + '-sigfile', sigf.path, + '-in', msgf.path, + '-rawin', + stdin = 'null', + stdout = 'pipe', + stderr = 'stdout', + } + + local out, st, code, sig, cerr = fibers.perform(cmd:combined_output_op()) + local detail = tostring(cerr or out or '') + + if st == 'exited' and code == 0 then + return true, nil + end + + if st == 'exited' and code == 1 and classify_verify_failure(detail) then + return false, 'signature_verify_failed' + end + + if st == 'signalled' then + return false, 'openssl_signalled:' .. tostring(sig) + end + + if st == 'exited' then + if detail == '' then + detail = 'exit_' .. tostring(code) + end + return false, 'openssl_verify_failed:' .. detail + end + + if detail == '' then + detail = tostring(st or 'unknown') + end + return false, 'openssl_verify_failed:' .. detail + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) + end) +end + +function M.new(opts) + opts = opts or {} + return setmetatable({ + file = opts.file or file, + exec = opts.exec or exec, + tmpdir = opts.tmpdir or os.getenv('TMPDIR') or '/tmp', + backend_name = 'openssl-cli', + }, Driver) +end + +M.Driver = Driver +return M diff --git a/src/services/hal/drivers/signature_verify_provider.lua b/src/services/hal/drivers/signature_verify_provider.lua new file mode 100644 index 00000000..1d5733fd --- /dev/null +++ b/src/services/hal/drivers/signature_verify_provider.lua @@ -0,0 +1,429 @@ +---@module 'services.hal.drivers.signature_verify_provider' + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local channel = require 'fibers.channel' + +local hal_types = require 'services.hal.types.core' +local cap_types = require 'services.hal.types.capabilities' +local cap_args = require 'services.hal.types.capability_args' +local control_loop = require 'services.hal.support.control_loop' +local tablex = require 'shared.table' +local scoped_work = require 'devicecode.support.scoped_work' + +local backend_mod = require 'services.hal.drivers.signature_verify_openssl' + +local M = {} + +local CONTROL_Q_LEN = 8 +local DONE_Q_LEN = 32 +local DEFAULT_STOP_TIMEOUT = 5.0 +local DEFAULT_MAX_IN_FLIGHT = 4 + +---@class SignatureVerifyDone +---@field reply_ch Channel +---@field ok boolean +---@field value_or_err any + +---@class SignatureVerifyProvider +---@field id string +---@field opts table +---@field logger table|nil +---@field scope Scope|nil +---@field control_ch Channel +---@field done_ch Channel +---@field emit_ch Channel|nil +---@field backend SignatureVerifyOpenSSL +---@field started boolean +---@field caps_applied boolean +---@field in_flight integer +---@field max_in_flight integer +local Driver = {} +Driver.__index = Driver + +local deep_copy = tablex.deep_copy + +local function dlog(self, level, payload) + if self.logger and self.logger[level] then + self.logger[level](self.logger, payload) + end +end + +local function finalise_shell_scope(self, shell_scope) + if self.scope ~= shell_scope then + return + end + self.started = false + self.scope = nil + self.in_flight = 0 +end + +local function normalise_max_in_flight(v) + if v == nil then + return DEFAULT_MAX_IN_FLIGHT + end + + if type(v) ~= 'number' then + error('signature_verify_provider.new: max_in_flight must be a number', 2) + end + + if v ~= v or v == math.huge or v == -math.huge then + error('signature_verify_provider.new: max_in_flight must be finite', 2) + end + + if v % 1 ~= 0 then + error('signature_verify_provider.new: max_in_flight must be an integer', 2) + end + + if v < 1 then + error('signature_verify_provider.new: max_in_flight must be >= 1', 2) + end + + return v +end + +local function emit_op(emit_ch, class, id, mode, key, data) + return op.guard(function () + local payload, err = hal_types.new.Emit(class, id, mode, key, data) + if not payload then + return op.always(false, tostring(err)) + end + + return emit_ch:put_op(payload):wrap(function () + return true, nil + end) + end) +end + +local function reply_now(self, reply_ch, ok, value_or_err, cancel_op) + local sent, err + if cancel_op ~= nil then + local which, a, b = fibers.perform(fibers.named_choice({ + reply = control_loop.reply_op(reply_ch, ok, value_or_err), + cancel = cancel_op, + })) + if which == 'cancel' and a ~= nil and a ~= false then return true, nil end + sent, err = a, b + else + sent, err = fibers.perform(control_loop.reply_op(reply_ch, ok, value_or_err)) + end + if not sent then + dlog(self, 'warn', { + what = 'signature_verify_reply_failed', + err = tostring(err), + }) + end + return sent, err +end + +local function try_put_now(ch, value, label) + local ok, err = fibers.perform(ch:put_op(value):or_else(function () + return nil, 'not_ready' + end)) + + -- channel:put_op returns no values on successful rendezvous/admission. + if (ok == true) or (ok == nil and err == nil) then + return true, nil + end + + return nil, tostring(label or 'put_now_failed') .. ': ' .. tostring(err or 'closed') +end + +local function validate_verify_opts(opts) + if type(opts) ~= 'table' or getmetatable(opts) ~= cap_args.SignatureVerifyEd25519Opts then + return false, 'invalid verify_ed25519 opts' + end + return true, nil +end + +local function spawn_verify_worker(self, request) + local ok_opts, opts_err = validate_verify_opts(request.opts) + if not ok_opts then + reply_now(self, request.reply_ch, false, opts_err, request.cancel_op) + return + end + + if self.in_flight >= self.max_in_flight then + reply_now(self, request.reply_ch, false, 'busy', request.cancel_op) + return + end + + local shell_scope = assert(self.scope, 'signature_verify worker spawn without scope') + local opts = request.opts + local counted = true + + self.in_flight = self.in_flight + 1 + + local function release_slot() + if counted then + counted = false + if self.in_flight > 0 then + self.in_flight = self.in_flight - 1 + end + end + end + + local handle, err = scoped_work.start { + lifetime_scope = shell_scope, + reaper_scope = shell_scope, + report_scope = shell_scope, + + identity = { + kind = 'verify_done', + reply_ch = request.reply_ch, + cancel_op = request.cancel_op, + }, + + run = function (verify_scope) + verify_scope:finally(function () + release_slot() + end) + + local vok, verr = fibers.perform(self.backend:verify_ed25519_op( + opts.pubkey_pem, + opts.message, + opts.signature + )) + + return { + ok = vok == true, + err = verr, + } + end, + + report = function (ev) + return try_put_now(self.done_ch, ev, 'signature_verify_completion_report_failed') + end, + cancel_op = request.cancel_op, + } + + if not handle then + release_slot() + reply_now(self, request.reply_ch, false, tostring(err or 'verify_worker_start_failed'), request.cancel_op) + end +end + +local function handle_request(self, request) + if not request then + return false + end + + if request.verb == 'verify_ed25519' then + spawn_verify_worker(self, request) + return true + end + + reply_now(self, request.reply_ch, false, 'unsupported verb: ' .. tostring(request.verb), request.cancel_op) + return true +end + +local function handle_done(self, done) + if not done then + return false + end + + if done.kind ~= 'verify_done' then + return true + end + + if done.status == 'ok' then + local result = done.result or {} + reply_now(self, done.reply_ch, result.ok == true, result.err, done.cancel_op) + else + reply_now(self, done.reply_ch, false, tostring(done.primary or done.status or 'verify_failed'), done.cancel_op) + end + + return true +end + +local function shell_main(self) + local shell_scope = assert(self.scope, 'signature_verify shell without scope') + assert(self.emit_ch, 'signature_verify shell without emit channel') + + shell_scope:finally(function () + finalise_shell_scope(self, shell_scope) + end) + + local meta_ok, meta_err = fibers.perform(emit_op( + self.emit_ch, + 'signature_verify', + self.id, + 'meta', + 'info', + { + provider = 'hal.signature_verify', + backend = self.backend.backend_name or 'openssl-cli', + version = 2, + max_in_flight = self.max_in_flight, + } + )) + if meta_ok ~= true then + error(tostring(meta_err or 'initial meta emit failed'), 0) + end + + local st_ok, st_err = fibers.perform(emit_op( + self.emit_ch, + 'signature_verify', + self.id, + 'state', + 'status', + { state = 'available' } + )) + if st_ok ~= true then + error(tostring(st_err or 'initial state emit failed'), 0) + end + + while true do + local which, a, b = fibers.perform(fibers.named_choice{ + req = self.control_ch:get_op(), + done = self.done_ch:get_op(), + }) + + if which == 'done' then + local done = a + if not done then + return + end + handle_done(self, done) + elseif which == 'req' then + local request = a + if not request then + return + end + handle_request(self, request) + end + end +end + +function Driver:capabilities_op(emit_ch) + return op.guard(function () + if self.caps_applied then + return op.always(false, 'capabilities already applied') + end + + self.emit_ch = emit_ch + + local cap, err = cap_types.new.SignatureVerifyCapability(self.id, self.control_ch) + if not cap then + return op.always(false, tostring(err)) + end + + self.caps_applied = true + return op.always(true, { cap }) + end) +end + +---@param owner_scope Scope +function Driver:start_op(owner_scope) + assert(owner_scope ~= nil, 'signature_verify provider start_op: owner_scope is required') + + return op.guard(function () + if self.started then + return op.always(false, 'already started') + end + if not self.caps_applied then + return op.always(false, 'capabilities not applied') + end + if not self.emit_ch then + return op.always(false, 'missing emit channel') + end + + local shell_scope, err = owner_scope:child() + if not shell_scope then + return op.always(false, tostring(err)) + end + + self.scope = shell_scope + + local ok, serr = shell_scope:spawn(function () + return shell_main(self) + end) + if not ok then + self.scope = nil + shell_scope:cancel(tostring(serr or 'signature_verify shell spawn failed')) + return op.always(false, tostring(serr)) + end + + self.started = true + return op.always(true, nil) + end) +end + +function Driver:terminate(reason) + if self.scope then + self.scope:cancel(reason or 'signature_verify provider terminated') + end + self.started = false + self.scope = nil + self.in_flight = 0 + return true, nil +end + +function Driver:shutdown_op(timeout) + timeout = timeout or DEFAULT_STOP_TIMEOUT + + return op.guard(function () + if not self.started or not self.scope then + return op.always(true, nil) + end + + local shell_scope = self.scope + shell_scope:cancel('signature_verify provider stopped') + + return fibers.boolean_choice( + shell_scope:join_op():wrap(function () + finalise_shell_scope(self, shell_scope) + return true, nil + end), + sleep.sleep_op(timeout):wrap(function () + return false, 'signature_verify provider stop timeout' + end) + ):wrap(function (completed, _a, b) + if completed then + return true, nil + end + return false, b + end) + end) +end + +function Driver:fault_op() + if self.scope and self.started then + return self.scope:fault_op() + end + return op.never() +end + +---@param id string +---@param opts table|nil +---@param logger table|nil +---@return SignatureVerifyProvider +function M.new(id, opts, logger) + assert(type(id) == 'string' and id ~= '', 'signature_verify_provider.new: invalid id') + + local raw_opts = opts or {} + local injected_backend = raw_opts.backend + + opts = deep_copy(raw_opts) + + local max_in_flight = normalise_max_in_flight(opts.max_in_flight) + + return setmetatable({ + id = id, + opts = opts, + logger = logger, + scope = nil, + control_ch = channel.new(CONTROL_Q_LEN), + done_ch = channel.new(DONE_Q_LEN), + emit_ch = nil, + backend = injected_backend or backend_mod.new(opts), + started = false, + caps_applied = false, + in_flight = 0, + max_in_flight = max_in_flight, + }, Driver) +end + +M.Driver = Driver +return M diff --git a/src/services/hal/drivers/thermal.lua b/src/services/hal/drivers/thermal.lua new file mode 100644 index 00000000..5d87d635 --- /dev/null +++ b/src/services/hal/drivers/thermal.lua @@ -0,0 +1,247 @@ +-- services/hal/drivers/thermal.lua +-- +-- Thermal zone HAL driver. +-- One driver instance per sysfs thermal zone (e.g. /sys/class/thermal/thermal_zone0). +-- Exposes a 'thermal' capability with a 'get' RPC offering. +-- Zone type is read once at startup for meta. Temperature is read on +-- cache miss by reading the sysfs 'temp' file (millidegrees → degrees C). + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local file = require "fibers.io.file" +local channel = require "fibers.channel" + +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" +local cap_args = require "services.hal.types.capability_args" +local cache_mod = require "shared.cache" + +local perform = fibers.perform + +local CONTROL_Q_LEN = 8 + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +---@class ThermalDriver +---@field zone_id string e.g. "zone0" +---@field sysfs_dir string e.g. "/sys/class/thermal/thermal_zone0" +---@field zone_type string? read from sysfs 'type' file +---@field scope Scope +---@field control_ch Channel +---@field cap_emit_ch Channel? +---@field cache Cache +---@field logger Logger? +local ThermalDriver = {} +ThermalDriver.__index = ThermalDriver + +---- helpers ---- + +---@param path string +---@return string? content +---@return string error +local function read_file(path) + local f, open_err = file.open(path, 'r') + if not f then + return nil, tostring(open_err) + end + local content, read_err = f:read_all() + f:close() + if not content then + return nil, tostring(read_err) + end + return content, "" +end + +--- Read the zone's type string (e.g. "cpu-thermal"). +---@param sysfs_dir string +---@param logger Logger? +---@return string? zone_type +local function read_zone_type(sysfs_dir, logger) + local raw, err = read_file(sysfs_dir .. '/type') + if not raw then + dlog(logger, 'debug', { what = 'zone_type_read_failed', err = tostring(err), path = sysfs_dir .. '/type' }) + return nil + end + return raw:match("^%s*(.-)%s*$") +end + +---- capability verbs ---- + +---@param opts ThermalGetOpts +---@return boolean ok +---@return any value_or_err +function ThermalDriver:get(opts) + if opts == nil or getmetatable(opts) ~= cap_args.ThermalGetOpts then + return false, "invalid opts" + end + local max_age = opts.max_age + + local cached = self.cache:get('temp', max_age) + if cached ~= nil then + return true, cached + end + + local raw, err = read_file(self.sysfs_dir .. '/temp') + if not raw then + return false, "failed to read temperature: " .. err + end + + local millideg = tonumber(raw:match("%d+")) + if not millideg then + return false, "failed to parse temperature value" + end + + local temp_c = millideg / 1000 + self.cache:set('temp', temp_c) + return true, temp_c +end + +---- control manager ---- + +function ThermalDriver:control_manager() + fibers.current_scope():finally(function() + dlog(self.logger, 'debug', { what = 'control_manager_exiting' }) + end) + + while true do + local request, req_err = self.control_ch:get() + if not request then + dlog(self.logger, 'debug', { what = 'control_ch_closed', err = tostring(req_err) }) + break + end + + local fn = self[request.verb] + local ok, value_or_err + if type(fn) ~= 'function' then + ok, value_or_err = false, "unsupported verb: " .. tostring(request.verb) + else + local st, _, r1, r2 = fibers.run_scope(function() + return fn(self, request.opts) + end) + if st ~= 'ok' then + ok, value_or_err = false, "internal error: " .. tostring(r1) + else + ok, value_or_err = r1, r2 + end + end + + local reply = hal_types.new.Reply(ok, value_or_err) + if reply then + request.reply_ch:put(reply) + end + end +end + +---- public interface ---- + +---@return string error +function ThermalDriver:init() + if self.initialised then + return "already initialised" + end + self.initialised = true + return "" +end + +---@param emit_ch Channel +---@return Capability[]? +---@return string error +function ThermalDriver:capabilities(emit_ch) + if not self.initialised then + return nil, "thermal driver not initialised" + end + self.cap_emit_ch = emit_ch + local cap, err = cap_types.new.ThermalCapability(self.zone_id, self.control_ch) + if not cap then + return {}, err + end + return { cap }, "" +end + +---@return boolean ok +---@return string error +function ThermalDriver:start() + if not self.initialised then + return false, "thermal driver not initialised" + end + local meta_payload, emit_err = hal_types.new.Emit('thermal', self.zone_id, 'meta', 'info', { + provider = 'hal', + version = 1, + zone = self.zone_id, + path = self.sysfs_dir, + zone_type = self.zone_type, + }) + if meta_payload and self.cap_emit_ch then + self.cap_emit_ch:put(meta_payload) + elseif not meta_payload then + dlog(self.logger, 'debug', { what = 'meta_emit_failed', err = tostring(emit_err) }) + end + + local ok, spawn_err = self.scope:spawn(function() + self:control_manager() + end) + if not ok then + return false, "failed to spawn control_manager: " .. tostring(spawn_err) + end + return true, "" +end + +---@param timeout number? +---@return boolean ok +---@return string error +function ThermalDriver:stop(timeout) + timeout = timeout or 5 + self.scope:cancel(('thermal driver [%s] stopped'):format(self.zone_id)) + local source = perform(op.named_choice { + join = self.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + if source == 'timeout' then + return false, ("thermal driver [%s] stop timeout"):format(self.zone_id) + end + return true, "" +end + +---@param zone_id string canonical zone id, e.g. "zone0" +---@param sysfs_dir string full path to zone sysfs dir, e.g. "/sys/class/thermal/thermal_zone0" +---@param logger Logger? +---@return ThermalDriver? +---@return string error +local function new(zone_id, sysfs_dir, logger) + assert(type(zone_id) == 'string' and zone_id ~= '', "zone_id must be a non-empty string") + assert(type(sysfs_dir) == 'string' and sysfs_dir ~= '', "sysfs_dir must be a non-empty string") + + local scope, err = fibers.current_scope():child() + if not scope then + return nil, "failed to create child scope: " .. tostring(err) + end + + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + dlog(logger, 'error', { what = 'scope_failed', err = tostring(primary), status = st }) + end + dlog(logger, 'debug', { what = 'stopped' }) + end) + + local zone_type = read_zone_type(sysfs_dir, logger) + + return setmetatable({ + zone_id = zone_id, + sysfs_dir = sysfs_dir, + zone_type = zone_type, + scope = scope, + control_ch = channel.new(CONTROL_Q_LEN), + cap_emit_ch = nil, + cache = cache_mod.new(), + logger = logger, + initialised = false, + }, ThermalDriver), "" +end + +return { new = new } diff --git a/src/services/hal/drivers/time.lua b/src/services/hal/drivers/time.lua new file mode 100644 index 00000000..39238818 --- /dev/null +++ b/src/services/hal/drivers/time.lua @@ -0,0 +1,337 @@ +-- HAL modules +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" + +-- Backend modules +local time_backend_provider = require "services.hal.backends.time.provider" + +-- Fibers modules +local fibers = require "fibers" +local op = require "fibers.op" +local channel = require "fibers.channel" +local sleep = require "fibers.sleep" +local exec = require "fibers.io.exec" + +-- Other modules +local uuid = require "uuid" + +---@class TimeDriver +---@field id CapabilityId UUID for this time source capability +---@field cap_emit_ch Channel Capability emit channel (Emit messages) +---@field scope Scope Child scope owning the monitor fiber +---@field backend TimeBackend Time backend for platform-specific NTP monitoring +---@field control_ch Channel RPC control channel (no offerings currently, reserved) +---@field logger Logger +---@field initialised boolean +---@field caps_applied boolean +---@field synced boolean Tracks last known sync state to detect transitions +local TimeDriver = {} +TimeDriver.__index = TimeDriver + +---- Constants ---- + +local DEFAULT_STOP_TIMEOUT = 5 +local CONTROL_Q_LEN = 4 + +---- Internal Utilities ---- + +---@param self TimeDriver +---@param level string +---@param payload any +local function dlog(self, level, payload) + if self.logger and self.logger[level] then + self.logger[level](self.logger, payload) + end +end + +---Emit a capability state, meta, or event via the cap emit channel. +---@param emit_ch Channel +---@param id CapabilityId +---@param mode EmitMode +---@param key string +---@param data any +---@return boolean ok +---@return string? error +local function emit(emit_ch, id, mode, key, data) + local payload, err = hal_types.new.Emit('time', id, mode, key, data) + if not payload then + return false, err + end + emit_ch:put(payload) + return true +end + +---Convert NTP stratum to an estimated absolute accuracy in seconds. +---Returns nil when unsynced (stratum >= 16) or invalid input. +---@param stratum number +---@return number? accuracy_seconds +local function accuracy_for_stratum(stratum) + if type(stratum) ~= 'number' then + return nil + end + if stratum >= 16 then + return nil + end + + -- Coarse operational heuristic: + -- lower stratum generally implies lower clock error. + if stratum <= 1 then + return 0.001 + elseif stratum <= 4 then + return 0.01 + elseif stratum <= 8 then + return 0.1 + else + return 1.0 + end +end + +---Build a meta payload table for this time source. +---@param accuracy_seconds number? +---@return table +local function build_meta(accuracy_seconds) + return { + provider = 'hal', + source = 'ntp', + version = 1, + accuracy_seconds = accuracy_seconds, + } +end + +---- Monitor Fiber ---- + +---Listen to NTP synchronization events via the time backend and emit state/events via +---the cap emit channel. Runs in self.scope. Exits on stream close, read error, or scope +---cancellation. Transition events (synced/unsynced) are non-retained; the current +---sync state is always published as a retained state emit on every hotplug event. +---@return nil +function TimeDriver:_ntpd_monitor() + fibers.current_scope():finally(function() + if self.backend and self.backend.terminate then + self.backend:terminate('time monitor stopped') + end + dlog(self, 'debug', { what = 'ntpd_monitor_exit' }) + end) + + -- Start the backend monitor here so the ubus command is bound to this fiber's + -- scope (the driver child scope). Cancelling the driver scope will then kill + -- the underlying process automatically. + local ok, start_err = self.backend:start_ntp_monitor() + if not ok then + dlog(self, 'error', { what = 'ntp_backend_start_failed', err = tostring(start_err) }) + return + end + + dlog(self, 'debug', { what = 'ntpd_monitor_started' }) + + while true do + local ntp_event, err = fibers.perform(self.backend:ntp_event_op()) + if err ~= nil then + -- Fatal: stream closed or read error + dlog(self, 'warn', { what = 'ntp_event_stream_closed', err = tostring(err) }) + break + end + if ntp_event then + local stratum = ntp_event.stratum + -- NTPEvent constructor guarantees stratum is a number, but guard anyway + if type(stratum) ~= 'number' then + dlog(self, 'warn', { what = 'ntp_event_invalid_stratum', stratum = tostring(stratum) }) + else + local now_synced = stratum ~= 16 + local was_synced = self.synced + local accuracy_seconds = accuracy_for_stratum(stratum) + + -- Always update retained state, even if sync status did not change, + -- so that the latest stratum value is always visible to subscribers. + local emit_ok, emit_err = emit( + self.cap_emit_ch, self.id, 'state', 'synced', + { + synced = now_synced, + stratum = stratum, + accuracy_seconds = accuracy_seconds, + } + ) + if not emit_ok then + dlog(self, 'warn', { what = 'emit_state_failed', err = tostring(emit_err) }) + end + + -- Update accuracy metadata only on a sync/unsync transition. + if now_synced ~= was_synced then + local meta_ok, meta_err = emit( + self.cap_emit_ch, self.id, 'meta', 'source', + build_meta(accuracy_seconds) + ) + if not meta_ok then + dlog(self, 'warn', { what = 'emit_meta_failed', err = tostring(meta_err) }) + end + end + + -- Emit non-retained transition events. + if now_synced and not was_synced then + dlog(self, 'debug', { what = 'ntp_synced', stratum = stratum }) + local ev_ok, ev_err = emit( + self.cap_emit_ch, self.id, 'event', 'synced', + { + stratum = stratum, + accuracy_seconds = accuracy_seconds, + } + ) + if not ev_ok then + dlog(self, 'warn', { what = 'emit_synced_event_failed', err = tostring(ev_err) }) + end + elseif not now_synced and was_synced then + dlog(self, 'debug', { what = 'ntp_unsynced', stratum = stratum }) + local ev_ok, ev_err = emit( + self.cap_emit_ch, self.id, 'event', 'unsynced', + { + stratum = stratum, + accuracy_seconds = accuracy_seconds, + } + ) + if not ev_ok then + dlog(self, 'warn', { what = 'emit_unsynced_event_failed', err = tostring(ev_err) }) + end + end + + self.synced = now_synced + end + end + -- ntp_event == nil and err == nil cannot occur: all parse failures are fatal + end +end + +---- Driver Lifecycle ---- + +---Initialise the time driver. Restarts sysntpd and marks the driver as initialised. +---Must be called from inside a fiber. +---@return string error Empty string on success. +function TimeDriver:init() + dlog(self, 'debug', { what = 'init_begin' }) + + local status, code, _, err = fibers.perform( + (exec.command { "/etc/init.d/sysntpd", "restart", stdin = "null", stdout = "null", stderr = "null" }):run_op() + ) + if status ~= 'exited' or code ~= 0 then + return "sysntpd restart failed: " .. tostring(err or ("exit code " .. tostring(code))) + end + + self.initialised = true + dlog(self, 'debug', { what = 'init_done' }) + return "" +end + +---Connect the driver to the capability emit channel and return the capability list. +---Must be called after init() and before start(). +---@param cap_emit_ch Channel +---@return Capability[]? capabilities +---@return string error Empty string on success. +function TimeDriver:capabilities(cap_emit_ch) + if not self.initialised then + return nil, "driver not initialised" + end + + self.cap_emit_ch = cap_emit_ch + + local cap, cap_err = cap_types.new.TimeCapability(self.id, self.control_ch) + if not cap then + return nil, "failed to create time capability: " .. tostring(cap_err) + end + + self.caps_applied = true + return { cap }, "" +end + +---Start the time driver. Emits initial meta and state, then spawns the NTP monitor +---fiber. Must be called after capabilities(). +---@return boolean ok +---@return string? error +function TimeDriver:start() + if not self.initialised then + return false, "driver not initialised" + end + if not self.caps_applied then + return false, "capabilities not applied" + end + + -- Publish initial meta (accuracy unknown until first NTP update). + local meta_ok, meta_err = emit( + self.cap_emit_ch, self.id, 'meta', 'source', + build_meta(nil) + ) + if not meta_ok then + dlog(self, 'warn', { what = 'emit_initial_meta_failed', err = tostring(meta_err) }) + end + + -- Publish initial retained state: not yet synced, stratum unknown. + local state_ok, state_err = emit( + self.cap_emit_ch, self.id, 'state', 'synced', + { synced = false, stratum = nil } + ) + if not state_ok then + dlog(self, 'warn', { what = 'emit_initial_state_failed', err = tostring(state_err) }) + end + + self.scope:spawn(function() self:_ntpd_monitor() end) + + dlog(self, 'debug', { what = 'started' }) + return true, nil +end + +---Stop the time driver. Cancels the driver scope, terminating the NTP monitor fiber +---and any running ubus listen process. +---@param timeout number? Timeout in seconds. Defaults to 5. +---@return boolean ok +---@return string? error +function TimeDriver:stop(timeout) + timeout = timeout or DEFAULT_STOP_TIMEOUT + if self.backend and self.backend.shutdown_op then + local ok, stop_err = fibers.perform(self.backend:shutdown_op(0.2)) + if not ok then + dlog(self, 'warn', { what = 'ntp_backend_stop_failed', err = tostring(stop_err) }) + end + end + + self.scope:cancel() + + local source = fibers.perform(op.named_choice { + join = self.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + + if source == 'timeout' then + return false, "time driver stop timeout" + end + return true, nil +end + +---- Constructor ---- + +---Create a new TimeDriver instance. Generates a UUID for the capability id, +---creates a child scope, and instantiates the platform-specific time backend. +---Must be called from inside a fiber. +---@return TimeDriver? driver +---@return string error Empty string on success. +local function new(logger) + local scope, sc_err = fibers.current_scope():child() + if not scope then + return nil, "failed to create child scope: " .. tostring(sc_err) + end + + local backend = time_backend_provider.new() + + return setmetatable({ + id = uuid.new(), + cap_emit_ch = nil, + scope = scope, + backend = backend, + control_ch = channel.new(CONTROL_Q_LEN), + logger = logger, + initialised = false, + caps_applied = false, + synced = false, + }, TimeDriver), "" +end + +return { + new = new, +} diff --git a/src/services/hal/drivers/uart.lua b/src/services/hal/drivers/uart.lua index 3107b3ac..4375540a 100644 --- a/src/services/hal/drivers/uart.lua +++ b/src/services/hal/drivers/uart.lua @@ -1,328 +1,782 @@ --- driver.lua -local queue = require "fibers.queue" -local fiber = require "fibers.fiber" -local op = require "fibers.op" -local file = require "fibers.stream.file" -local service = require "service" -local hal_capabilities = require "services.hal.hal_capabilities" -local sc = require "fibers.utils.syscall" -local log = require "services.log" -local unpack = table.unpack or unpack -- Lua 5.2 compatibility - ----@class Driver ----@field ctx Context ----@field address string ----@field command_q Queue ----@field refresh_rate_channel Channel +---@module 'services.hal.drivers.uart' + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local channel = require 'fibers.channel' +local file = require 'fibers.io.file' +local exec = require 'fibers.io.exec' +local uuid = require 'uuid' + +local hal_types = require 'services.hal.types.core' +local cap_types = require 'services.hal.types.capabilities' +local cap_args = require 'services.hal.types.capability_args' +local resource = require 'devicecode.support.resource' + +local unpack = rawget(table, 'unpack') or _G.unpack +local pack = rawget(table, 'pack') or function (...) + return { n = select('#', ...), ... } +end + +local M = {} + +local CONTROL_Q_LEN = 8 +local DEFAULT_STOP_TIMEOUT = 5.0 + +---@class UARTSession +---@field lease_id string +---@field stream Stream +---@field release_lease_now function|nil +---@field release_lease_op function|nil +---@field closed boolean +---@field lease_released boolean +local UARTSession = {} +UARTSession.__index = UARTSession + +---@class UARTDriver +---@field id string +---@field path string +---@field default_baud integer|nil +---@field default_mode string|nil +---@field termios_ok boolean +---@field termios_error string|nil +---@field scope Scope|nil +---@field control_ch Channel +---@field emit_ch Channel|nil +---@field logger table|nil +---@field started boolean +---@field caps_applied boolean +---@field active_session UARTSession|nil +---@field active_lease_id string|nil +---@field closing_lease_id string|nil local Driver = {} Driver.__index = Driver ---[[ --- Helper function to set baudrate on a serial port -function M.set_baudrate(fd, baudrate) - -- Convert baudrate to termios constant - local baud_const = M.BAUDRATES[baudrate] - if not baud_const then - return nil, "Unsupported baudrate: " .. tostring(baudrate) +local function dlog(self, level, payload) + if self.logger and self.logger[level] then + self.logger[level](self.logger, payload) end +end - -- Get current termios settings - local termios = ffi.new("struct termios") - if ffi.C.tcgetattr(fd, termios) ~= 0 then - local errno = ffi.errno() - return nil, "Failed to get terminal attributes: " .. M.strerror(errno), errno +local function finalise_shell_scope(self, shell_scope, status, primary) + if self.scope ~= shell_scope then + return end - -- Set input and output baud rates - if ffi.C.cfsetispeed(termios, baud_const) ~= 0 or - ffi.C.cfsetospeed(termios, baud_const) ~= 0 then - local errno = ffi.errno() - return nil, "Failed to set baudrate: " .. M.strerror(errno), errno + local session = self.active_session + if session then + resource.terminate_checked( + session, + primary or status or 'uart shell closed', + 'UART session cleanup failed' + ) end - -- Apply settings - if ffi.C.tcsetattr(fd, M.TCSANOW, termios) ~= 0 then - local errno = ffi.errno() - return nil, "Failed to apply terminal settings: " .. M.strerror(errno), errno - end + self.started = false + self.scope = nil + self.active_session = nil + self.active_lease_id = nil + self.closing_lease_id = nil +end - return true +local function emit_op(emit_ch, class, id, mode, key, data) + return op.guard(function () + local payload, err = hal_types.new.Emit(class, id, mode, key, data) + if not payload then + return op.always(false, tostring(err)) + end + + return emit_ch:put_op(payload):wrap(function () + return true, nil + end) + end) end --- Helper function to get current baudrate -function M.get_baudrate(fd) - local termios = ffi.new("struct termios") - if ffi.C.tcgetattr(fd, termios) ~= 0 then - local errno = ffi.errno() - return nil, "Failed to get terminal attributes: " .. M.strerror(errno), errno - end +local function status_payload(self) + return { + state = 'available', + available = true, + open = self.active_session ~= nil or self.closing_lease_id ~= nil, + lease_id = self.active_lease_id or self.closing_lease_id, + path = self.path, + baud = self.default_baud, + mode = self.default_mode, + config_source = 'devicetree', + termios_ok = self.termios_ok == true, + termios_error = self.termios_error, + } +end - local speed = ffi.C.cfgetospeed(termios) +local function meta_payload(self) + return { + kind = 'uart', + path = self.path, + baud = self.default_baud, + mode = self.default_mode, + config_source = 'devicetree', + termios = { + configured = self.termios_ok == true, + error = self.termios_error, + }, + } +end - -- Find baudrate by constant - for rate, const in pairs(M.BAUDRATES) do - if const == speed then - return rate +local function reply_request_op(reply_ch, ok, value_or_err) + return op.guard(function () + local reply, err = hal_types.new.Reply(ok, value_or_err) + if not reply then + return op.always(false, 'invalid reply: ' .. tostring(err)) end - end - return nil, "Unknown baudrate constant: " .. tostring(speed) + return reply_ch:put_op(reply):wrap(function (sent, send_err) + if sent == true then + return true, nil + end + if sent == nil then + return false, tostring(send_err or 'reply channel closed') + end + return false, tostring(send_err or 'reply delivery failed') + end) + end) end --- Serial port setup helper function -function M.setup_serial_port(fd, baudrate, databits, parity, stopbits, flow_control) - local termios = ffi.new("struct termios") - if ffi.C.tcgetattr(fd, termios) ~= 0 then - local errno = ffi.errno() - return nil, "Failed to get terminal attributes: " .. M.strerror(errno), errno +local function release_session_now(driver, lease_id, _reason) + if driver.active_lease_id ~= lease_id and driver.closing_lease_id ~= lease_id then + return true, nil end - -- Clear all settings - termios.c_cflag = 0 - termios.c_iflag = 0 - termios.c_oflag = 0 - termios.c_lflag = 0 - - -- Set baudrate - local baud_const = M.BAUDRATES[baudrate] - if not baud_const then - return nil, "Unsupported baudrate: " .. tostring(baudrate) + driver.active_session = nil + driver.active_lease_id = nil + if driver.closing_lease_id == lease_id then + driver.closing_lease_id = nil end + return true, nil +end - if ffi.C.cfsetispeed(termios, baud_const) ~= 0 or - ffi.C.cfsetospeed(termios, baud_const) ~= 0 then - local errno = ffi.errno() - return nil, "Failed to set baudrate: " .. M.strerror(errno), errno +local function session_release_lease_now(session, reason) + if session.lease_released then + return true, nil end - -- Apply settings (other settings like databits, parity, etc. could be added here) - if ffi.C.tcsetattr(fd, M.TCSANOW, termios) ~= 0 then - local errno = ffi.errno() - return nil, "Failed to apply terminal settings: " .. M.strerror(errno), errno + session.lease_released = true + + if type(session.release_lease_now) ~= 'function' then + return true, nil end - return true + return session.release_lease_now(session.lease_id, reason) end ---]] +local function new_session(lease_id, stream, release_lease_now, release_lease_op) + return setmetatable({ + lease_id = lease_id, + stream = stream, + release_lease_now = release_lease_now, + release_lease_op = release_lease_op, + closed = false, + lease_released = false, + }, UARTSession) +end ------------------------------------------------------------------- Capability functions ---------------------------------------------------------------------------------------- -function Driver:open(ctx, opts) - print("BAUDRATE:", opts.baudrate) - if self.is_open then - return nil, "Port already open" - end - local baudrate = opts.baudrate or 115200 - local open_mode = "" - if opts.read and opts.write then - open_mode = "r+" - elseif opts.read then - open_mode = "r" - elseif opts.write then - open_mode = "w" - else - return nil, "Either read or write must be true" - end +function UARTSession:read_some_op(max) + return op.guard(function () + if self.closed then + return op.always(nil, 'uart session closed') + end + return self.stream:read_some_op(max) + end) +end - local filestream, err = file.open(self.port, open_mode) - local fd = filestream and filestream.io and filestream.io.fd or nil - if not fd then - log.error(string.format("Failed to open serial port %s: %s", self.name, err)) - return nil, err - end - if err then - log.error(string.format("Failed to open serial port %s: %s", self.name, err)) - return nil, err - end +function UARTSession:read_exactly_op(n) + return op.guard(function () + if self.closed then + return op.always(nil, 'uart session closed') + end + return self.stream:read_exactly_op(n) + end) +end - local termios = sc.new_termios() - local ok, err = sc.tcgetattr(fd, termios) - if not ok then - log.error(string.format("Failed to get terminal attributes for %s: %s", self.name, err)) - file.close(fd) - return nil, err - end +function UARTSession:read_line_op(opts) + return op.guard(function () + if self.closed then + return op.always(nil, 'uart session closed') + end + return self.stream:read_line_op(opts) + end) +end + +function UARTSession:read_all_op() + return op.guard(function () + if self.closed then + return op.always('', 'uart session closed') + end + return self.stream:read_all_op() + end) +end + +function UARTSession:write_op(...) + local parts = { ... } + return op.guard(function () + if self.closed then + return op.always(nil, 'uart session closed') + end + return self.stream:write_op(unpack(parts)) + end) +end - local baud_const = sc.BAUDRATES[baudrate] - if not baud_const then - log.error(string.format("Unsupported baudrate: %s", tostring(baudrate))) - file.close(fd) - return nil, "Unsupported baudrate" +function UARTSession:flush_op() + return op.guard(function () + if self.closed then + return op.always(nil, 'uart session closed') + end + return self.stream:flush_op() + end) +end + +-- Gracefully close the underlying stream and release the active driver lease. +function UARTSession:close_op() + return fibers.run_scope_op(function () + if self.closed then + return session_release_lease_now(self, 'uart session already closed') + end + + local stream = self.stream + local ok, err = fibers.perform(stream:close_op()) + if ok == nil then + return false, tostring(err) + end + + self.stream = nil + self.closed = true + + local ok_release, release_err + if type(self.release_lease_op) == 'function' then + ok_release, release_err = fibers.perform(self.release_lease_op(self.lease_id, 'uart session closed')) + else + ok_release, release_err = session_release_lease_now(self, 'uart session closed') + end + if ok_release ~= true then + return false, tostring(release_err or 'uart session lease release failed') + end + self.lease_released = true + + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +function UARTSession:terminate(reason) + local why = reason or 'uart session terminated' + local first_err + + if self.closed and self.lease_released then + return true, nil end - ok, err = sc.cfsetispeed(termios, baud_const) - if not ok then - log.error(string.format("Failed to set input speed for %s: %s", self.name, err)) - file.close(fd) - return nil, err + self.closed = true + + local stream = self.stream + self.stream = nil + + local ok_stream, stream_err = resource.terminate(stream, why) + if ok_stream ~= true and first_err == nil then + first_err = stream_err or 'uart stream termination failed' end - ok, err = sc.cfsetospeed(termios, baud_const) - if not ok then - log.error(string.format("Failed to set output speed for %s: %s", self.name, err)) - file.close(fd) - return nil, err + local ok_release, release_err = session_release_lease_now(self, why) + if ok_release ~= true and first_err == nil then + first_err = release_err or 'uart session lease release failed' end - ok, err = sc.tcsetattr(fd, sc.TCSANOW, termios) - if not ok then - log.error(string.format("Failed to apply terminal settings for %s: %s", self.name, err)) - file.close(fd) - return nil, err + if first_err then + return nil, first_err end - self.filestream = filestream - self.is_open = true - print("Opened serial port", self.name, "at", baudrate) return true, nil end -function Driver:close(ctx) - if not self.is_open then - return nil, "Port not open" + +local function mode_stty_args(mode) + mode = mode or '8N1' + if mode == '8N1' then + return { 'cs8', '-cstopb', '-parenb' } + elseif mode == '7E1' then + return { 'cs7', '-cstopb', 'parenb', '-parodd' } + elseif mode == '8O1' then + return { 'cs8', '-cstopb', 'parenb', 'parodd' } end - self.filestream:close() - self.filestream = nil - self.is_open = false - self.info_q:put({ - type = "uart", - id = self.name, - sub_topic = { "status" }, - endpoints = "single", - info = "closed" - }) - return true, nil + return nil, 'unsupported uart mode: ' .. tostring(mode) end -function Driver:write(ctx, data) - if not self.is_open then - return nil, "Port not open" +local function stty_args_for(self) + local baud = self.default_baud or 115200 + local mode_args, merr = mode_stty_args(self.default_mode) + if not mode_args then + return nil, merr end - local bytes_written, err = op.choice( - self.filestream:write_op(data), - ctx:done_op():wrap(function() - return nil, "Driver cancelled" - end) - ):perform() - if err then - self:close(ctx) + local args = { + 'stty', '-F', tostring(self.path), tostring(baud), + } + for _, a in ipairs(mode_args) do args[#args + 1] = a end + for _, a in ipairs({ + '-crtscts', + '-ixon', '-ixoff', + '-icrnl', + '-icanon', '-echo', '-isig', '-iexten', + '-opost', '-onlcr', + 'min', '1', 'time', '0', + 'clocal', 'cread', + }) do + args[#args + 1] = a end - return bytes_written, err + return args, nil end --- unlike other drivers, capabilities here block the main loop to provide --- deterministic order of execution -function Driver:_handle_capability(ctx, req) - local command = req.command - local args = req.args or {} - local ret_ch = req.return_channel +local function configure_termios_op(self, why) + return fibers.run_scope_op(function () + local args, aerr = stty_args_for(self) + if not args then + self.termios_ok = false + self.termios_error = tostring(aerr) + return false, self.termios_error + end - if type(ret_ch) == 'nil' then return end + local spec = { stdin = 'null', stdout = 'pipe', stderr = 'stdout' } + for i = 1, #args do spec[i] = args[i] end + local cmd = exec.command(spec) + local output, status, code, sig, err = fibers.perform(cmd:combined_output_op()) + if status == 'exited' and code == 0 then + self.termios_ok = true + self.termios_error = nil + dlog(self, 'debug', { + what = 'uart_termios_configured', + why = why, + path = self.path, + baud = self.default_baud or 115200, + mode = self.default_mode or '8N1', + }) + return true, nil + end - if type(command) == "nil" then - ret_ch:put({ - result = nil, - err = 'No command was provided' - }) - return + local detail = tostring(err or output or ('status=' .. tostring(status))) + if status == 'exited' then + detail = detail .. ' (exit ' .. tostring(code) .. ')' + elseif status == 'signalled' then + detail = detail .. ' (signal ' .. tostring(sig) .. ')' + end + self.termios_ok = false + self.termios_error = detail + return false, 'uart termios stty failed for ' .. tostring(self.path) .. ': ' .. detail + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + self.termios_ok = false + self.termios_error = tostring(err or rep) + return false, self.termios_error + end + return ok, err + end) +end + +local function open_stream_op(path) + return fibers.run_scope_op(function () + -- Box the currently synchronous file.open(...) impurity inside one + -- operation-owned subtree. Surface remains op-native. + local stream, err = file.open(path, 'r+') + if not stream then + return false, tostring(err) + end + return true, stream + end):wrap(function (st, rep, ok, value_or_err) + if st ~= 'ok' then + return false, tostring(value_or_err or rep) + end + return ok, value_or_err + end) +end + +local release_session_gracefully_op + +local function open_session_op(self) + return fibers.run_scope_op(function (scope) + local ok_cfg, cfg_err = fibers.perform(configure_termios_op(self, 'open')) + if ok_cfg ~= true then + return false, tostring(cfg_err) + end + + local ok, stream_or_err = fibers.perform(open_stream_op(self.path)) + if not ok then + return false, stream_or_err + end + + local stream = stream_or_err + local handed_off = false + scope:finally(function (_, status, primary) + if not handed_off then + resource.terminate_checked(stream, primary or status or 'uart open failed', 'uart open stream cleanup failed') + end + end) + + local lease_id = uuid.new() + local session = new_session( + lease_id, + stream, + function (active_lease_id, reason) + return release_session_now(self, active_lease_id, reason) + end, + function (active_lease_id, _reason) + return release_session_gracefully_op(self, active_lease_id, true) + end + ) + + local reply, rerr = hal_types.new.UARTOpenReply( + lease_id, + session, + self.path, + self.default_baud, + self.default_mode + ) + if not reply then + return false, tostring(rerr) + end + + handed_off = true + return true, reply + end):wrap(function (st, rep, ok, value_or_err) + if st ~= 'ok' then + return false, tostring(value_or_err or rep) + end + return ok, value_or_err + end) +end + +local function publish_status_op(self) + return emit_op(self.emit_ch, 'uart', self.id, 'state', 'status', status_payload(self)) +end + +local function publish_event_op(self, event_name, data) + return emit_op(self.emit_ch, 'uart', self.id, 'event', event_name, data) +end + +function release_session_gracefully_op(self, lease_id, emit_closed_event) + return fibers.run_scope_op(function () + if self.active_lease_id ~= lease_id then + return true, nil + end + + self.active_session = nil + self.active_lease_id = nil + if self.closing_lease_id == lease_id then + self.closing_lease_id = nil + end + + local ok_status, status_err = fibers.perform(publish_status_op(self)) + if not ok_status then + return false, tostring(status_err) + end + + if emit_closed_event then + local ok_event, event_err = fibers.perform(publish_event_op(self, 'closed', { + lease_id = lease_id, + path = self.path, + })) + if not ok_event then + return false, tostring(event_err) + end + end + + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +local function methods_for(self) + return { + status = function (_opts, _request) + return op.always(true, status_payload(self)) + end, + + open = function (opts, _request) + if opts ~= nil and (type(opts) ~= 'table' or getmetatable(opts) ~= cap_args.UARTOpenOpts) then + return op.always(false, 'invalid open opts') + end + + if self.active_session ~= nil or self.closing_lease_id ~= nil then + return op.always(false, 'busy') + end + + return fibers.run_scope_op(function (scope) + local ok, reply_or_err = fibers.perform(open_session_op(self)) + if not ok then + return false, reply_or_err + end + + local reply = reply_or_err + local handed_off = false + + scope:finally(function () + if not handed_off and self.active_session == reply.session then + resource.terminate_checked(reply.session, 'uart open abandoned', 'UART open session cleanup failed') + self.active_session = nil + self.active_lease_id = nil + if self.closing_lease_id == reply.lease_id then + self.closing_lease_id = nil + end + end + end) + + self.active_session = reply.session + self.active_lease_id = reply.lease_id + + local ok_status, status_err = fibers.perform(publish_status_op(self)) + if not ok_status then + return false, tostring(status_err) + end + + local ok_event, event_err = fibers.perform(publish_event_op(self, 'opened', { + lease_id = reply.lease_id, + path = self.path, + })) + if not ok_event then + return false, tostring(event_err) + end + + handed_off = true + return true, reply + end):wrap(function (st, rep, ok, value_or_err) + if st ~= 'ok' then + return false, tostring(value_or_err or rep) + end + return ok, value_or_err + end) + end, + } +end + +local function handle_request_op(self, request) + return fibers.run_scope_op(function () + local methods = methods_for(self) + local fn = methods[request.verb] + + local ok, value_or_err + if type(fn) ~= 'function' then + ok = false + value_or_err = 'unsupported verb: ' .. tostring(request.verb) + else + ok, value_or_err = fibers.perform(fn(request.opts, request)) + end + + local replied, reply_err = + fibers.perform(reply_request_op(request.reply_ch, ok, value_or_err)) + + if not replied then + return false, tostring(reply_err) + end + + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +local function shell_main(self) + local shell_scope = assert(self.scope, 'uart shell without scope') + assert(self.emit_ch, 'uart shell without emit channel') + + shell_scope:finally(function (_, status, primary) + finalise_shell_scope(self, shell_scope, status, primary) + end) + + local ok_meta, meta_err = + fibers.perform(emit_op(self.emit_ch, 'uart', self.id, 'meta', 'details', meta_payload(self))) + if ok_meta ~= true then + error(tostring(meta_err or 'initial uart meta emit failed'), 0) end - local func = self[command] - if type(func) ~= "function" then - ret_ch:put({ - result = nil, - err = "Command does not exist" - }) - return + local ok_status, status_err = fibers.perform(publish_status_op(self)) + if ok_status ~= true then + error(tostring(status_err or 'initial uart status emit failed'), 0) end - local res, err = func(self, ctx, unpack(args)) - fiber.spawn(function() - op.choice( - ret_ch:put_op({ - res = res, - err = err - }), - ctx:done_op() - ):perform() + while true do + local request = fibers.perform(self.control_ch:get_op()) + if not request then + return + end + + local ok_req, req_err = fibers.perform(handle_request_op(self, request)) + if not ok_req then + dlog(self, 'warn', { + what = 'uart_request_failed', + err = tostring(req_err), + }) + end + end +end + +function Driver:capabilities_op(emit_ch) + return op.guard(function () + if self.caps_applied then + return op.always(false, 'capabilities already applied') + end + + self.emit_ch = emit_ch + + local cap, err = cap_types.new.UARTCapability(self.id, self.control_ch) + if not cap then + return op.always(false, tostring(err)) + end + + self.caps_applied = true + return op.always(true, { cap }) end) end -function Driver:_main(ctx) - log.trace(string.format( - "%s - %s: Starting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) +---@param owner_scope Scope +function Driver:start_op(owner_scope) + assert(owner_scope ~= nil, 'uart driver start_op: owner_scope is required') - while not ctx:err() do - local ops = { - self.capability_q:get_op():wrap(function(req) - self:_handle_capability(ctx, req) - end), - ctx:done_op() - } - if self.is_open then - table.insert(ops, self.filestream:read_line_op():wrap(function(line, _, err) - if (not line) or err then - self:close(ctx) - return - end - self.info_q:put({ - type = "uart", - id = self.name, - sub_topic = { "out" }, - endpoints = "single", - info = line - }) - end)) - end - op.choice(unpack(ops)):perform() + return fibers.run_scope_op(function () + if self.started then + return false, 'already started' + end + if not self.caps_applied then + return false, 'capabilities not applied' + end + if not self.emit_ch then + return false, 'missing emit channel' + end + + local ok_cfg, cfg_err = fibers.perform(configure_termios_op(self, 'start')) + if ok_cfg ~= true then + return false, tostring(cfg_err) + end + + local shell_scope, serr = owner_scope:child() + if not shell_scope then + return false, tostring(serr) + end + + self.scope = shell_scope + + local ok, err = shell_scope:spawn(function () + return shell_main(self) + end) + if not ok then + self.scope = nil + shell_scope:cancel(tostring(err or 'uart shell spawn failed')) + return false, tostring(err) + end + + self.started = true + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +function Driver:terminate(reason) + if self.scope then + self.scope:cancel(reason or 'uart driver terminated') end - self:close(ctx) - log.trace(string.format( - "%s - %s: Exiting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) -end - -function Driver:get_port() - return self.port -end - ----returns a list of control and info capabilities for the modem ----@return table -function Driver:apply_capabilities(capability_info_q) - self.info_q = capability_info_q - local capabilities = {} - capabilities.uart = { - control = hal_capabilities.new_serial_capability(self.capability_q), - id = self.name - } - return capabilities + if self.active_session then + self.active_session:terminate(reason or 'uart driver terminated') + end + self.started = false + self.scope = nil + self.active_session = nil + self.active_lease_id = nil + self.closing_lease_id = nil + return true, nil end ----@param conn Connection -function Driver:spawn(conn) - service.spawn_fiber('Serial Main (' .. self.name .. ')', conn, self.ctx, function(fctx) - self:_main(fctx) +function Driver:shutdown_op(timeout) + timeout = timeout or DEFAULT_STOP_TIMEOUT + + return op.guard(function () + if not self.started or not self.scope then + return op.always(true, nil) + end + + local shell_scope = self.scope + local session = self.active_session + + shell_scope:cancel() + + return fibers.boolean_choice( + shell_scope:join_op():wrap(function () + -- Make the post-stop contract explicit: any previously returned + -- session wrapper is no longer usable, even if the shell stop + -- path did not get to mark it in time. + if session then + session.closed = true + end + + finalise_shell_scope(self, shell_scope, 'ok', nil) + return true, nil + end), + sleep.sleep_op(timeout):wrap(function () + return false, 'uart driver stop timeout' + end) + ):wrap(function (completed, _a, b) + if completed then + return true, nil + end + return false, b + end) end) end -local function new(ctx, name, port) - local self = setmetatable({}, Driver) - self.ctx = ctx - self.name = name - self.port = port - self.capability_q = queue.new(10) - self.filestream = nil - -- Other initial properties - return self +function Driver:fault_op() + if self.scope and self.started then + return self.scope:fault_op() + end + return op.never() +end + +---@param id string +---@param path string +---@param baud integer|nil +---@param mode string|nil +---@param logger table|nil +---@return UARTDriver +function M.new(id, path, baud, mode, logger) + assert(type(id) == 'string' and id ~= '', 'uart.new: invalid id') + assert(type(path) == 'string' and path ~= '', 'uart.new: invalid path') + + return setmetatable({ + id = id, + path = path, + default_baud = baud, + default_mode = mode, + scope = nil, + control_ch = channel.new(CONTROL_Q_LEN), + emit_ch = nil, + logger = logger, + termios_ok = false, + termios_error = nil, + started = false, + caps_applied = false, + active_session = nil, + active_lease_id = nil, + closing_lease_id = nil, + }, Driver) end -return { - new = new -} +M.Driver = Driver +M.UARTSession = UARTSession +return M diff --git a/src/services/hal/drivers/ubus.lua b/src/services/hal/drivers/ubus.lua deleted file mode 100644 index c2023f70..00000000 --- a/src/services/hal/drivers/ubus.lua +++ /dev/null @@ -1,371 +0,0 @@ -local queue = require "fibers.queue" -local op = require "fibers.op" -local fiber = require "fibers.fiber" -local exec = require "fibers.exec" -local context = require "fibers.context" -local sc = require "fibers.utils.syscall" -local log = require "services.log" -local hal_capabilities = require "services.hal.hal_capabilities" -local service = require "service" -local uuid = require "uuid" -local unpack = table.unpack or unpack -local cjson = require "cjson.safe" - -local STRING_SEPARATOR = string.char(31) - -local UBus = {} -UBus.__index = UBus - -function UBus.new(ctx) - local ubus = { ctx = ctx, cap_control_q = queue.new(10) } - return setmetatable(ubus, UBus) -end - -local streams = { - by_key = {}, - by_id = {} -} - ---- Check all paths are of string type ---- @param paths string[] ---- @return boolean ---- @return string? -local function validate_paths(paths) - for _, path in ipairs(paths) do - if type(path) ~= 'string' or path == '' then - return false, "Invalid path: " .. tostring(path) - end - end - return true, nil -end - ---- Compare two strings alphabetically ---- @param str1 string ---- @param str2 string ---- @return boolean -local function sort_alphabetically(str1, str2) - return str1:lower() < str2:lower() -end - ---- -------------------------------------------------------------------------------- ---- ubus capabilities ---- -------------------------------------------------------------------------------- - ---- list available ubus endpoints ---- @param ctx Context ---- @return table? ---- @return string? -function UBus:list(ctx) - local result, err = exec.command_context(ctx, 'ubus', 'list'):output() - if err then - return result, err - end - - local list = {} - for line in result:gmatch("[^\r\n]+") do - table.insert(list, line) - end - return list, nil -end - ---- Call a ubus method ---- @param ctx Context ---- @param path string ---- @param method string ---- @param ... string[] ---- @return table? ---- @return string? -function UBus:call(ctx, path, method, ...) - local args = { ... } - local result, err = exec.command_context(ctx, 'ubus', 'call', path, method, unpack(args)):output() - if err then - return nil, "Failed to call ubus method: " .. method .. ": " .. err - end - - local data, decode_err = cjson.decode(result) - if decode_err then - return nil, "Failed to decode ubus call output: " .. decode_err - end - - return data, nil -end - ---- Cancel a stream if it exists and has no users left ---- @param stream_id string ---- @return boolean success ---- @return string? err -function UBus:stop_stream(ctx, stream_id) - local stream = streams.by_id[stream_id] - if not stream then - return false, "Stream not found" - end - - stream.users = stream.users - 1 - if stream.users > 0 then - return true, nil - end - - stream.cancel_fn() - streams.by_key[stream.key] = nil - streams.by_id[stream_id] = nil - - log.trace(string.format( - "%s - %s: Stopped ubus stream %s", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - stream_id - )) - - fiber.spawn(function() - op.choice( - self.info_q:put_op({ -- report stream as closed - type = "ubus", - id = "1", - sub_topic = { "stream", stream_id, "closed" }, - endpoints = "single", - info = true - }), - ctx:done_op() - ):perform() - end) - - return true, nil -end - ---- Create a streaming endpoint that outputs listen events given for paths, ---- or return existing stream if it already exists. ---- @param ctx Context ---- @param ... string[] ---- @return table? stream_info ---- @return string? err -function UBus:listen(ctx, ...) - local paths = { ... } - local valid, err = validate_paths(paths) - if not valid then - return nil, err - end - -- sort paths alphabetically to ensure consistent key generation - table.sort(paths, sort_alphabetically) - table.insert(paths, 1, 'listen') - local path_key = table.concat(paths, STRING_SEPARATOR) - if streams.by_key[path_key] then - local stream_info = streams.by_id[streams.by_key[path_key]] - stream_info.users = stream_info.users + 1 - return { stream_id = streams.by_key[path_key] }, nil - end - - local stream_id = uuid.new() -- create a unique identifier for the listen command - local stream_ctx, cancel = context.with_cancel(ctx) - - streams.by_key[path_key] = stream_id - streams.by_id[stream_id] = { - key = path_key, - cancel_fn = cancel, - users = 1, - } - - fiber.spawn(function() - log.trace(string.format( - "%s - %s: Starting ubus listen command for stream %s with paths: %s", - stream_ctx:value("service_name"), - stream_ctx:value("fiber_name"), - stream_id, - table.concat(paths, ", ") - )) - - local cmd = exec.command('ubus', 'listen', unpack(paths)) - cmd:setprdeathsig(sc.SIGKILL) -- make sure that the command is killed if the parent dies - local stdout = cmd:stdout_pipe() - if not stdout then - log.error(string.format( - "%s - %s: Failed to create stdout pipe for ubus listen command", - stream_ctx:value("service_name"), - stream_ctx:value("fiber_name") - )) - op.choice( - self.cap_control_q:put_op({ -- stop the stream if stdout pipe creation failed - command = "stop_stream", - args = { stream_id }, - return_channel = { put = function() end } - }), - stream_ctx:done_op() - ):perform() - return - end - - local cmd_err = cmd:start() - if cmd_err then - log.error(string.format( - "%s - %s: Failed to start ubus listen command: %s", - stream_ctx:value("service_name"), - stream_ctx:value("fiber_name"), - cmd_err - )) - cmd:wait() - stdout:close() - op.choice( - self.cap_control_q:put_op({ -- stop the stream if the command fails to start - command = "stop_stream", - args = { stream_id }, - return_channel = { put = function() end } - }), - stream_ctx:done_op() - ):perform() - return - end - - while not stream_ctx:err() do - op.choice( - stdout:read_line_op():wrap(function(line) - if not line then - stream_ctx:cancel("command finished") - return - end - local data, decode_err = cjson.decode(line) - if decode_err then - log.error(string.format( - "%s - %s: Failed to decode ubus listen data: %s, reason: %s", - stream_ctx:value("service_name"), - stream_ctx:value("fiber_name"), - tostring(line), - decode_err - )) - return - end - - op.choice( - self.info_q:put_op({ -- send listen data to stream endpoint - type = "ubus", - id = "1", - sub_topic = { "stream", stream_id }, - endpoints = "single", - info = data - }), - stream_ctx:done_op() - ):perform() - end), - stream_ctx:done_op():wrap(function() - cmd:kill() - end) - ):perform() - end - cmd:wait() - stdout:close() - - -- cleanup on command stop - op.choice( - self.cap_control_q:put_op({ - command = "stop_stream", - args = { stream_id }, - return_channel = { put = function() end } - }), - stream_ctx:done_op() - ):perform() - - log.trace(string.format( - "%s - %s: Ubus listen command finished for stream %s", - stream_ctx:value("service_name"), - stream_ctx:value("fiber_name"), - stream_id - )) - end) - - return { stream_id = stream_id }, nil -end - ---- Send a ubus message ---- @param ctx Context ---- @param type string ---- @param message table ---- @return table? ---- @return string? -function UBus:send(ctx, type, message) - local result, err = exec.command_context(ctx, 'ubus', 'send', type, cjson.encode(message)):output() - if err then - return result, "Failed to send ubus message: " .. err - end - return result, nil -end - --- ubus docs reference wait_for and monitor but don't show what they do, test these functions and implement late --- not a priority right now, but should be done --- function UBus:wait_for(ctx, object, ...) --- function UBus:monitor(ctx) - ---- -------------------------------------------------------------------------------- ---- -------------------------------------------------------------------------------- - ---- Handle a ubus capability request ---- @param ctx Context ---- @param request table -function UBus:handle_capability(ctx, request) - local command = request.command - local args = request.args or {} - local ret_ch = request.return_channel - - if type(ret_ch) == 'nil' then return end - - if type(command) == "nil" then - ret_ch:put({ - result = nil, - err = 'No command was provided' - }) - return - end - - local func = self[command] - if type(func) ~= "function" then - ret_ch:put({ - result = nil, - err = "Command does not exist" - }) - return - end - - fiber.spawn(function() - local result, err = func(self, ctx, unpack(args)) - - ret_ch:put({ - result = result, - err = err - }) - end) -end - ---- Apply ubus capabilities ---- @param capability_info_q Queue ---- @return table? ---- @return string? -function UBus:apply_capabilities(capability_info_q) - self.info_q = capability_info_q - local capabilities = { - ubus = { - control = hal_capabilities.new_ubus_capability(self.cap_control_q), - id = "1" - } - } - return capabilities, nil -end - ---- Main ubus fiber ---- @param ctx Context -function UBus:_main(ctx) - while not ctx:err() do - op.choice( - self.cap_control_q:get_op():wrap(function(req) - self:handle_capability(ctx, req) - end), - ctx:done_op() - ):perform() - end -end - ---- Spawn all ubus driver fibers ---- @param conn Connection -function UBus:spawn(conn) - service.spawn_fiber("UBus Driver", conn, self.ctx, function(fctx) - self:_main(fctx) - end) -end - -return { new = UBus.new } diff --git a/src/services/hal/drivers/uci.lua b/src/services/hal/drivers/uci.lua deleted file mode 100644 index cece00e2..00000000 --- a/src/services/hal/drivers/uci.lua +++ /dev/null @@ -1,459 +0,0 @@ -local queue = require "fibers.queue" -local channel = require "fibers.channel" -local op = require "fibers.op" -local fiber = require "fibers.fiber" -local sc = require "fibers.utils.syscall" -local sleep = require "fibers.sleep" -local exec = require "fibers.exec" -local hal_capabilities = require "services.hal.hal_capabilities" -local log = require "services.log" -local service = require "service" -local uci = require "uci" -local unpack = table.unpack or unpack - -local cursor = uci.cursor() - -local restart_policies = {} - -local UCI = {} -UCI.__index = UCI - -function UCI.new(ctx) - local uci = { - ctx = ctx, - cap_control_q = queue.new(10), -- source of capability commands - info_q = nil, -- to be assigned at initalisation - actions_q = queue.new(10), -- source of restart actions - config_update_q = queue.new(10), -- signals when a config has been updated for restart actions - restart_q = queue.new(10), -- scheduled config restarts - restart_state_ch = channel.new(), -- outputs state of restart worker - restart_halt_ch = channel.new() -- signals restart worker to halt any restarts - } - return setmetatable(uci, UCI) -end - ---- UCI command functions - ---- Get the value of a UCI entry ---- @param _ Context ---- @param config string ---- @param section string ---- @param option string ---- @return any ---- @return string? -function UCI:get(_, config, section, option) - local val, err = cursor:get(config, section, option) - if err then - return nil, err - end - return val, nil -end - ---- Set the value of a UCI entry ---- @param _ Context ---- @param config string ---- @param section string ---- @param option string ---- @param value any ---- @return boolean ---- @return string? -function UCI:set(ctx, config, section, option, value) - local success, err - if value == nil then - success, err = cursor:set(config, section, option) - else - if type(value) == 'boolean' then - value = value and 1 or 0 - end - success, err = cursor:set(config, section, option, value) - end - if not success then - return false, string.format("Failed to set %s.%s.%s to %s: %s", config, section, option, value, err) - end - - return true, nil -end - ---- Delete a UCI entry ---- @param _ Context ---- @param config string ---- @param section string ---- @param option string ---- @return boolean ---- @return string? -function UCI:delete(ctx, config, section, option) - local success, err - if option then - success, err = cursor:delete(config, section, option) - else - success, err = cursor:delete(config, section) - end - if not success then - return false, string.format("Failed to delete %s.%s.%s: %s", config, section, option, err) - end - - return true, nil -end - ---- Commit changes to a UCI configuration ---- @param ctx Context ---- @param config string ---- @return boolean ---- @return string? -function UCI:commit(ctx, config) - local config_restart_ch = channel.new() - op.choice( - self.config_update_q:put_op({ config = config, notify_ch = config_restart_ch }), - ctx:done_op() - ):perform() - local restart_result = op.choice( - config_restart_ch:get_op(), - ctx:done_op() - ):perform() - return restart_result and restart_result.ret or false, restart_result and restart_result.err or ctx:err() -end - ---- Add a new section to a UCI configuration ---- @param _ Context ---- @param config string ---- @param section_type string ---- @return string ---- @return string? -function UCI:add(ctx, config, section_type) - local name = cursor:add(config, section_type) - return name, nil -end - ---- Revert saved but uncommitted changes ---- @param _ Context ---- @param config string ---- @return boolean ---- @return string? -function UCI:revert(ctx, config) - local success, err = cursor:revert(config) - if not success then - return false, string.format("Failed to revert %s: %s", config, err) - end - return true, nil -end - ---- Get a table of saved but uncommitted changes ---- @param _ Context ---- @param config string ---- @return table ---- @return string? -function UCI:changes(_, config) - local changes, err = cursor:changes(config) - if err then - return nil, string.format("Failed to get changes for %s: %s", config, err) - end - return changes, nil -end - ---- Bring up a network interface ---- @param ctx Context ---- @param interface string ---- @return boolean ---- @return string? -function UCI:ifup(ctx, interface) - local ret, err = self:halt_restarts(ctx) - if err then return ret, err end - err = exec.command_context(ctx, 'ifup', interface):run() - local ret2, err2 = self:continue_restarts(ctx) -- signal restart fiber to continue whether command succeeds or not - return ret2, err or err2 -end - ---- Call a function for every section of a certain type ---- @param _ Context ---- @param config string ---- @param type string ---- @param callback fun(cursor: Cursor, section: table) -function UCI:foreach(ctx, config, type, callback) - local success = cursor:foreach(config, type, function(section) - callback(cursor, section) - end) - if not success then - return false, string.format("Failed to iterate over %s.%s", config, type) - end - return true, nil -end - ---- Assign a restart policy to a UCI config ---- @param ctx Context ---- @param config string ---- @param policy table ---- @param actions table ---- @return boolean ---- @return string? -function UCI:set_restart_actions(ctx, config, actions) - if not actions then - return false, "Actions must be specified" - end - - op.choice( - self.actions_q:put_op({ - config = config, - actions = actions - }), - ctx:done_op() - ):perform() - - return true, nil -end - ---- Signal restart fiber to halt ---- @param ctx Context ---- @return boolean ---- @return string? -function UCI:halt_restarts(ctx) - op.choice( - self.restart_halt_ch:put_op(true), - ctx:done_op() - ):perform() - local state = "active" - -- wait for restart fiber to halt - while state ~= "halt" and not ctx:err() do - op.choice( - self.restart_state_ch:get_op():wrap(function(msg) - state = msg - end), - ctx:done_op() - ):perform() - end - return ctx:err() == nil, ctx:err() -end - ---- Signal restart fiber to continue ---- @param ctx Context ---- @return boolean ---- @return string? -function UCI:continue_restarts(ctx) - op.choice( - self.restart_halt_ch:put_op(false), - ctx:done_op() - ):perform() - return ctx:err() == nil, ctx:err() -end - ---- Run requested method if exists ---- @param ctx Context ---- @param request table -function UCI:handle_capability(ctx, request) - local command = request.command - local args = request.args or {} - local ret_ch = request.return_channel - - if type(ret_ch) == 'nil' then return end - - if type(command) == "nil" then - ret_ch:put({ - result = nil, - err = 'No command was provided' - }) - return - end - - local func = self[command] - if type(func) ~= "function" then - ret_ch:put({ - result = nil, - err = "Command does not exist" - }) - return - end - - fiber.spawn(function() - local result, err = func(self, ctx, unpack(args)) - - ret_ch:put({ - result = result, - err = err - }) - end) -end - ---- Assign a restart policy to a specific config ---- @param config string ---- @param actions table -function UCI:handle_restart_actions(config, actions) - if not restart_policies[config] then - restart_policies[config] = { actions = actions } - end -end - ---- Apply UCI capabilities ---- @param capability_info_q Queue ---- @return table ---- @return string? -function UCI:apply_capabilities(capability_info_q) - self.info_q = capability_info_q - local capabilities = { - uci = { - control = hal_capabilities.new_uci_capability(self.cap_control_q), - id = "1" - } - } - return capabilities, nil -end - ---- Main loop for UCI driver ---- @param ctx Context -function UCI:_main(ctx) - log.info(string.format( - "%s - %s: UCI Main Starting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - local restart_op = nil - while not ctx:err() do - local ops = { - self.cap_control_q:get_op():wrap(function(req) - self:handle_capability(ctx, req) - end), - self.actions_q:get_op():wrap(function(msg) - self:handle_restart_actions(msg.config, msg.actions) - end), - self.config_update_q:get_op():wrap(function(commit_request) - local restarter = restart_policies[commit_request.config] - if not restarter then - commit_request.notify_ch:put({ - err = "No restart actions set for config " .. commit_request.config - }) - return - end - fiber.spawn(function() - self.restart_q:put({ - config = commit_request.config, - actions = restarter.actions, - notify_ch = commit_request.notify_ch - }) - end) - end), - ctx:done_op() - } - op.choice(unpack(ops)):perform() - end - log.info(string.format( - "%s - %s: UCI Main Exiting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) -end - ---- Restart Worker ---- @param ctx Context -function UCI:_restart_worker(ctx) - local to_restart = {} - local next_deadline = sc.monotime() + 1 - local state = "active" - local halt_num = 0 - while not ctx:err() do - op.choice( - self.restart_state_ch:put_op(state), - self.restart_halt_ch:get_op():wrap(function(msg) - local halt = msg - halt_num = halt and (halt_num + 1) or (halt_num - 1) -- count the number of halts - if halt_num <= 0 then - halt_num = 0 -- halts should never go below 0 but just in case - return - end - state = "halt" - log.trace(string.format( - "%s - %s: Halted", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - while halt_num > 0 and not ctx:err() do -- listen for halt messages until all halts are resolved - op.choice( - self.restart_halt_ch:get_op():wrap(function(msg) - halt = msg - halt_num = halt and (halt_num + 1) or (halt_num - 1) - end), - self.restart_state_ch:put_op(state), -- update restart state - ctx:done_op() - ):perform() - end - state = "active" - log.trace(string.format( - "%s - %s: Active", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - end), - self.restart_q:get_op():wrap(function(msg) - local restarter = to_restart[msg.config] - if not restarter then - restarter = { config = msg.config, actions = msg.actions, notify_channels = {} } - to_restart[msg.config] = restarter - end - table.insert(restarter.notify_channels, msg.notify_ch) - end), - sleep.sleep_until_op(next_deadline):wrap(function() -- iterate over all scheduled restarts every 1 second - next_deadline = sc.monotime() + 1 - if next(to_restart) == nil then return end - for config, restarter in pairs(to_restart) do - local _, err = cursor:commit(config) - if err then - for _, ch in ipairs(restarter.notify_channels) do - ch:put({ ret = false, err = err }) - end - return - end - for i, action in ipairs(restarter.actions) do - log.trace(string.format( - "%s - %s: Restarting %s action %d", - ctx:value("service_name"), - ctx:value("fiber_name"), - config, - i - )) - local err = exec.command_context(ctx, unpack(action)):run() - if err then - log.error(string.format( - "%s - %s: Restarting %s action %d failed: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - config, - i, - err - )) - for _, ch in ipairs(restarter.notify_channels) do - ch:put({ ret = false, err = err }) - end - break - end - log.trace(string.format( - "%s - %s: Restarting %s action %d completed", - ctx:value("service_name"), - ctx:value("fiber_name"), - config, - i - )) - end - for _, ch in ipairs(restarter.notify_channels) do - ch:put({ ret = err == nil, err = err }) - end - end - to_restart = {} - end) - ):perform() - end - log.info(string.format( - "%s - %s: UCI Main Exiting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) -end - ---- Spin up all fibers for UCI driver ---- @param conn Connection -function UCI:spawn(conn) - service.spawn_fiber("UCI Driver", conn, self.ctx, function(fctx) - self:_main(fctx) - end) - service.spawn_fiber("UCI Restarter", conn, self.ctx, function(fctx) - self:_restart_worker(fctx) - end) -end - -return { new = UCI.new } diff --git a/src/services/hal/drivers/usb.lua b/src/services/hal/drivers/usb.lua new file mode 100644 index 00000000..8f6b5ec8 --- /dev/null +++ b/src/services/hal/drivers/usb.lua @@ -0,0 +1,437 @@ +-- services/hal/drivers/usb.lua +-- +-- USB bus HAL driver. +-- Manages the USB 3.0 bus enable/disable state via sysfs power control. +-- Exposes a 'usb' capability with 'enable' and 'disable' RPC offerings. +-- Current state is probed at driver creation and published as state/bus +-- on start(). + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local file = require "fibers.io.file" +local exec = require "fibers.io.exec" +local channel = require "fibers.channel" + +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" + +local perform = fibers.perform + +local CONTROL_Q_LEN = 8 + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +-- Sysfs base path for USB hubs on bigbox-ss hardware (BCM2711 / VL805). +local USB_HUB_PREFIX = "/sys/devices/platform/scb/fd500000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0/usb" + +-- Sysfs subpaths for each hub port, relative to USB_HUB_PREFIX .. hub_number. +-- Hub 1 = USB 2.0 host; Hub 2 = USB 3.0 host. +local USB_PORT_SUBPATHS = { + [1] = { [1] = "1-1/1-1.1", [2] = "1-1/1-1.2", [3] = "1-1/1-1.3", [4] = "1-1/1-1.4" }, + [2] = { [1] = "2-1", [2] = "2-2" }, +} + +-- Minimum VL805 firmware timestamp for hub power control support (2019-09-10). +local VL805_SUPPORTED_FROM = os.time({ year = 2019, month = 9, day = 10, hour = 0, min = 0, sec = 0 }) + +---@class UsbDriver +---@field bus_id string capability id, e.g. "usb3" +---@field scope Scope +---@field control_ch Channel +---@field cap_emit_ch Channel? +---@field enabled boolean current logical state +---@field logger Logger? +local UsbDriver = {} +UsbDriver.__index = UsbDriver + +---- helpers ---- + +--- Probe USB3 state from hub 2's authorized_default sysfs node. +--- Returns true (enabled) as a safe default when unavailable. +---@return boolean +local function probe_usb3_state() + local path = USB_HUB_PREFIX .. '2/authorized_default' + local f, _ = file.open(path, 'r') + if not f then return true end + local raw, _ = f:read_all() + f:close() + if raw then + local val = tonumber(raw:match("%d+")) + return val ~= nil and val ~= 0 + end + return true +end + +--- Return true if a USB device is present at the given hub/port sysfs path. +---@param hub integer +---@param port integer +---@return boolean present +---@return string? error +local function is_device_on_hub_port(hub, port) + local subpaths = USB_PORT_SUBPATHS[hub] + if not subpaths then return false, "invalid hub" end + if not subpaths[port] then return false, "invalid port" end + local path = USB_HUB_PREFIX .. hub .. '/' .. subpaths[port] + local cmd = exec.command { 'test', '-e', path, stdin = 'null', stdout = 'null', stderr = 'null' } + local _, _, code = perform(cmd:run_op()) + return code == 0, nil +end + +--- Write an integer to a sysfs path. +---@param path string +---@param value integer +---@return string? error +local function exec_write_to_file(path, value) + local f, ferr = file.open(path, 'w') + if not f then + return ("open %s failed: %s"):format(path, tostring(ferr or 'unknown')) + end + + local n, werr = perform(f:write_op(tostring(value) .. '\n')) + local ok, cerr = perform(f:close_op()) + if n == nil then + return ("write to %s failed: %s"):format(path, tostring(werr or 'unknown')) + end + if not ok then + return ("close %s failed: %s"):format(path, tostring(cerr or 'unknown')) + end + return nil +end + +---@param enabled boolean +---@param hub integer +---@return string? error +local function set_usb_hub_auth_default(enabled, hub) + return exec_write_to_file(USB_HUB_PREFIX .. hub .. '/authorized_default', enabled and 1 or 0) +end + +---@param enabled boolean +---@param hub integer +---@param port integer +---@return string? error +local function set_usb_port_auth(enabled, hub, port) + return exec_write_to_file( + USB_HUB_PREFIX .. hub .. '/' .. USB_PORT_SUBPATHS[hub][port] .. '/authorized', + enabled and 1 or 0) +end + +--- Deauthorize all occupied USB3 (hub 2) ports. +---@return boolean hub_was_used +---@return string? error +local function clear_usb3_hub() + local hub_used = false + for i = 1, #USB_PORT_SUBPATHS[2] do + local present, err = is_device_on_hub_port(2, i) + if err then return false, "error checking port " .. i .. ": " .. err end + if present then + hub_used = true + err = set_usb_port_auth(false, 2, i) + if err then return true, "error deauthorising port " .. i .. ": " .. err end + end + end + return hub_used, nil +end + +--- Re-authorize all occupied USB3 (hub 2) ports. +---@return boolean hub_was_used +---@return string? error +local function repopulate_usb3_hub() + local hub_used = false + for i = 1, #USB_PORT_SUBPATHS[2] do + local present, err = is_device_on_hub_port(2, i) + if err then return false, "error checking port " .. i .. ": " .. err end + if present then + hub_used = true + err = set_usb_port_auth(true, 2, i) + if err then return true, "error reauthorising port " .. i .. ": " .. err end + end + end + return hub_used, nil +end + +--- Control USB hub power via uhubctl. +---@param enabled boolean +---@param hub integer +---@return string? error +local function set_usb_hub_power(enabled, hub) + local cmd = exec.command { 'uhubctl', '-e', '-l', tostring(hub), '-a', tostring(enabled and 1 or 0), stdin = 'null', stdout = 'null', stderr = 'null' } + local _, _, code = perform(cmd:run_op()) + if code ~= 0 then + return ("uhubctl hub %d power %s failed (exit %d)"):format( + hub, enabled and 'on' or 'off', code or -1) + end + return nil +end + +--- Read the VL805 hub controller firmware timestamp via vcgencmd. +---@return integer? timestamp +---@return string? error +local function get_vl805_timestamp() + local cmd = exec.command { 'vcgencmd', 'bootloader_version', stdin = 'null', stdout = 'pipe', stderr = 'stdout' } + local out, _, code = perform(cmd:output_op()) + if code ~= 0 or not out then + return nil, "vcgencmd bootloader_version failed" + end + local timestamp = out:match("timestamp%s+(%d+)") + if not timestamp then + return nil, "timestamp not found in bootloader version output" + end + return tonumber(timestamp), nil +end + +---@param enabled boolean +---@param logger Logger? +local function emit_state(emit_ch, bus_id, enabled, logger) + if not emit_ch then return end + local payload, err = hal_types.new.Emit('usb', bus_id, 'state', 'bus', { enabled = enabled }) + if not payload then + dlog(logger, 'debug', { what = 'state_emit_failed', err = tostring(err) }) + return + end + emit_ch:put(payload) +end + +---- capability verbs ---- + +---@param _opts table? +---@return boolean ok +---@return any value_or_err +function UsbDriver:enable(_opts) + if self.enabled then + return true, nil + end + + local auth_err = set_usb_hub_auth_default(true, 2) + if auth_err then + dlog(self.logger, 'warn', { what = 'restore_authorized_default_failed', err = tostring(auth_err) }) + end + + local power_err = set_usb_hub_power(true, 2) + if power_err then + return false, power_err + end + + self.enabled = true + emit_state(self.cap_emit_ch, self.bus_id, true) + return true, nil +end + +---@param _opts table? +---@return boolean ok +---@return any value_or_err +function UsbDriver:disable(_opts) + if not self.enabled then + return true, nil + end + + -- Verify VL805 firmware supports hub power control. + local vl805_ts, ts_err = get_vl805_timestamp() + if ts_err then + return false, "could not verify VL805 firmware: " .. ts_err + end + if vl805_ts < VL805_SUPPORTED_FROM then + return false, ("VL805 firmware too old (timestamp %d, need >= %d)"):format( + vl805_ts, VL805_SUPPORTED_FROM) + end + + -- Deauthorize each connected USB3 port so devices fall back to USB2. + local hub_used, clear_err = clear_usb3_hub() + if clear_err then + dlog(self.logger, 'error', { what = 'clear_usb3_hub_failed', err = tostring(clear_err) }) + repopulate_usb3_hub() + set_usb_hub_auth_default(true, 2) + return false, clear_err + end + if not hub_used then + -- No active USB3 connections; still prevent future connections. + dlog(self.logger, 'info', { what = 'no_usb3_devices_connected' }) + set_usb_hub_auth_default(false, 2) + self.enabled = false + emit_state(self.cap_emit_ch, self.bus_id, false, self.logger) + return true, nil + end + + -- Prevent future USB3 connections. + local auth_err = set_usb_hub_auth_default(false, 2) + if auth_err then + dlog(self.logger, 'warn', { what = 'set_authorized_default_failed', err = tostring(auth_err) }) + end + + -- Power down USB3 hub to force devices onto USB2. + local power_err = set_usb_hub_power(false, 2) + if power_err then + dlog(self.logger, 'error', { what = 'power_down_usb3_hub_failed', err = tostring(power_err) }) + set_usb_hub_power(true, 2) + repopulate_usb3_hub() + set_usb_hub_auth_default(true, 2) + return false, power_err + end + + -- Wait up to 10 seconds for USB3 devices to re-enumerate on the USB2 hub. + local awaiting_1 = is_device_on_hub_port(2, 1) + local awaiting_2 = is_device_on_hub_port(2, 2) + local migrated = false + for _ = 1, 10 do + local p1_ok = not (awaiting_1 and not is_device_on_hub_port(1, 1)) + local p2_ok = not (awaiting_2 and not is_device_on_hub_port(1, 2)) + if p1_ok and p2_ok then migrated = true; break end + perform(sleep.sleep_op(1)) + end + if not migrated then + dlog(self.logger, 'warn', { what = 'usb3_migration_incomplete' }) + end + + self.enabled = false + emit_state(self.cap_emit_ch, self.bus_id, false, self.logger) + return true, nil +end + +---- control manager ---- + +function UsbDriver:control_manager() + fibers.current_scope():finally(function() + dlog(self.logger, 'debug', { what = 'control_manager_exiting' }) + end) + + while true do + local request, req_err = self.control_ch:get() + if not request then + dlog(self.logger, 'debug', { what = 'control_ch_closed', err = tostring(req_err) }) + break + end + + local fn = self[request.verb] + local ok, value_or_err + if type(fn) ~= 'function' then + ok, value_or_err = false, "unsupported verb: " .. tostring(request.verb) + else + local st, _, r1, r2 = fibers.run_scope(function() + return fn(self, request.opts) + end) + if st ~= 'ok' then + ok, value_or_err = false, "internal error: " .. tostring(r1) + else + ok, value_or_err = r1, r2 + end + end + + local reply = hal_types.new.Reply(ok, value_or_err) + if reply then + request.reply_ch:put(reply) + end + end +end + +---- public interface ---- + +---@return string error +function UsbDriver:init() + if self.initialised then + return "already initialised" + end + self.initialised = true + return "" +end + +---@param emit_ch Channel +---@return Capability[]? +---@return string error +function UsbDriver:capabilities(emit_ch) + if not self.initialised then + return nil, "usb driver not initialised" + end + self.cap_emit_ch = emit_ch + local cap, err = cap_types.new.UsbCapability(self.bus_id, self.control_ch) + if not cap then + return {}, err + end + return { cap }, "" +end + +---@return boolean ok +---@return string error +function UsbDriver:start() + if not self.initialised then + return false, "usb driver not initialised" + end + if self.cap_emit_ch then + -- Publish initial bus state. + emit_state(self.cap_emit_ch, self.bus_id, self.enabled, self.logger) + + -- Publish meta. + local meta_payload, meta_err = hal_types.new.Emit('usb', self.bus_id, 'meta', 'info', { + provider = 'hal', + version = 1, + bus_id = self.bus_id, + }) + if meta_payload then + self.cap_emit_ch:put(meta_payload) + else + dlog(self.logger, 'debug', { what = 'meta_emit_failed', err = tostring(meta_err) }) + end + end + + local ok, spawn_err = self.scope:spawn(function() + self:control_manager() + end) + if not ok then + return false, "failed to spawn control_manager: " .. tostring(spawn_err) + end + return true, "" +end + +---@param timeout number? +---@return boolean ok +---@return string error +function UsbDriver:stop(timeout) + timeout = timeout or 5 + self.scope:cancel(('usb driver [%s] stopped'):format(self.bus_id)) + local source = perform(op.named_choice { + join = self.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + if source == 'timeout' then + return false, ("usb driver [%s] stop timeout"):format(self.bus_id) + end + return true, "" +end + +---@param bus_id string capability id, e.g. "usb3" +---@param logger Logger? +---@return UsbDriver? +---@return string error +local function new(bus_id, logger) + bus_id = bus_id or 'usb3' + + local scope, err = fibers.current_scope():child() + if not scope then + return nil, "failed to create child scope: " .. tostring(err) + end + + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + dlog(logger, 'error', { what = 'scope_failed', err = tostring(primary), status = st }) + end + dlog(logger, 'debug', { what = 'stopped' }) + end) + + local enabled = probe_usb3_state() + + return setmetatable({ + bus_id = bus_id, + scope = scope, + control_ch = channel.new(CONTROL_Q_LEN), + cap_emit_ch = nil, + enabled = enabled, + logger = logger, + initialised = false, + }, UsbDriver), "" +end + +return { new = new } diff --git a/src/services/hal/drivers/wired.lua b/src/services/hal/drivers/wired.lua new file mode 100644 index 00000000..4e2b3faf --- /dev/null +++ b/src/services/hal/drivers/wired.lua @@ -0,0 +1,56 @@ +-- services/hal/drivers/wired.lua +-- Thin semantic driver facade around wired-provider backends. + +local provider_loader = require 'services.hal.backends.wired.provider' + +local M = {} +local Driver = {} +Driver.__index = Driver + +local REQUIRED_BACKEND_OPS = { + 'snapshot_op', + 'watch_op', + 'observe_groups_op', + 'apply_attachments_op', + 'set_poe_op', + 'bounce_op', +} + +local function validate_backend(backend, provider_name) + for _, opname in ipairs(REQUIRED_BACKEND_OPS) do + if type(backend[opname]) ~= 'function' then + return nil, ('wired provider %s missing required %s'):format(tostring(provider_name), opname) + end + end + return true, nil +end + +function M.new(config, opts) + opts = opts or {} + local backend, err, provider_name = provider_loader.new(config or {}, opts) + if not backend then return nil, err end + local ok, verr = validate_backend(backend, provider_name) + if not ok then + if type(backend.terminate) == 'function' then backend:terminate('invalid backend') end + return nil, verr + end + return setmetatable({ + backend = backend, + provider_name = provider_name, + provider_id = opts.provider_id, + }, Driver), nil +end + +function Driver:snapshot_op(req) return self.backend:snapshot_op(req) end +function Driver:watch_op(req) return self.backend:watch_op(req) end +function Driver:observe_groups_op(req) return self.backend:observe_groups_op(req) end +function Driver:apply_attachments_op(req) return self.backend:apply_attachments_op(req) end +function Driver:set_poe_op(req) return self.backend:set_poe_op(req) end +function Driver:bounce_op(req) return self.backend:bounce_op(req) end + +function Driver:terminate(reason) + if self.backend and type(self.backend.terminate) == 'function' then return self.backend:terminate(reason) end + return true, nil +end + +return M diff --git a/src/services/hal/drivers/wireless.lua b/src/services/hal/drivers/wireless.lua deleted file mode 100644 index 5cc52aa1..00000000 --- a/src/services/hal/drivers/wireless.lua +++ /dev/null @@ -1,777 +0,0 @@ -local fiber = require "fibers.fiber" -local op = require "fibers.op" -local queue = require "fibers.queue" -local channel = require "fibers.channel" -local sleep = require "fibers.sleep" -local service = require "service" -local log = require "services.log" -local hal_capabilities = require "services.hal.hal_capabilities" -local iw = require "services.hal.drivers.wireless.iw" -local hal_utils = require "services.hal.utils" -local utils = require "services.hal.drivers.wireless.utils" -local sc = require "fibers.utils.syscall" -local new_msg = require "bus".new_msg - -local unpack = table.unpack or unpack - -local DEFAULT_REPORT_PERIOD = 3600 -local VALID_BANDS = { "2g", "5g" } -local VALID_HTMODES = { - "HE20", "HE40+", "HE40-", "HE80", "HE160", - "HT20", "HT40+", "HT40-", - "VHT20", "VHT40+", "VHT40-", "VHT80", "VHT160" -} -local VALID_ENCRYPTIONS = { - "none", "wep", "psk", "psk2", "psk-mixed", - "sae", "sae-mixed", "owe", "wpa", "wpa2", "wpa3" -} -local VALID_MODES = { "ap", "sta", "adhoc", "mesh", "monitor" } -local DELETE_MARKER = true - ----@class WirelessDriver ----@field ctx Context The driver's context ----@field interface string ----@field name string ----@field path string ----@field type string ----@field phy string ----@field report_period_ch Channel Channel for setting the report period of metrics ----@field phy_channel Channel Channel for sending the phy name once known ----@field interface_event_queue Queue Queue for receiving interface events ----@field client_event_queue Queue Queue for receiving client events ----@field command_q Queue Queue for receiving commands ----@field info_q Queue Queue for sending information updates -local WirelessDriver = {} -WirelessDriver.__index = WirelessDriver - -local function list_is_type(value, expected_type) - for _, t in ipairs(value) do - if type(t) == expected_type then - return true - end - end - return false -end - -local function is_list(list) - local i = 1 - for k, _ in pairs(list) do - if i ~= k then - return false - end - i = i + 1 - end - return true -end - -------------------------------------------------------------------------- ---- WirelessDriverCapabilities ------------------------------------------- - -function WirelessDriver:set_report_period(ctx, period) - op.choice( - self.report_period_ch:put_op(period), - ctx:done_op() - ):perform() - return ctx:err() == nil, ctx:err() -end - -function WirelessDriver:set_channels(ctx, band, channel, htmode, channels) - if type(band) ~= "string" or not hal_utils.is_in(band, VALID_BANDS) then - return nil, "Invalid band, must be one of: " .. table.concat(VALID_BANDS, ", ") - end - if type(htmode) ~= "string" or not hal_utils.is_in(htmode, VALID_HTMODES) then - return nil, "Invalid htmode, must be one of: " .. table.concat(VALID_HTMODES, ", ") - end - if channel == 'auto' then - if channels == nil then - return nil, "Channels must be provided when setting channel to 'auto'" - end - if not is_list(channels) then - return nil, "Channels must be a list when setting channel to 'auto'" - end - if not (list_is_type(channels, "number") or list_is_type(channels, "string")) then - return nil, "Channels must be a list of numbers or strings when setting channel to 'auto'" - end - end - if channel ~= 'auto' and type(channel) ~= "number" and type(channel) ~= "string" then - return nil, "Invalid channel, must be a number, string or 'auto'" - end - - local reqs = {} - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', self.name, 'band', band } - )) - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', self.name, 'htmode', htmode } - )) - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', self.name, 'channel', channel } - )) - - if channel == 'auto' then - local cat_channels = table.concat(channels, " ") - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', self.name, 'channels', cat_channels } - )) - end - - for _, req in ipairs(reqs) do - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - end - - return true, nil -end - -function WirelessDriver:set_country(ctx, country_code) - if type(country_code) ~= "string" or #country_code ~= 2 then - return nil, "Invalid country code, must be a 2-letter ISO code" - end - - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', self.name, 'country', country_code:upper() } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - return true, nil -end - -function WirelessDriver:set_txpower(ctx, txpower) - if type(txpower) ~= "number" and type(txpower) ~= "string" then - return nil, "Invalid txpower, must be a number or a string" - end - - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', self.name, 'txpower', txpower } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - return true, nil -end - -function WirelessDriver:set_type(ctx, radio_type) - if type(radio_type) ~= "string" then - return nil, "Invalid radio type, must be a string" - end - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', self.name, 'type', radio_type } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - return true, nil -end - -function WirelessDriver:set_enabled(ctx, enabled) - if type(enabled) ~= 'boolean' then - return nil, "Invalid enabled state, must be a boolean" - end - local disabled = enabled and '0' or '1' - - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', self.name, 'disabled', disabled } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - return true, nil -end - -function WirelessDriver:add_interface(ctx, ssid, encryption, password, net_interface, mode, optionals) - if type(ssid) ~= "string" or #ssid == 0 then - return nil, "Invalid SSID, must be a non-empty string" - end - if type(encryption) ~= "string" or not hal_utils.is_in(encryption, VALID_ENCRYPTIONS) then - return nil, "Invalid encryption, must be one of: " .. table.concat(VALID_ENCRYPTIONS, ", ") - end - if type(password) ~= 'string' then - return nil, "Invalid password, must be a string" - end - if type(net_interface) ~= "string" or #net_interface == 0 then - return nil, "Invalid network interface, must be a non-empty string" - end - if not hal_utils.is_in(mode, VALID_MODES) then - return nil, "Invalid mode, must be one of: " .. table.concat(VALID_MODES, ", ") - end - - local wifi_interface = string.format("%s_i%s", self.name, self.iface_num) - self.iface_num = self.iface_num + 1 - local set_sub = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', wifi_interface, 'wifi-iface' } - )) - local set_response, ctx_err = set_sub:next_msg_with_context(ctx) - set_sub:unsubscribe() - if set_response.payload and set_response.payload.err or ctx_err then - return nil, set_response.payload.err or ctx_err - end - - local reqs = {} - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', wifi_interface, 'device', self.name } - )) - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', wifi_interface, 'ssid', ssid } - )) - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', wifi_interface, 'encryption', encryption } - )) - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', wifi_interface, 'key', password } - )) - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', wifi_interface, 'network', net_interface } - )) - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', wifi_interface, 'mode', mode } - )) - - if optionals.enable_steering then - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', wifi_interface, 'bss_transition', '1' } - )) - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', wifi_interface, 'ieee80211k', '1' } - )) - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', wifi_interface, 'rrm_neighbor_report', '1' } - )) - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', wifi_interface, 'rrm_beacon_report', '1' } - )) - end - - for _, req in ipairs(reqs) do - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - end - - return wifi_interface, nil -end - -function WirelessDriver:delete_interface(ctx, interface) - if type(interface) ~= "string" or #interface == 0 then - return nil, "Invalid interface, must be a non-empty string" - end - - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'delete' }, - { 'wireless', interface } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - - return true, nil -end - -function WirelessDriver:clear_radio_config(ctx) - local delete_req = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'delete' }, - { 'wireless', self.name } - )) - local delete_resp, ctx_err = delete_req:next_msg_with_context(ctx) - delete_req:unsubscribe() - if ctx_err or (delete_resp.payload and delete_resp.payload.err) then - return nil, ctx_err or delete_resp.payload.err - end - - local reqs = {} - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', self.name, 'wifi-device' } - )) - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', self.name, 'path', self.path } - )) - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { 'wireless', self.name, 'type', self.type } - )) - - for _, req in ipairs(reqs) do - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or (resp.payload and resp.payload.err) then - return nil, ctx_err or resp.payload.err - end - end - return true, nil -end - -function WirelessDriver:apply(ctx) - local commit_sub = self.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'commit' }, - { 'wireless' } - )) - local commit_response, ctx_err = commit_sub:next_msg_with_context(ctx) - if ctx_err or (commit_response.payload and commit_response.payload.err) then - return nil, ctx_err or commit_response.payload.err - end - return true, nil -end - -------------------------------------------------------------------------- -------------------------------------------------------------------------- - ---- Handle a capability request ---- @param ctx Context ---- @param request table The capability request to handle -function WirelessDriver:handle_capability(ctx, request) - local command = request.command - local args = request.args or {} - local ret_ch = request.return_channel - - if type(ret_ch) == 'nil' then return end - - if type(command) == "nil" then - ret_ch:put({ - result = nil, - err = 'No command was provided' - }) - return - end - - local func = self[command] - if type(func) ~= "function" then - ret_ch:put({ - result = nil, - err = "Command does not exist" - }) - return - end - - fiber.spawn(function() - local result, err = func(self, ctx, unpack(args)) - - ret_ch:put({ - result = result, - err = err - }) - end) -end - ---- Main driver loop ---- @param ctx Context The driver's context -function WirelessDriver:_main(ctx) - local ubus_sub = self.conn:subscribe({ 'hal', 'capability', 'uci', '1' }) - ubus_sub:next_msg_with_context(ctx) -- Wait for initial message - log.info(string.format( - "%s - %s: Starting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - - -- Main event loop - while not ctx:err() do - op.choice( - self.command_q:get_op():wrap(function(req) - self:handle_capability(ctx, req) - end), - self.ctx:done_op() - ):perform() - end - - log.info(string.format( - "%s - %s: Exiting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) -end - --- pause till enabled and interface set -function WirelessDriver:_monitor_clients(ctx) - local phy = self.phy_channel:get() - log.trace(string.format( - "%s - %s: Starting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - - local cmd = iw.get_iw_event_stream(ctx) - local stdout = cmd:stdout_pipe() - if not stdout then - log.error(string.format( - "%s - %s: Failed to get stdout pipe for iw event stream", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - cmd:kill() - cmd:wait() - return - end - - local err = cmd:start() - if err then - log.error(string.format( - "%s - %s: Failed to start iw event stream, reason: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - err - )) - stdout:close() - return - end - - local done = false - while not ctx:err() and not done do - op.choice( - stdout:read_line_op():wrap(function(line) - if not line then - ctx:cancel("command closed") - return - end - -- Pattern: interface: (new|del) station - local iface, event, mac = string.match(line, "^([%w%-]+):%s*(new)%s+station%s+([%x:]+)") - if not iface then - iface, event, mac = string.match(line, "^([%w%-]+):%s*(del)%s+station%s+([%x:]+)") - end - if iface and event and mac then - local client_phy = iface:match("^(phy%d+)") - if client_phy == phy then - op.choice( - self.client_event_queue:put_op({ - connected = (event == "new"), - interface = iface, - mac = mac - }), - ctx:done_op() - ):perform() - end - end - end), - ctx:done_op():wrap(function() - cmd:kill() - done = true - end) - ):perform() - end - cmd:wait() - stdout:close() - - log.trace(string.format( - "%s - %s: Exiting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) -end - -local function get_iface_stats(ctx, interface) - local net_statistics = { - "rx_bytes", - "tx_bytes", - "rx_packets", - "tx_packets", - "rx_dropped", - "tx_dropped", - "rx_errors", - "tx_errors", - } - local stats = {} - - local info, info_err = iw.get_iw_dev_info(ctx, interface) - if not info_err then - for k, v in pairs(info) do - stats[k] = v - end - end - - for _, stat in ipairs(net_statistics) do - local value, value_err = utils.get_net_statistic(interface, stat) - if not value_err then - stats[stat] = value - end - end - - local noise, noise_err = iw.get_dev_noise(ctx, interface) - if not noise_err then - stats["noise"] = noise - end - - return stats -end - -local dump = require "fibers.utils.helper".dump --- pause till enabled and interface set -function WirelessDriver:_report_metrics(ctx) - log.trace(string.format( - "%s - %s: Starting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - local report_period = DEFAULT_REPORT_PERIOD - local next_report = sc.monotime() + report_period - local interfaces = {} - local phy = nil - - while not ctx:err() do - op.choice( - self.report_period_ch:get_op():wrap(function(period) - next_report = next_report - report_period + period -- Adjust next_report for new period - report_period = period - end), - self.interface_event_queue:get_op():wrap(function(event) - if not phy then - phy = event.interface:match("^(phy%d+)") - if phy then self.phy_channel:put(phy) end - end - if event.connected then - if not interfaces[event.interface] then - interfaces[event.interface] = {} - end - else - interfaces[event.interface] = nil - end - end), - self.client_event_queue:get_op():wrap(function(event) - if not event then return end - if not interfaces[event.interface] then return end - if type(event.connected) ~= "nil" then - self.info_q:put({ - type = "wireless", - id = self.name, - sub_topic = { "interface", event.interface, "client", event.mac }, - endpoints = "single", - info = { - connected = event.connected, - timestamp = sc.realtime() - } - }) - end - if event.connected then - interfaces[event.interface][event.mac] = true - local client_info, client_err = iw.get_client_info(ctx, event.interface, event.mac) - if not client_err then - -- Try to get hostname for the client - local hostname, hostname_err = utils.get_hostname(ctx, event.mac) - if hostname and not hostname_err then - client_info.hostname = hostname - end - - self.info_q:put({ - type = "wireless", - id = self.name, - sub_topic = { "interface", event.interface, "client", event.mac }, - endpoints = "multiple", - info = client_info - }) - end - else - interfaces[event.interface][event.mac] = nil - end - end), - sleep.sleep_until_op(next_report):wrap(function() - local client_stats = {} - for interface, iclients in pairs(interfaces) do - local stats = get_iface_stats(ctx, interface) - if next(stats) then - client_stats[interface] = stats - end - for mac, _ in pairs(iclients) do - local client_info, client_err = iw.get_client_info(ctx, interface, mac) - if not client_err then - client_stats[interface] = client_stats[interface] or {} - client_stats[interface].client = client_stats[interface].client or {} - client_stats[interface].client[mac] = client_info - end - end - end - if next(client_stats) then - self.info_q:put({ - type = "wireless", - id = self.name, - sub_topic = { "interface" }, - endpoints = "multiple", - info = client_stats - }) - end - next_report = sc.monotime() + report_period - end), - ctx:done_op() - ):perform() - end - log.trace(string.format( - "%s - %s: Stopping", - ctx:value("service_name"), - ctx:value("fiber_name") - )) -end - -function WirelessDriver:get_name() - return self.name -end - -function WirelessDriver:get_phy() - return self.phy -end - -function WirelessDriver:get_path() - return self.path -end - ---- Register and apply driver capabilities ---- @param capability_info_q Queue Queue for sending capability information updates ---- @return table capabilities The capabilities exposed by this driver ---- @return string? error Error message if capabilities couldn't be applied -function WirelessDriver:apply_capabilities(capability_info_q) - self.info_q = capability_info_q - - -- Example of registering capabilities - customize based on driver needs - local capabilities = { - wireless = { - control = hal_capabilities.new_wireless_capability(self.command_q), - id = self.name -- unique identifier for this capability instance - } - } - - return capabilities, nil -end - --- This is not ideal --- Figuring out phy from ifname should be done in init --- but getting the associated interface from uci data is not reliable --- due to out wifi card being one device split into two phys with the same path --- therefore we cannot know the phy until we have an interface up event. --- We could have an interface up event listener per driver but it is much more efficient and simple --- to centralize it in the manager and just set the phy here when we get the event. --- more research is needed to find a better way to assign a phy to a driver -function WirelessDriver:attach_interface(interface) - if not self.phy then - self.phy = interface:match("^(phy%d+)") - end - local interface_event = { - connected = true, - interface = interface - } - self.interface_event_queue:put(interface_event) -end - -function WirelessDriver:detach_interface(interface) - if not self.phy then - self.phy = interface:match("^(phy%d+)") - end - local interface_event = { - connected = false, - interface = interface - } - self.interface_event_queue:put(interface_event) -end - ---- Spawn driver fiber ---- @param conn Connection The bus connection -function WirelessDriver:spawn(conn) - service.spawn_fiber(string.format("Wireless Main (%s)", self.name), conn, self.ctx, function(fctx) - self:_main(fctx) - end) - service.spawn_fiber(string.format("Wireless Metrics Reporter (%s)", self.name), conn, self.ctx, function(fctx) - self:_report_metrics(fctx) - end) - service.spawn_fiber(string.format("Wireless Client Monitor (%s)", self.name), conn, self.ctx, function(fctx) - self:_monitor_clients(fctx) - end) -end - -function WirelessDriver:init(conn) - self.conn = conn - -- Delete the radio device section and rebuild it with only the name, path and type - local ok, err = self:clear_radio_config(self.ctx) - if not ok then - return err - end - - -- Delete all wifi-iface sections associated with this device - local req = conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'foreach' }, - { 'wireless', 'wifi-iface', function(cursor, section) - if section["device"] == self.name then - cursor:delete('wireless', section[".name"]) - end - end } - )) - local resp, ctx_err = req:next_msg_with_context(self.ctx) - req:unsubscribe() - -- We don't care if there was an error here, the wifi-iface sections might not exist - - -- Apply changes - local commit_sub = conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'commit' }, - { 'wireless' } - )) - - local commit_response, ctx_err = commit_sub:next_msg_with_context(self.ctx) - if ctx_err or (commit_response.payload and commit_response.payload.err) then - return ctx_err or commit_response.payload.err - end - - return nil -end - ---- Create a new driver instance ---- @param ctx Context The context for this driver ---- @return WirelessDriver The new driver instance -local function new(ctx, name, path, type) - local self = setmetatable({}, WirelessDriver) - self.ctx = ctx - self.command_q = queue.new(10) - self.name = name - self.path = path - self.type = type - self.phy = nil - self.iface_num = 0 - self.report_period_ch = channel.new() - self.client_event_queue = queue.new(10) - self.interface_event_queue = queue.new(10) - self.phy_channel = channel.new() - - return self -end - -return { - new = new -} diff --git a/src/services/hal/drivers/wireless/iw.lua b/src/services/hal/drivers/wireless/iw.lua deleted file mode 100644 index f155a4b8..00000000 --- a/src/services/hal/drivers/wireless/iw.lua +++ /dev/null @@ -1,40 +0,0 @@ -local exec = require "fibers.exec" -local utils = require "services.hal.drivers.wireless.utils" - -local function get_iw_dev_info(ctx, interface) - local out, err = exec.command_context(ctx, "iw", "dev", interface, "info"):output() - if err then - return nil, err - end - - return utils.format_iw_dev_info(out) -end - -local function get_iw_event_stream(ctx) - return exec.command_context(ctx, "iw", "event") -end - -local function get_client_info(ctx, interface, mac) - local out, err = exec.command_context(ctx, "iw", "dev", interface, "station", "get", mac):output() - if err then - return nil, err - end - - return utils.format_iw_client_info(out) -end - -local function get_dev_noise(ctx, interface) - local out, err = exec.command_context(ctx, "iw", interface, "survey", "dump"):output() - if err then - return nil, err - end - - return utils.parse_dev_noise(out) -end - -return { - get_iw_dev_info = get_iw_dev_info, - get_iw_event_stream = get_iw_event_stream, - get_client_info = get_client_info, - get_dev_noise = get_dev_noise -} diff --git a/src/services/hal/drivers/wireless/utils.lua b/src/services/hal/drivers/wireless/utils.lua deleted file mode 100644 index 1cb3502e..00000000 --- a/src/services/hal/drivers/wireless/utils.lua +++ /dev/null @@ -1,242 +0,0 @@ -local file = require "fibers.stream.file" -local exec = require "fibers.exec" - -local function extract_channel(line) - local chan, freq, width, center1 = line:match( - "%s*channel%s+(%d+)%s+%(([%d%s]+)MHz%)%s*,%s*width:%s*([%d%s]+)MHz,%s*center1:%s*([%d%s]+)MHz" - ) - if chan and freq and width and center1 then - return { - chan = tonumber(chan), - freq = tonumber(freq), - width = tonumber(width), - center1 = tonumber(center1) - } - end - return nil -end - ---- Convert a string output of the 'iw dev info' command into a lua table ---- @param raw_output string The string output from the 'iw dev info' command ---- @return table? ---- @return string? error -local function format_iw_dev_info(raw_output) - if not raw_output or raw_output == "" then - return nil, "No input provided" - end - - local result = {} - local lines = {} - - -- Split the output into lines - for line in raw_output:gmatch("[^\r\n]+") do - table.insert(lines, line) - end - - -- Extract interface name from the first line - result.interface = lines[1]:match("Interface%s+(.+)") - if not result.interface then - return nil, "Failed to parse interface name" - end - - local i = 2 -- Start from the second line - while i <= #lines do - local line = lines[i] - - -- Handle multicast TXQ section - if line:match("%s*multicast TXQ:") then - local multicast = {} - result.multicast = multicast - - -- Next line should contain headers - if i + 1 <= #lines then - local headers = {} - for header in lines[i + 1]:gsub("^%s+", ""):gmatch("%S+") do - table.insert(headers, header) - end - - -- Line after headers should contain values - if i + 2 <= #lines then - -- Parse the line once and extract all values - local values = {} - for value in lines[i + 2]:gsub("^%s+", ""):gmatch("%S+") do - table.insert(values, value) - end - - -- Now associate each header with its corresponding value - for j, header in ipairs(headers) do - if values[j] then - multicast[header] = tonumber(values[j]) or values[j] - end - end - end - i = i + 2 -- Skip the headers and values lines - end - -- Handle regular key-value pairs - else - if line:match("%s*ifindex%s+(%d+)") then - result.ifindex = tonumber(line:match("%s*ifindex%s+(%d+)")) - elseif line:match("%s*wdev%s+(%x+)") then - result.wdev = line:match("%s*wdev%s+(%x+)") - elseif line:match("%s*addr%s+([%x:]+)") then - result.addr = line:match("%s*addr%s+([%x:]+)") - elseif line:match("%s*ssid%s+(.+)") then - result.ssid = line:match("%s*ssid%s+(.+)") - elseif line:match("%s*type%s+(%S+)") then - result.type = line:match("%s*type%s+(%S+)") - elseif line:match("%s*wiphy%s+(%d+)") then - result.wiphy = tonumber(line:match("%s*wiphy%s+(%d+)")) - elseif line:match("%s*txpower%s+([%d.]+)%s+dBm") then - result.txpower = tonumber(line:match("%s*txpower%s+([%d.]+)%s+dBm")) - elseif line:match("%s*channel") then - result.channel = extract_channel(line) - end - end - - i = i + 1 - end - - return result, nil -end - ---- Parse the output of "iw dev station get " into a structured table ---- @param raw_output string The raw output from the iw command ---- @return table Parsed station information -local function format_iw_client_info(raw_output) - local result = {} - - -- Process each line - for line in raw_output:gmatch("[^\r\n]+") do - -- Skip the header line - if not line:match("^Station") then - -- Extract key and value - local key, value = line:match("^%s+([^:]+):%s+(.+)$") - - if key and value then - -- Clean up key name: convert spaces to underscores and lowercase - key = key:gsub("%s+", "_"):lower() - - -- Handle special case for associated at [boottime] - if key == "associated_at_[boottime]" then - local boottime = value:match("(%d+%.%d+)s") - result["associated_at_boottime"] = tonumber(boottime) - -- Handle bitrates (e.g. "6.0 MBit/s") - elseif value:match("^%-?%d+%.%d+%s+%w+/s$") then - local num, unit = value:match("^(%-?%d+%.%d+)%s+(.+)") - result[key] = tonumber(num) - result[key .. "_unit"] = unit - -- Handle signal with range (e.g. "-33 [-36, -35] dBm") - elseif value:match("^%-?%d+%s+%[") then - local main = value:match("^(%-?%d+)") - result[key] = tonumber(main) - -- Handle values with units (e.g. "1390 ms") - elseif value:match("^%-?%d+%s+%w+$") then - local num, unit = value:match("^(%-?%d+)%s+(.+)") - result[key] = tonumber(num) - result[key .. "_unit"] = unit - -- Handle integer values - elseif value:match("^%-?%d+$") then - result[key] = tonumber(value) - -- Handle boolean-like values - elseif value == "yes" or value == "no" then - result[key] = value - -- Handle associated timestamp - elseif key == "associated_at" or key == "current_time" then - result[key] = tonumber(value:match("(%d+)")) - -- Default case: keep as string - else - result[key] = value - end - end - end - end - - return result -end - -local function get_net_statistic(interface, statistic) - local filedata, file_err = file.open("/sys/class/net/" .. interface .. "/statistics/" .. statistic, "r") - if file_err then - return nil, file_err - end - local content, content_err = filedata:read_all_chars() - if content_err then - return nil, content_err - end - - filedata:close() - - local num = tonumber(string.match(content or "", "(%d+)")) - if num == nil then return nil, statistic .. " invalid number" end - - return num, nil -end - -local function parse_dev_noise(raw_output) - if not raw_output or raw_output == "" then - return nil, "No input provided" - end - - local in_use_section = false - - for line in raw_output:gmatch("[^\r\n]+") do - if not in_use_section then - if line:find("%[in use%]") then - in_use_section = true - end - else - if line:find("noise:") then - local value = line:match("noise:%s*([%-]?%d+%.?%d*)") - if value then - return tonumber(value), nil - end - return nil, "Noise value missing" - end - end - end - - return nil, "Noise value not found" -end - ---- Get the hostname for a MAC address by searching DHCP lease files ---- @param ctx Context The context for this operation (for cancellation) ---- @param mac_address string The MAC address to look up ---- @return string? hostname The hostname if found ---- @return string? error Error message if any -local function get_hostname(ctx, mac_address) - -- Find all DHCP lease files using command_context - local cmd = exec.command_context(ctx, "sh", "-c", "ls /tmp/dhcp.* 2>/dev/null") - local output, err = cmd:output() - - if err or not output or output == "" then - return nil, nil -- No DHCP lease files found, not an error - end - - -- Search each DHCP lease file - for dhcp_lease_file in string.gmatch(output, "%S+") do - local filedata, file_err = file.open(dhcp_lease_file, "r") - if not file_err then - local content, content_err = filedata:read_all_chars() - filedata:close() - - if not content_err and content then - for line in string.gmatch(content, "[^\n]+") do - local _, mac, _, hostname = string.match(line, "(%S+)%s+(%S+)%s+(%S+)%s+(%S+)") - if mac and hostname and mac == mac_address then - return hostname, nil - end - end - end - end - end - - return nil, nil -- Not found, but not an error -end - -return { - format_iw_dev_info = format_iw_dev_info, - format_iw_client_info = format_iw_client_info, - get_net_statistic = get_net_statistic, - parse_dev_noise = parse_dev_noise, - get_hostname = get_hostname -} diff --git a/src/services/hal/hal_capabilities.lua b/src/services/hal/hal_capabilities.lua deleted file mode 100644 index e4658cf6..00000000 --- a/src/services/hal/hal_capabilities.lua +++ /dev/null @@ -1,357 +0,0 @@ -local channel = require "fibers.channel" - -local function do_command(driver_q, cmd) - cmd.return_channel = channel.new() - driver_q:put(cmd) - return cmd.return_channel:get() -end - -local ModemCapability = {} -ModemCapability.__index = ModemCapability - -local function new_modem_capability(driver_q) - return setmetatable({driver_q = driver_q}, ModemCapability) -end - -function ModemCapability:enable() - local cmd = {command = "enable"} - return do_command(self.driver_q, cmd) -end - -function ModemCapability:disable() - local cmd = {command = "disable"} - return do_command(self.driver_q, cmd) -end - -function ModemCapability:restart() - local cmd = {command = "restart"} - return do_command(self.driver_q, cmd) -end - -function ModemCapability:connect(args) - local cmd = {command = "connect", args = args} - return do_command(self.driver_q, cmd) -end - -function ModemCapability:disconnect() - local cmd = {command = "disconnect"} - return do_command(self.driver_q, cmd) -end - -function ModemCapability:sim_detect() - local cmd = { command = "sim_detect" } - return do_command(self.driver_q, cmd) -end - -function ModemCapability:fix_failure() - local cmd = { command = "fix_failure" } - return do_command(self.driver_q, cmd) -end - -function ModemCapability:set_signal_update_freq(args) - local cmd = { command = "set_signal_update_freq", args = args } - return do_command(self.driver_q, cmd) -end - --- geo cap -local GeoCapability = {} -GeoCapability.__index = GeoCapability - -local function new_geo_capability(driver_q) - return setmetatable({driver_q = driver_q}, GeoCapability) -end - -local TimeCapability = {} -TimeCapability.__index = TimeCapability - -local function new_time_capability(driver_q) - return setmetatable({ driver_q = driver_q }, TimeCapability) -end - --- uci cap -local UCICapability = {} -UCICapability.__index = UCICapability - -local function new_uci_capability(driver_q) - return setmetatable({driver_q = driver_q}, UCICapability) -end - -function UCICapability:get(args) - local cmd = {command = "get", args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:set(args) - local cmd = {command = "set", args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:delete(args) - local cmd = {command = "delete", args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:commit(args) - local cmd = {command = "commit", args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:show(args) - local cmd = {command = "show", args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:add(args) - local cmd = {command = "add", args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:revert(args) - local cmd = {command = "revert", args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:changes(args) - local cmd = {command = "changes", args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:foreach(args) - local cmd = {command = "foreach", args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:set_restart_actions(args) - local cmd = {command = "set_restart_actions", args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:ifup(args) - local cmd = {command = 'ifup', args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:halt_restarts() - local cmd = {command = 'halt_restarts'} - return do_command(self.driver_q, cmd) -end - -function UCICapability:continue_restarts() - local cmd = {command = 'continue_restarts'} - return do_command(self.driver_q, cmd) -end - -function UCICapability:ifup(args) - local cmd = {command = 'ifup', args = args} - return do_command(self.driver_q, cmd) -end - -function UCICapability:halt_restarts() - local cmd = {command = 'halt_restarts'} - return do_command(self.driver_q, cmd) -end - -function UCICapability:continue_restarts() - local cmd = {command = 'continue_restarts'} - return do_command(self.driver_q, cmd) -end - --- ubus cap -local UBusCapability = {} -UBusCapability.__index = UBusCapability - -local function new_ubus_capability(driver_q) - return setmetatable({driver_q = driver_q}, UBusCapability) -end - -function UBusCapability:list() - local cmd = {command = "list"} - return do_command(self.driver_q, cmd) -end - -function UBusCapability:call(args) - local cmd = {command = "call", args = args} - return do_command(self.driver_q, cmd) -end - -function UBusCapability:listen(args) - local cmd = {command = "listen", args = args} - return do_command(self.driver_q, cmd) -end - -function UBusCapability:stop_stream(args) - local cmd = {command = "stop_stream", args = args} - return do_command(self.driver_q, cmd) -end - -function UBusCapability:send(args) - local cmd = {command = "send", args = args} - return do_command(self.driver_q, cmd) -end - --- wireless cap -local WirelessCapability = {} -WirelessCapability.__index = WirelessCapability - -local function new_wireless_capability(driver_q) - return setmetatable({driver_q = driver_q}, WirelessCapability) -end - -function WirelessCapability:set_report_period(args) - local cmd = {command = "set_report_period", args = args} - return do_command(self.driver_q, cmd) -end - -function WirelessCapability:set_channels(args) - local cmd = {command = "set_channels", args = args} - return do_command(self.driver_q, cmd) -end - -function WirelessCapability:set_country(args) - local cmd = {command = "set_country", args = args} - return do_command(self.driver_q, cmd) -end - -function WirelessCapability:set_txpower(args) - local cmd = {command = "set_txpower", args = args} - return do_command(self.driver_q, cmd) -end - -function WirelessCapability:set_type(args) - local cmd = {command = "set_type", args = args} - return do_command(self.driver_q, cmd) -end - -function WirelessCapability:set_enabled(args) - local cmd = {command = "set_enabled", args = args} - return do_command(self.driver_q, cmd) -end - -function WirelessCapability:add_interface(args) - local cmd = {command = "add_interface", args = args} - return do_command(self.driver_q, cmd) -end - -function WirelessCapability:delete_interface(args) - local cmd = {command = "delete_interface", args = args} - return do_command(self.driver_q, cmd) -end - -function WirelessCapability:clear_radio_config() - local cmd = {command = "clear_radio_config"} - return do_command(self.driver_q, cmd) - -end - -function WirelessCapability:apply() - local cmd = {command = "apply"} - return do_command(self.driver_q, cmd) -end - --- band cap -local BandCapability = {} -BandCapability.__index = BandCapability - -local function new_band_capability(driver_q) - return setmetatable({driver_q = driver_q}, BandCapability) -end - -function BandCapability:set_log_level(args) - local cmd = {command = "set_log_level", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_kicking(args) - local cmd = {command = "set_kicking", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_station_counting(args) - local cmd = {command = "set_station_counting", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_rrm_mode(args) - local cmd = {command = "set_rrm_mode", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_neighbour_reports(args) - local cmd = {command = "set_neighbour_reports", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_legacy_options(args) - local cmd = {command = "set_legacy_options", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_band_priority(args) - local cmd = {command = "set_band_priority", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_band_kicking(args) - local cmd = {command = "set_band_kicking", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_support_bonus(args) - local cmd = {command = "set_support_bonus", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_update_freq(args) - local cmd = {command = "set_update_freq", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_client_inactive_kickoff(args) - local cmd = {command = "set_client_inactive_kickoff", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_cleanup(args) - local cmd = {command = "set_cleanup", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:set_networking(args) - local cmd = {command = "set_networking", args = args} - return do_command(self.driver_q, cmd) -end - -function BandCapability:apply() - local cmd = {command = "apply"} - return do_command(self.driver_q, cmd) -end - -local SerialCapability = {} -SerialCapability.__index = SerialCapability - -local function new_serial_capability(driver_q) - return setmetatable({driver_q = driver_q}, SerialCapability) -end -function SerialCapability:open(args) - local cmd = { command = "open", args = args } - return do_command(self.driver_q, cmd) -end -function SerialCapability:close() - local cmd = { command = "close" } - return do_command(self.driver_q, cmd) -end -function SerialCapability:write(args) - local cmd = { command = "write", args = args } - return do_command(self.driver_q, cmd) -end - -return { - new_modem_capability = new_modem_capability, - new_ubus_capability = new_ubus_capability, - new_geo_capability = new_geo_capability, - new_time_capability = new_time_capability, - new_uci_capability = new_uci_capability, - new_wireless_capability = new_wireless_capability, - new_band_capability = new_band_capability, - new_serial_capability = new_serial_capability -} diff --git a/src/services/hal/logger.lua b/src/services/hal/logger.lua new file mode 100644 index 00000000..4b497095 --- /dev/null +++ b/src/services/hal/logger.lua @@ -0,0 +1,72 @@ +-- services/hal/logger.lua +-- +-- Lightweight structured logger for HAL managers and drivers. +-- Wraps an emit function (level, payload) and merges a set of +-- static metadata fields into every emitted payload. + +---@class Logger +---@field _emit fun(level: string, payload: any) +---@field _fields table +local Logger = {} +Logger.__index = Logger + +local function merge(a, b) + local out = {} + if type(a) == 'table' then + for k, v in pairs(a) do out[k] = v end + end + if type(b) == 'table' then + for k, v in pairs(b) do out[k] = v end + end + return out +end + +---Create a new Logger. +---@param emit_fn fun(level: string, payload: any) +---@param fields table +---@return Logger +function Logger.new(emit_fn, fields) + return setmetatable({ + _emit = emit_fn, + _fields = fields or {}, + }, Logger) +end + +---Create a child logger that inherits fields and adds extra ones. +---@param extra_fields table +---@return Logger +function Logger:child(extra_fields) + return Logger.new(self._emit, merge(self._fields, extra_fields)) +end + +---@param payload any +function Logger:debug(payload) + if type(payload) == 'table' then payload = merge(self._fields, payload) end + self._emit('debug', payload) +end + +---@param payload any +function Logger:info(payload) + if type(payload) == 'table' then payload = merge(self._fields, payload) end + self._emit('info', payload) +end + +---@param payload any +function Logger:warn(payload) + if type(payload) == 'table' then payload = merge(self._fields, payload) end + self._emit('warn', payload) +end + +---@param payload any +function Logger:error(payload) + if type(payload) == 'table' then payload = merge(self._fields, payload) end + self._emit('error', payload) +end + +---@param payload any +function Logger:trace(payload) + if type(payload) == 'table' then payload = merge(self._fields, payload) end + self._emit('trace', payload) +end + +return Logger diff --git a/src/services/hal/managers/artifact_store.lua b/src/services/hal/managers/artifact_store.lua new file mode 100644 index 00000000..240a8a12 --- /dev/null +++ b/src/services/hal/managers/artifact_store.lua @@ -0,0 +1,408 @@ +---@module 'services.hal.managers.artifact_store' + +local fibers = require 'fibers' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local channel = require 'fibers.channel' + +local device_events = require 'services.hal.support.device_events' +local driver_reconcile = require 'services.hal.support.driver_reconcile' +local strict_manager = require 'services.hal.support.strict_manager' +local tablex = require 'shared.table' +local resource = require 'devicecode.support.resource' +local driver_mod = require 'services.hal.drivers.artifact_store_provider' + +local M = { + api_mode = 'op_only', +} + + +local STOP_TIMEOUT = 5.0 + +---@class ArtifactStoreManagerState +---@field started boolean +---@field scope Scope|nil +---@field logger table|nil +---@field dev_ev_ch Channel|nil +---@field cap_emit_ch Channel|nil +---@field cfg_ch Channel|nil +---@field generation integer +---@field drivers table +local S = { + started = false, + scope = nil, + logger = nil, + dev_ev_ch = nil, + cap_emit_ch = nil, + cfg_ch = nil, + generation = 0, + drivers = {}, +} + +local function finalise_manager_scope(scope, generation) + if S.scope ~= scope or S.generation ~= generation then + return + end + + for _, driver in pairs(S.drivers or {}) do + resource.terminate_checked(driver, 'manager finalised', 'HAL manager driver cleanup failed') + end + + S.started = false + S.scope = nil + S.logger = nil + S.dev_ev_ch = nil + S.cap_emit_ch = nil + S.cfg_ch = nil + S.drivers = {} +end + + +local function child_logger(id) + if S.logger and S.logger.child then + return S.logger:child({ + component = 'driver', + driver = 'artifact_store', + id = id, + }) + end + return S.logger +end + +local function validate_config(cfg) + cfg = cfg or {} + + if type(cfg) ~= 'table' then + return false, 'artifact_store config must be a table' + end + + local specs = cfg.stores + if specs == nil and cfg[1] == nil then + specs = { { id = 'main' } } + elseif specs == nil then + specs = cfg + end + + if type(specs) ~= 'table' then + return false, 'artifact_store.stores must be a table' + end + + local seen = {} + for i, rec in ipairs(specs) do + if type(rec) ~= 'table' then + return false, ('artifact_store.stores[%d] must be a table'):format(i) + end + + local id = rec.id or rec.name or 'main' + if type(id) ~= 'string' or id == '' then + return false, ('artifact_store.stores[%d].id must be a non-empty string'):format(i) + end + + if seen[id] then + return false, 'duplicate artifact_store provider id: ' .. id + end + seen[id] = true + end + + return true, nil +end + +local function normalise_config(cfg) + local specs = cfg.stores + if specs == nil and cfg[1] == nil then + specs = { { id = 'main' } } + elseif specs == nil then + specs = cfg + end + + local out = {} + for i = 1, #specs do + local rec = specs[i] + local id = rec.id or rec.name or 'main' + local opts = {} + for k, v in pairs(rec) do + if k ~= 'id' and k ~= 'name' then + opts[k] = v + end + end + out[id] = { id = id, opts = opts } + end + return out +end + +local deep_equal = tablex.deep_equal + +local function driver_matches_spec(driver, spec) + if not driver or not spec then + return false + end + + return deep_equal(driver.opts or {}, spec.opts or {}) +end + +local function emit_device_added_op(driver, caps) + return device_events.added_op(S.dev_ev_ch, 'artifact-store', driver.id, { provider = 'hal.artifact_store', source_id = 'platform' }, caps) +end + +local function emit_device_removed_op(driver) + return device_events.removed_op(S.dev_ev_ch, 'artifact-store', driver.id, { provider = 'hal.artifact_store', source_id = 'platform' }) +end + +local function start_driver_op(id, opts) + return fibers.run_scope_op(function () + local driver = driver_mod.new(id, opts, child_logger(id)) + + local ok_caps, caps_or_err = fibers.perform(driver:capabilities_op(S.cap_emit_ch)) + if not ok_caps then + return false, tostring(caps_or_err) + end + local caps = caps_or_err + + local started = false + local handed_off = false + + local function cleanup_started_driver() + if started and not handed_off then + resource.terminate_checked(driver, 'manager cleanup', 'HAL manager driver rollback failed') + end + end + + fibers.current_scope():finally(function () + cleanup_started_driver() + end) + + local ok_start, start_err = + fibers.perform(driver:start_op(assert(S.scope, 'artifact_store manager scope missing'))) + if not ok_start then + return false, tostring(start_err) + end + started = true + + local ok_emit, emit_err = fibers.perform(emit_device_added_op(driver, caps)) + if not ok_emit then + return false, tostring(emit_err) + end + + S.drivers[id] = driver + handed_off = true + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +local function stop_driver_op(id, driver) + return fibers.run_scope_op(function () + local ok_emit, emit_err = fibers.perform(emit_device_removed_op(driver)) + if not ok_emit then + return false, tostring(emit_err) + end + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + if not ok_stop then + return false, tostring(stop_err) + end + + S.drivers[id] = nil + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +local function reconcile_op(cfg) + local desired = normalise_config(cfg) + return driver_reconcile.reconcile_op { + current = S.drivers, + desired = desired, + same = driver_matches_spec, + stop = stop_driver_op, + start = function (id, spec) return start_driver_op(id, spec.opts) end, + } +end + +local function reply_config_op(req, reply) + if type(req) ~= 'table' + or type(req.reply_ch) ~= 'table' + or type(req.reply_ch.put_op) ~= 'function' + then + return op.always(false, 'config reply channel missing') + end + + return req.reply_ch:put_op(reply):wrap(function () + return true, nil + end):or_else(function () + return false, 'reply_not_ready' + end) +end + +local function shell_loop(generation, cfg_ch) + assert(S.scope, 'artifact_store shell without scope') + + while true do + local req = fibers.perform(cfg_ch:get_op()) + if not req then + return + end + + if req.generation ~= generation or S.generation ~= generation then + fibers.perform(reply_config_op(req, { + ok = false, + err = 'stale manager generation', + })) + else + local ok_reconcile, reconcile_err = fibers.perform(reconcile_op(req.config)) + local replied, reply_err = fibers.perform(reply_config_op(req, { + ok = ok_reconcile, + err = reconcile_err, + })) + + if replied ~= true and S.logger and S.logger.warn then + S.logger:warn({ + what = 'config_reply_failed', + err = tostring(reply_err), + }) + end + end + end +end + +function M.start_op(logger, dev_ev_ch, cap_emit_ch) + local owner_scope = fibers.current_scope() + assert(owner_scope ~= nil, 'artifact_store.start_op must be called from inside a fiber') + + return op.guard(function () + if S.started then + return op.always(false, 'already started') + end + + local scope, err = owner_scope:child() + if not scope then + return op.always(false, tostring(err)) + end + + S.scope = scope + S.logger = logger + S.dev_ev_ch = dev_ev_ch + S.cap_emit_ch = cap_emit_ch + + S.cfg_ch = channel.new(8) + S.generation = S.generation + 1 + local generation = S.generation + local cfg_ch = S.cfg_ch + + local detach_finaliser = scope:finally(function () + finalise_manager_scope(scope, generation) + end) + + local ok, serr = scope:spawn(function () shell_loop(generation, cfg_ch) end) + if not ok then + detach_finaliser() + finalise_manager_scope(scope, generation) + scope:cancel(tostring(serr or 'manager shell spawn failed')) + return op.always(false, tostring(serr)) + end + + S.started = true + return op.always(true, nil) + end) +end + +function M.apply_config_op(cfg) + return fibers.run_scope_op(function () + local ok, err = validate_config(cfg) + if not ok then + return false, err + end + if not S.started then + return false, 'artifact_store manager not started' + end + + local cfg_ch = S.cfg_ch + local generation = S.generation + if not cfg_ch then + return false, 'artifact_store manager not started' + end + + local reply_ch = channel.new(1) + local admitted, admit_err = fibers.perform(cfg_ch:put_op({ + generation = generation, + config = cfg, + reply_ch = reply_ch, + }):wrap(function () + return true, nil + end):or_else(function () + return false, 'artifact_store_manager_config_busy' + end)) + + if admitted ~= true then + return false, tostring(admit_err or 'artifact_store_manager_config_busy') + end + + local reply, recv_err = fibers.perform(reply_ch:get_op()) + if not reply then + return false, tostring(recv_err or 'config reply missing') + end + + return reply.ok, reply.err + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +function M.shutdown_op(timeout) + timeout = timeout or STOP_TIMEOUT + + return op.guard(function () + if not S.started or not S.scope then + return op.always(true, nil) + end + + local scope = S.scope + local generation = S.generation + scope:cancel('artifact_store manager stopped') + + return fibers.boolean_choice( + scope:join_op():wrap(function () + finalise_manager_scope(scope, generation) + return true, nil + end), + sleep.sleep_op(timeout):wrap(function () + return false, 'artifact_store manager stop timeout' + end) + ):wrap(function (completed, _a, b) + if completed then + return true, nil + end + return false, b + end) + end) +end + +function M.terminate(reason) + reason = tostring(reason or 'artifact_store manager terminated') + + local scope = S.scope + local generation = S.generation + if scope then + scope:cancel(reason) + end + + finalise_manager_scope(scope, generation) + return true, nil +end + +function M.fault_op() + return strict_manager.fault_op_for_state(S) +end + +return M diff --git a/src/services/hal/managers/control_store.lua b/src/services/hal/managers/control_store.lua new file mode 100644 index 00000000..abad7f05 --- /dev/null +++ b/src/services/hal/managers/control_store.lua @@ -0,0 +1,348 @@ +---@module 'services.hal.managers.control_store' + +local fibers = require 'fibers' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local channel = require 'fibers.channel' + +local device_events = require 'services.hal.support.device_events' +local driver_reconcile = require 'services.hal.support.driver_reconcile' +local strict_manager = require 'services.hal.support.strict_manager' +local resource = require 'devicecode.support.resource' +local driver_mod = require 'services.hal.drivers.control_store' + +local M = { + api_mode = 'op_only', +} + + +local STOP_TIMEOUT = 5.0 + +---@class ControlStoreManagerState +---@field started boolean +---@field scope Scope|nil +---@field logger table|nil +---@field dev_ev_ch Channel|nil +---@field cap_emit_ch Channel|nil +---@field cfg_ch Channel|nil +---@field generation integer +---@field drivers table +local S = { + started = false, + scope = nil, + logger = nil, + dev_ev_ch = nil, + cap_emit_ch = nil, + cfg_ch = nil, + generation = 0, + drivers = {}, +} + +local function finalise_manager_scope(scope, generation) + if S.scope ~= scope or S.generation ~= generation then + return + end + + for _, driver in pairs(S.drivers or {}) do + resource.terminate_checked(driver, 'manager finalised', 'HAL manager driver cleanup failed') + end + + S.started = false + S.scope = nil + S.logger = nil + S.dev_ev_ch = nil + S.cap_emit_ch = nil + S.cfg_ch = nil + S.drivers = {} +end + +local function validate_config(namespaces) + if type(namespaces) ~= 'table' then + return false, 'config must be a list' + end + for _, ns in ipairs(namespaces) do + if type(ns) ~= 'table' then + return false, 'each namespace must be a table' + end + if type(ns.name) ~= 'string' or ns.name == '' then + return false, 'namespace.name must be a non-empty string' + end + if type(ns.root) ~= 'string' or ns.root == '' then + return false, 'namespace.root must be a non-empty string' + end + end + return true, nil +end + +local function emit_device_added_op(driver, caps) + return device_events.added_op(S.dev_ev_ch, 'control-store', driver.id, { root = driver.root, source = 'control-store-manager' }, caps) +end + +local function emit_device_removed_op(driver) + return device_events.removed_op(S.dev_ev_ch, 'control-store', driver.id, {}) +end + +local function start_driver_op(name, root) + return fibers.run_scope_op(function () + local driver = driver_mod.new(name, root, S.logger) + + local ok_caps, caps_or_err = fibers.perform(driver:capabilities_op(S.cap_emit_ch)) + if not ok_caps then + return false, tostring(caps_or_err) + end + local caps = caps_or_err + + local started = false + local handed_off = false + + local function cleanup_started_driver() + if started and not handed_off then + -- Best-effort rollback; this resource belongs to this start attempt + resource.terminate_checked(driver, 'manager cleanup', 'HAL manager driver rollback failed') + end + end + + -- If anything below errors, rollback the started driver. + fibers.current_scope():finally(function () + cleanup_started_driver() + end) + + local ok_start, start_err = + fibers.perform(driver:start_op(assert(S.scope, 'control_store manager scope missing'))) + if not ok_start then + return false, tostring(start_err) + end + started = true + + local ok_emit, emit_err = fibers.perform(emit_device_added_op(driver, caps)) + if not ok_emit then + return false, tostring(emit_err) + end + + S.drivers[name] = driver + handed_off = true + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +local function stop_driver_op(name, driver) + return fibers.run_scope_op(function () + local ok_emit, emit_err = fibers.perform(emit_device_removed_op(driver)) + if not ok_emit then + return false, tostring(emit_err) + end + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + if not ok_stop then + return false, tostring(stop_err) + end + + S.drivers[name] = nil + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +local function reconcile_op(namespaces) + local desired = {} + for _, ns in ipairs(namespaces or {}) do desired[ns.name] = ns.root end + return driver_reconcile.reconcile_op { + current = S.drivers, + desired = desired, + same = function (driver, root) return driver.root == root end, + stop = stop_driver_op, + start = start_driver_op, + } +end + +local function reply_config_op(req, reply) + if type(req) ~= 'table' + or type(req.reply_ch) ~= 'table' + or type(req.reply_ch.put_op) ~= 'function' + then + return op.always(false, 'config reply channel missing') + end + + return req.reply_ch:put_op(reply):wrap(function () + return true, nil + end):or_else(function () + return false, 'reply_not_ready' + end) +end + +local function shell_loop(generation, cfg_ch) + assert(S.scope, 'control_store shell without scope') + + while true do + local req = fibers.perform(cfg_ch:get_op()) + if not req then + return + end + + if req.generation ~= generation or S.generation ~= generation then + fibers.perform(reply_config_op(req, { + ok = false, + err = 'stale manager generation', + })) + else + local ok_reconcile, reconcile_err = fibers.perform(reconcile_op(req.config)) + local replied, reply_err = fibers.perform(reply_config_op(req, { + ok = ok_reconcile, + err = reconcile_err, + })) + + if replied ~= true and S.logger and S.logger.warn then + S.logger:warn({ + what = 'config_reply_failed', + err = tostring(reply_err), + }) + end + end + end +end + +function M.start_op(logger, dev_ev_ch, cap_emit_ch) + -- Capture the owning long-lived HAL scope now, not at perform time. + -- This op may be passed around before being performed; the driver shell must + -- still be parented to the manager/service scope that initiated startup. + local owner_scope = fibers.current_scope() + assert(owner_scope ~= nil, 'control_store.start_op must be called from inside a fiber') + + return op.guard(function () + if S.started then + return op.always(false, 'already started') + end + + local scope, err = owner_scope:child() + if not scope then + return op.always(false, tostring(err)) + end + + S.scope = scope + S.logger = logger + S.dev_ev_ch = dev_ev_ch + S.cap_emit_ch = cap_emit_ch + + S.cfg_ch = channel.new(8) + S.generation = S.generation + 1 + local generation = S.generation + local cfg_ch = S.cfg_ch + + local detach_finaliser = scope:finally(function () + finalise_manager_scope(scope, generation) + end) + + local ok, serr = scope:spawn(function () shell_loop(generation, cfg_ch) end) + if not ok then + detach_finaliser() + finalise_manager_scope(scope, generation) + scope:cancel(tostring(serr or 'manager shell spawn failed')) + return op.always(false, tostring(serr)) + end + + S.started = true + return op.always(true, nil) + end) +end + +function M.apply_config_op(namespaces) + return fibers.run_scope_op(function () + local ok, err = validate_config(namespaces) + if not ok then + return false, err + end + if not S.started then + return false, 'control_store manager not started' + end + + local cfg_ch = S.cfg_ch + local generation = S.generation + if not cfg_ch then + return false, 'control_store manager not started' + end + + local reply_ch = channel.new(1) + local admitted, admit_err = fibers.perform(cfg_ch:put_op({ + generation = generation, + config = namespaces, + reply_ch = reply_ch, + }):wrap(function () + return true, nil + end):or_else(function () + return false, 'control_store_manager_config_busy' + end)) + + if admitted ~= true then + return false, tostring(admit_err or 'control_store_manager_config_busy') + end + + local reply, recv_err = fibers.perform(reply_ch:get_op()) + if not reply then + return false, tostring(recv_err or 'config reply missing') + end + + return reply.ok, reply.err + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +function M.shutdown_op(timeout) + timeout = timeout or STOP_TIMEOUT + + return op.guard(function () + if not S.started or not S.scope then + return op.always(true, nil) + end + + local scope = S.scope + local generation = S.generation + scope:cancel() + + return fibers.boolean_choice( + scope:join_op():wrap(function () + finalise_manager_scope(scope, generation) + return true, nil + end), + sleep.sleep_op(timeout):wrap(function () + return false, 'control_store manager stop timeout' + end) + ):wrap(function (completed, a, b) + if completed then + return true, nil + end + return false, b + end) + end) +end + +function M.terminate(reason) + reason = tostring(reason or 'control_store manager terminated') + + local scope = S.scope + local generation = S.generation + if scope then + scope:cancel(reason) + end + + finalise_manager_scope(scope, generation) + return true, nil +end + +function M.fault_op() + return strict_manager.fault_op_for_state(S) +end + +return M diff --git a/src/services/hal/managers/filesystem.lua b/src/services/hal/managers/filesystem.lua new file mode 100644 index 00000000..ddb5d010 --- /dev/null +++ b/src/services/hal/managers/filesystem.lua @@ -0,0 +1,284 @@ +-- HAL modules +local fs_driver = require "services.hal.drivers.filesystem" +local hal_types = require "services.hal.types.core" + +-- Fibers modules +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" + +-- Constants + +local STOP_TIMEOUT = 5.0 -- seconds + +---@alias Namespace { name: string, root: string } + + +---@class FilesystemManager +---@field scope Scope +---@field started boolean +---@field drivers table +---@field dev_ev_ch Channel? +---@field cap_emit_ch Channel? +---@field logger table? +---@field last_names string[] +local FilesystemManager = { + started = false, + drivers = {}, + dev_ev_ch = nil, + cap_emit_ch = nil, + logger = nil, + last_names = {}, +} + +---@param namespaces string[] +local function emit_removed_device(namespaces) + local removed_event, removed_err = hal_types.new.DeviceEvent( + "removed", + "fs", + "main", + { namespaces = namespaces } + ) + if not removed_event then + FilesystemManager.logger:error({ what = 'removed_device_event_create_failed', err = tostring(removed_err) }) + return + end + FilesystemManager.dev_ev_ch:put(removed_event) +end + +---@param driver FSDriver +local function stop_driver(driver) + FilesystemManager.logger:debug({ what = 'stopping_previous_driver' }) + local stop_ok, stop_err = driver:stop(STOP_TIMEOUT) + if not stop_ok then + FilesystemManager.logger:warn({ what = 'stop_previous_driver_failed', err = tostring(stop_err) }) + end +end + +local function stop_previous_driver_and_emit() + local prev_driver = FilesystemManager.drivers["main"] + if not prev_driver then + return + end + + stop_driver(prev_driver) + + emit_removed_device(FilesystemManager.last_names) + FilesystemManager.drivers["main"] = nil +end + +---@param namespaces Namespace[] +---@return table roots +---@return string[] names +---@return string error +local function build_roots(namespaces) + local roots = {} + local names = {} + for _, ns in ipairs(namespaces) do + if type(ns.name) ~= 'string' or type(ns.root) ~= 'string' then + return {}, {}, "invalid namespace config" + end + roots[ns.name] = ns.root + names[#names + 1] = ns.name + end + return roots, names, "" +end + +---@param roots table +---@return FSDriver? driver +---@return string error +local function init_driver(roots) + local driver_logger = nil + if FilesystemManager.logger and FilesystemManager.logger.child then + driver_logger = FilesystemManager.logger:child({ component = 'driver', driver = 'filesystem', id = 'main' }) + end + + ---@type any + local fs_driver_any = fs_driver + local driver, drv_err = fs_driver_any.new(roots, driver_logger) + if not driver then + return nil, "failed to create filesystem driver: " .. tostring(drv_err) + end + + local init_err = driver:init() + if init_err ~= "" then + return nil, "failed to init filesystem driver: " .. tostring(init_err) + end + + return driver, "" +end + +---@param driver FSDriver +---@return Capability[]? capabilities +---@return string error +local function apply_driver_capabilities(driver) + local capabilities, cap_err = driver:capabilities(FilesystemManager.cap_emit_ch) + if cap_err ~= "" then + return nil, "failed to apply capabilities: " .. tostring(cap_err) + end + + return capabilities, "" +end + +---@param driver FSDriver +---@return string error +local function start_driver(driver) + local ok, start_err = driver:start() + if not ok then + return "failed to start driver: " .. tostring(start_err) + end + + return "" +end + +---@param namespaces string[] +---@param capabilities Capability[] +local function emit_added_device(namespaces, capabilities) + local device_event, ev_err = hal_types.new.DeviceEvent( + "added", + "fs", + "main", + { namespaces = namespaces }, + capabilities + ) + if not device_event then + FilesystemManager.logger:error({ what = 'added_device_event_create_failed', err = tostring(ev_err) }) + return + end + + FilesystemManager.dev_ev_ch:put(device_event) +end + + +---Starts the Filesystem Manager. +---@param logger table? +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +---@return string error +function FilesystemManager.start(logger, dev_ev_ch, cap_emit_ch) + if FilesystemManager.started then + return "Already started" + end + + local scope, err = fibers.current_scope():child() + if not scope then + return "Failed to create child scope: " .. tostring(err) + end + FilesystemManager.scope = scope + FilesystemManager.dev_ev_ch = dev_ev_ch + FilesystemManager.cap_emit_ch = cap_emit_ch + FilesystemManager.logger = logger + + -- Print out manager stack trace if scope closes on a failure + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + FilesystemManager.logger:error({ what = 'scope_failed', err = tostring(primary), status = st }) + end + FilesystemManager.logger:debug({ what = 'stopped' }) + end) + + FilesystemManager.started = true + FilesystemManager.logger:debug({ what = 'started' }) + return "" +end + +---Stops the Filesystem Manager. +---@param timeout number? Timeout in seconds +---@return boolean ok +---@return string error +function FilesystemManager.stop(timeout) + if not FilesystemManager.started then + return false, "Not started" + end + timeout = timeout or STOP_TIMEOUT + FilesystemManager.scope:cancel() + + local source = fibers.perform(op.named_choice { + join = FilesystemManager.scope:join_op(), + timeout = sleep.sleep_op(timeout) + }) + + if source == "timeout" then + return false, "filesystem manager stop timeout" + end + FilesystemManager.started = false + return true, "" +end + +--- Check that config is a set of name-root pairs and that paths are valid +---@param namespaces Namespace[] +local function validate_config(namespaces) + if type(namespaces) ~= 'table' then + return false, "config must be a table of namespaces" + end + for _, ns in ipairs(namespaces) do + if type(ns.name) ~= 'string' or type(ns.root) ~= 'string' then + return false, "each namespace must have a name and root string" + end + end + return true, "" +end + +---Apply filesystem configuration by creating a driver with the given namespaces. +---This function is non-blocking and spawns a fiber to initialize the driver. +---Stored channels from start() are used. Must be called after start(). +---@param namespaces Namespace[] List of {name, root} namespace configs +---@return boolean ok +---@return string error +function FilesystemManager.apply_config(namespaces) + local valid, validate_err = validate_config(namespaces) + if not valid then + return false, validate_err + end + + if not FilesystemManager.started then + return false, "filesystem manager not started" + end + + if FilesystemManager.dev_ev_ch == nil or FilesystemManager.cap_emit_ch == nil then + return false, "channels not initialized (start must be called first)" + end + + -- Spawn non-blocking fiber to create and initialize driver + local ok, spawn_err = FilesystemManager.scope:spawn(function() + stop_previous_driver_and_emit() + + local roots, names, roots_err = build_roots(namespaces) + if roots_err ~= "" then + FilesystemManager.logger:error({ what = 'build_roots_failed', err = tostring(roots_err) }) + return + end + + local driver, init_err = init_driver(roots) + if not driver then + FilesystemManager.logger:error({ what = 'driver_init_failed', err = tostring(init_err) }) + return + end + + local capabilities, cap_err = apply_driver_capabilities(driver) + if not capabilities then + FilesystemManager.logger:error({ what = 'apply_capabilities_failed', err = tostring(cap_err) }) + return + end + + local start_err = start_driver(driver) + if start_err ~= "" then + FilesystemManager.logger:error({ what = 'driver_start_failed', err = tostring(start_err) }) + return + end + + FilesystemManager.drivers["main"] = driver + FilesystemManager.last_names = names + emit_added_device(names, capabilities) + FilesystemManager.logger:debug({ what = 'applied_config_created_driver', namespaces = names }) + end) + + if not ok then + return false, "failed to spawn driver initialization: " .. tostring(spawn_err) + end + + return true, "" +end + +return FilesystemManager diff --git a/src/services/hal/managers/modemcard.lua b/src/services/hal/managers/modemcard.lua index bba3e6c9..c9d28a64 100644 --- a/src/services/hal/managers/modemcard.lua +++ b/src/services/hal/managers/modemcard.lua @@ -1,183 +1,364 @@ -local fiber = require "fibers.fiber" +-- HAL modules +local modem_provider = require "services.hal.backends.modem.provider" +local hal_types = require "services.hal.types.core" +local capability_args = require "services.hal.types.capability_args" +local modem_driver = require "services.hal.drivers.modem" + +-- Fiber modules +local fibers = require "fibers" +local op = require "fibers.op" local channel = require "fibers.channel" local sleep = require "fibers.sleep" -local context = require "fibers.context" -local op = require "fibers.op" -local sc = require "fibers.utils.syscall" -local utils = require "services.hal.utils" -local modem_driver = require "services.hal.drivers.modem" -local mmcli = require "services.hal.drivers.modem.mmcli" -local service = require "service" -local log = require "services.log" - ----@class ModemManagement -local ModemManagement = { - modem_remove_channel = channel.new(), - modem_detect_channel = channel.new() +local cond = require "fibers.cond" + +-- Other modules +local log -- set in start() + + +-- Constants + +local STOP_TIMEOUT = 5.0 -- seconds + +---@class ModemDriver + +---@class ModemcardManager +---@field scope Scope +---@field started boolean +---@field modem_remove_ch Channel +---@field modem_detect_ch Channel +---@field driver_ch Channel +---@field modems table +---@field monitor ModemMonitor? +local ModemcardManager = { + started = false, + modem_remove_ch = channel.new(), + modem_detect_ch = channel.new(), + driver_ch = channel.new(), + modems = {}, + monitor = nil, } -ModemManagement.__index = ModemManagement ----Create a new Modem Manager ----@return ModemManagement -local function new() - local modem_management = {} - return setmetatable(modem_management, ModemManagement) -end +---Continuously monitors modem add/remove events and publishes them onto +---`ModemcardManager.modem_detect_ch` and `ModemcardManager.modem_remove_ch`. +---@param scope Scope +local function detector(scope) + log:debug("Modem Detector: started") -local modems = {} + local monitor -function ModemManagement:apply_config(config) - -- Currently no config options -end + scope:finally(function () + if monitor and monitor.terminate then + monitor:terminate("modem_detector_scope_exit") + end + if ModemcardManager.monitor == monitor then + ModemcardManager.monitor = nil + end + log:debug("Modem Detector: closed") + end) ---- Detect modems via mmcli modem monitor ----@param ctx Context -function ModemManagement:_detector(ctx) - log.trace("Modem Detector: starting...") - - while not ctx:err() do - -- First, we start the modem detector - local cmd = mmcli.monitor_modems() - cmd:setprdeathsig(sc.SIGKILL) - local stdout = assert(cmd:stdout_pipe()) - local err = cmd:start() - - if err then - log.error("Failed to start modem detection:", err) - sleep.sleep(5) - else - -- Now we loop over every line of output - -- for line in stdout:lines() do - while not ctx:err() do - local line, ctx_err = op.choice( - stdout:read_line_op(), - ctx:done_op():wrap(function() - return nil, ctx:err() - end) - ):perform() - if ctx_err or line == nil then break end - local is_added, address, parse_err = utils.parse_monitor(line) - - if is_added==true then - log.trace("Modem Detector: detected at:", address) - self.modem_detect_channel:put(address) - elseif is_added==false then - log.trace("Modem Detector: removed at:", address) - self.modem_remove_channel:put(address) - end - if parse_err then - log.debug(string.format("%s - %s: %s", - ctx:value('service_name'), - ctx:value('fiber_name'), - parse_err)) - end + local err + monitor, err = modem_provider.new_monitor() + if not monitor then + error("Modem Detector: failed to create monitor: " .. tostring(err)) + end + ModemcardManager.monitor = monitor + + while true do + local event, mon_err = fibers.perform(monitor:next_event_op()) + if mon_err == "Command closed" then + break + elseif mon_err and mon_err ~= "" then + log:warn({ what = 'unparse_monitor_line', err = tostring(mon_err) }) + elseif event then + ---@cast event ModemMonitorEvent + if event.is_added then + log:debug({ what = 'modem_detected', summary = string.format('modem detected %s', tostring(event.address)), address = event.address }) + ModemcardManager.modem_detect_ch:put(event.address) + else + log:debug({ what = 'modem_removed', summary = string.format('modem removed %s', tostring(event.address)), address = event.address }) + ModemcardManager.modem_remove_ch:put(event.address) end - cmd:kill() - cmd:wait() end - stdout:close() end - log.trace(string.format("HAL: Modemcard Manager - Detector Closing, reason '%s'", ctx:err())) end ---- Creates and deletes modem drivers ----@param ctx Context ----@param bus_conn Connection ----@param device_event_q Queue -function ModemManagement:_manager( - ctx, - bus_conn, - device_event_q, - capability_info_q) - log.trace("Modem Manager: starting") - - local driver_channel = channel.new() - - local function handle_removal(address) - log.trace(string.format("HAL - Modem Manager: removing modem at %s", address)) - local instance = modems[address] - if not instance then return end - instance.driver.ctx:cancel('removed') - - modems[address] = nil - - local device = instance.device - - local device_event = { - connected = false, - type = 'usb', - id_field = "port", - data = { - device = 'modemcard', - port = device - } - } - - op.choice( - device_event_q:put_op(device_event), - ctx:done_op() - ):perform() - end - - local function handle_detection(address) - log.trace(string.format("HAL - Modem Manager: detected modem at %s", address)) - local driver = modem_driver.new(context.with_cancel(ctx), address) - fiber.spawn(function () - local err = driver:init() - if err then - log.error("HAL - Modem Manager Handle Detection: modem initialisation failed, removing modem") - handle_removal(address) - else - driver_channel:put(driver) - end - end) +---Handle modem removal. +---@param dev_ev_ch Channel Device event channel (DeviceEvent messages) +---@param address ModemAddress +local function on_remove(dev_ev_ch, address) + if type(address) ~= 'string' or address == '' then + log:error({ what = 'invalid_address_removal' }) + return end - local function handle_driver(driver) - if driver.ctx:err() then return end + log:debug({ what = 'removing_modem', address = address }) - modems[driver.address] = { driver = driver, device = driver.device } + local driver = ModemcardManager.modems[address] + if driver == nil then + log:error({ what = 'modem_not_found', address = address }) + return + end + + -- Get device, no need to have a fresh value so set cache lifetime to infinity + -- Also asking for a fresh value when the modem may have disconnected could cause errors + local get_ok, primary = driver:get(capability_args.new.ModemGetOpts("device", math.huge)) + if not get_ok then + log:error({ what = 'get_device_failed', address = address, err = tostring(primary) }) + return + end + local device = primary + + fibers.current_scope():spawn(function() + local ok, stop_err = driver:stop(STOP_TIMEOUT) + if not ok then + log:error({ what = 'stop_driver_failed', address = address, err = tostring(stop_err) }) + end + end) + + ModemcardManager.modems[address] = nil + + local device_event, ev_err = hal_types.new.DeviceEvent( + "removed", + "modemcard", + device + ) + if not device_event then + log:error({ what = 'create_device_event_failed', address = address, err = tostring(ev_err) }) + return + end + + dev_ev_ch:put(device_event) +end - local capabilities, cap_err = driver:apply_capabilities(capability_info_q) - if cap_err then - log.error(cap_err) +---Handle modem detection by creating and initializing a driver. +---@param address ModemAddress +---@return nil +local function on_detection(address) + if type(address) ~= 'string' or address == '' then + log:error({ what = 'invalid_address_detection' }) + return + end + + log:debug({ what = 'creating_modem', summary = string.format('creating modem %s', tostring(address)), address = address }) + + local driver, drv_err = modem_driver.new(address, log:child({ modem = address })) + if not driver then + log:error({ what = 'create_driver_failed', address = address, err = tostring(drv_err) }) + return + end + + fibers.current_scope():spawn(function() + local init_err = driver:init() + if init_err ~= "" then + log:error({ what = 'init_driver_failed', address = address, err = init_err }) return end - local device_event = { - connected = true, - type = 'usb', - capabilities = capabilities, - device_control = {}, - id_field = "port", - data = { - device = 'modemcard', - port = driver.device - } - } - - driver:spawn(bus_conn) - - device_event_q:put(device_event) - end - - while not ctx:err() do - op.choice( - self.modem_detect_channel:get_op():wrap(handle_detection), - self.modem_remove_channel:get_op():wrap(handle_removal), - driver_channel:get_op():wrap(handle_driver), - ctx:done_op() - ):perform() - end - log.trace(string.format("HAL: Modemcard Manager - Manager Closing, reason '%s'", ctx:err())) + ModemcardManager.driver_ch:put(driver) + end) +end + +---Handle a fully initialized driver by creating the modem device, applying +---capabilities, and emitting a HAL device event. +---@param dev_ev_ch Channel Device event channel (DeviceEvent messages) +---@param cap_emit_ch Channel Capability emit channel (Emit messages) +---@param driver Modem +---@return nil +local function on_driver(dev_ev_ch, cap_emit_ch, driver) + local address = driver.address + -- Get device, no need to have a fresh value so set cache lifetime to infinity + local get_ok, primary = driver:get(capability_args.new.ModemGetOpts("device", math.huge)) + if not get_ok then + log:error({ what = 'get_device_failed', address = address, err = tostring(primary) }) + return + end + local device = primary + + ModemcardManager.modems[driver.address] = driver + + -- Build capabilities + local capabilities, cap_err = driver:capabilities(cap_emit_ch) + if cap_err then + log:error({ what = 'apply_capabilities_failed', address = address, err = tostring(cap_err) }) + return + end + + -- Register the capability before starting the driver. The modem driver + -- starts lifecycle workers that immediately emit retained state such as + -- card/sim changes. If those emissions occur before HAL has registered the + -- capability they are dropped by hal.lua:on_cap_emit(), leaving GSM waiting + -- for a retained state that may never exist. This mirrors the platform, + -- power and sysmon managers: publish the DeviceEvent, wait until HAL has + -- registered it, then start the emitting workers. + local ready_cond = cond.new() + + local device_event, ev_err = hal_types.new.DeviceEvent( + "added", + "modemcard", + device, + { + address = address, + port = device -- the device field holds the usb or pcie port info + }, + capabilities, + ready_cond + ) + if not device_event then + log:error({ what = 'create_device_event_failed', address = address, err = tostring(ev_err) }) + return + end + + -- Notify HAL of the new modem device and wait until the device and its + -- capabilities are registered before starting the driver. + dev_ev_ch:put(device_event) + ready_cond:wait() + + -- Start the driver only after capability registration, so initial retained + -- state emissions cannot race ahead of the registry. + local ok, start_err = driver:start() + if not ok then + log:error({ what = 'start_driver_failed', address = address, err = tostring(start_err) }) + ModemcardManager.modems[driver.address] = nil + local remove_event = hal_types.new.DeviceEvent("removed", "modemcard", device) + if remove_event then + dev_ev_ch:put(remove_event) + end + return + end end -function ModemManagement:spawn(ctx, bus_conn, device_event_q, capability_info_q) - service.spawn_fiber('Modem Detector', bus_conn, ctx, function(detector_ctx) - self:_detector(detector_ctx) +---Modemcard Manager notifies HAL of modem additions/removals. +---@param scope Scope +---@param dev_ev_ch Channel Device event channel (DeviceEvent messages) +---@param cap_emit_ch Channel Capability emit channel (Emit messages) +---@return nil +local function manager(scope, dev_ev_ch, cap_emit_ch) + log:debug("Modemcard Manager: started") + + scope:finally(function () + log:debug("Modemcard Manager: closed") end) - service.spawn_fiber('Modem Manager', bus_conn, ctx, function(manager_ctx) - self:_manager(manager_ctx, bus_conn, device_event_q, capability_info_q) + while true do + local fault_ops = {} + for address, driver in pairs(ModemcardManager.modems) do + table.insert(fault_ops, driver.scope:fault_op():wrap(function () return address end)) + end + + local fault_op = op.never() + if #fault_ops > 0 then + fault_op = op.choice(unpack(fault_ops)) + end + + local source, msg, err = fibers.perform(op.named_choice{ + detect = ModemcardManager.modem_detect_ch:get_op(), + remove = ModemcardManager.modem_remove_ch:get_op(), + driver = ModemcardManager.driver_ch:get_op(), + driver_fault = fault_op, + }) + + if not msg then + log:error({ what = 'operation_failed', err = tostring(err) }) + break + end + + if source == "detect" then + on_detection(msg) + elseif source == "remove" then + on_remove(dev_ev_ch, msg) + elseif source == "driver" then + on_driver(dev_ev_ch, cap_emit_ch, msg) + elseif source == "driver_fault" then + log:error({ what = 'driver_fault', address = tostring(msg) }) + on_remove(dev_ev_ch, msg) + else + log:error({ what = 'unknown_source', source = tostring(source) }) + end + end +end + +---Starts the Modemcard Manager's detector and manager fibers. +---@param logger Logger +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +---@return string error +function ModemcardManager.start(logger, dev_ev_ch, cap_emit_ch) + log = logger + if ModemcardManager.started then + return "Already started" + end + + local scope, err = fibers.current_scope():child() + if not scope then + return "Failed to create child scope: " .. tostring(err) + end + ModemcardManager.scope = scope + + -- Print out manager stack trace if scope closes on a failure + scope:finally(function () + local st, primary = scope:status() + if st == 'failed' then + log:error({ what = 'scope_error', err = tostring(primary) }) + log:debug({ what = 'scope_exit', status = st }) + end + log:debug("Modem Manager: stopped") end) + + ModemcardManager.scope:spawn(detector) + ModemcardManager.scope:spawn(manager, dev_ev_ch, cap_emit_ch) + + ModemcardManager.started = true + log:debug("Modemcard Manager: started") + return "" +end + +---Stops the Modemcard Manager. +---@param timeout number? Timeout in seconds +---@return boolean ok +---@return string error +function ModemcardManager.stop(timeout) + if not ModemcardManager.started then + return false, "Not started" + end + timeout = timeout or STOP_TIMEOUT + + for address, driver in pairs(ModemcardManager.modems) do + local ok, stop_err = driver:stop(STOP_TIMEOUT) + if not ok then + log:error({ what = "stop_driver_failed", address = address, err = tostring(stop_err) }) + end + end + ModemcardManager.modems = {} + + local monitor = ModemcardManager.monitor + if monitor and monitor.shutdown_op then + local ok, mon_err = fibers.perform(monitor:shutdown_op(1.0)) + if not ok then + log:error({ what = "stop_modem_monitor_failed", err = tostring(mon_err) }) + end + end + ModemcardManager.monitor = nil + + ModemcardManager.scope:cancel("modemcard manager stopping") + + local source = fibers.perform(op.named_choice { + join = ModemcardManager.scope:join_op(), + timeout = sleep.sleep_op(timeout) + }) + + if source == "timeout" then + return false, "modemcard manager stop timeout" + end + ModemcardManager.started = false + return true, "" end -return {new = new} + +---Apply configuration for modemcard manager (no-op, kept for interface consistency). +---@param namespaces table +---@return boolean ok +---@return string error +function ModemcardManager.apply_config(namespaces) -- luacheck: ignore + -- No-op: modemcard manager does not support dynamic configuration + return true, "" +end + +return ModemcardManager diff --git a/src/services/hal/managers/network.lua b/src/services/hal/managers/network.lua new file mode 100644 index 00000000..89a6b72e --- /dev/null +++ b/src/services/hal/managers/network.lua @@ -0,0 +1,221 @@ +-- services/hal/managers/network.lua +-- Strict op-only HAL manager for semantic network capabilities. + +local op = require 'fibers.op' +local channel = require 'fibers.channel' + +local strict = require 'services.hal.support.strict_manager' +local hal_types = require 'services.hal.types.core' +local cap_types = require 'services.hal.types.capabilities' +local control_loop = require 'services.hal.support.control_loop' +local driver_mod = require 'services.hal.drivers.network' + +local M = strict.api_table() + +local state = { + started = false, + scope = nil, + logger = nil, + dev_ev_ch = nil, + cap_emit_ch = nil, + driver = nil, + controls = {}, + caps = nil, + device_added = false, + controls_started = false, +} + +local function log(level, payload) + if state.logger and type(state.logger[level]) == 'function' then + state.logger[level](state.logger, payload) + end +end + +local function result_to_reply_tuple(result) + if type(result) == 'table' then + return result.ok == true, result + end + return result == true, result +end + +local function driver_method_op(method, req) + return op.guard(function () + if not state.driver then + return op.always(false, { ok = false, err = 'network driver not configured' }) + end + + local opname = tostring(method) .. '_op' + local fn = state.driver[opname] + if type(fn) ~= 'function' then + return op.always(false, { ok = false, err = 'network driver missing ' .. opname }) + end + + local ok, driver_op = pcall(function () return fn(state.driver, req and req.opts or {}) end) + if not ok then + return op.always(false, { ok = false, err = tostring(driver_op) }) + end + if type(driver_op) ~= 'table' then + return op.always(false, { ok = false, err = opname .. ' did not return an Op' }) + end + + return driver_op:wrap(result_to_reply_tuple) + end) +end + +local CONFIG_METHODS = { + __cancel_policy = { apply = 'detach_after_admission' }, + validate = function (_opts, req) return driver_method_op('validate', req) end, + plan = function (_opts, req) return driver_method_op('plan', req) end, + apply = function (_opts, req) return driver_method_op('apply', req) end, + apply_live_weights = function (_opts, req) return driver_method_op('apply_live_weights', req) end, + apply_shaping = function (_opts, req) return driver_method_op('apply_shaping', req) end, +} + +local STATE_METHODS = { + snapshot = function (_opts, req) return driver_method_op('snapshot', req) end, + watch = function (_opts, req) return driver_method_op('watch', req) end, +} + +local DIAGNOSTICS_METHODS = { + probe_link = function (_opts, req) return driver_method_op('probe_link', req) end, + read_counters = function (_opts, req) return driver_method_op('read_counters', req) end, + speedtest = function (_opts, req) return driver_method_op('speedtest', req) end, +} + +local function control_loop_for(kind, ch, methods) + if tostring(kind) == 'config' then + log('debug', { + what = 'network_config_control_instrumented_build', + marker = 'owned_activation_runner_v1', + }) + end + control_loop.run_request_loop(ch, methods, state.logger, 'network_' .. tostring(kind)) +end + +local function start_control_loops() + if state.controls_started == true then return true, nil end + if not state.scope then return nil, 'network manager scope not started' end + state.scope:spawn(function () control_loop_for('config', state.controls.config, CONFIG_METHODS) end) + state.scope:spawn(function () control_loop_for('state', state.controls.state, STATE_METHODS) end) + state.scope:spawn(function () control_loop_for('diagnostics', state.controls.diagnostics, DIAGNOSTICS_METHODS) end) + state.controls_started = true + return true, nil +end + +local function make_capabilities() + local cfg_ch = channel.new(16) + local state_ch = channel.new(16) + local diag_ch = channel.new(16) + state.controls.config = cfg_ch + state.controls.state = state_ch + state.controls.diagnostics = diag_ch + + local cfg_cap = assert(cap_types.new.NetworkConfigCapability('main', cfg_ch)) + local state_cap = assert(cap_types.new.NetworkStateCapability('main', state_ch)) + local diag_cap = assert(cap_types.new.NetworkDiagnosticsCapability('main', diag_ch)) + + return { cfg_cap, state_cap, diag_cap } +end + +local function emit_added(dev_ev_ch, caps) + local ev = assert(hal_types.new.DeviceEvent('added', 'network', 'main', { + source = 'host', + manager = 'network', + }, caps)) + return dev_ev_ch:put_op(ev) +end + +-- Network capabilities are published during start so consumers can discover +-- passive handles early. The request loops are deliberately not started here: +-- apply_config_op() must first create the provider driver, otherwise callers +-- can observe "network driver not configured" during HAL config generation. +function M.start_op(logger, dev_ev_ch, cap_emit_ch) + return op.guard(function () + if state.started then return op.always(true, nil) end + local parent = require('fibers').current_scope() + local child, cerr = parent:child() + if not child then return op.always(false, cerr or 'network manager scope create failed') end + + state.scope = child + state.logger = logger + state.dev_ev_ch = dev_ev_ch + state.cap_emit_ch = cap_emit_ch + state.controls = {} + state.caps = make_capabilities() + state.device_added = false + state.controls_started = false + + child:finally(function (_, status, primary) + M.terminate(primary or status or 'network manager closed') + end) + + state.started = true + log('debug', { what = 'network_manager_started' }) + return emit_added(dev_ev_ch, state.caps):wrap(function (ok, emit_err) + if ok == false then return false, emit_err or 'network device event failed' end + state.device_added = true + return true, nil + end) + end) +end + +-- Configuring the driver is the point at which the passive handles become +-- serviceable. Start the control loops only after state.driver is ready so +-- queued requests wait rather than receiving a premature driver-not-configured +-- failure. +function M.apply_config_op(config) + return op.guard(function () + if not state.started then return op.always(false, 'network manager not started') end + local driver, err = driver_mod.new(config or {}, { + cap_emit_ch = state.cap_emit_ch, + logger = state.logger, + owner_scope = state.scope, + }) + if not driver then return op.always(false, err or 'network driver create failed') end + if state.driver and type(state.driver.terminate) == 'function' then + state.driver:terminate('replaced') + end + state.driver = driver + log('debug', { what = 'network_driver_configured', provider = (config and (config.provider or config.backend)) or 'fake' }) + local loops_ok, loops_err = start_control_loops() + if loops_ok ~= true then return op.always(false, loops_err or 'network control loops failed to start') end + return op.always(true, nil) + end) +end + +function M.shutdown_op(_timeout_s) + return op.guard(function () + M.terminate('shutdown') + return op.always(true, nil) + end) +end + +function M.terminate(reason) + if state.driver and type(state.driver.terminate) == 'function' then + state.driver:terminate(reason or 'terminated') + end + state.driver = nil + state.caps = nil + state.device_added = false + state.controls_started = false + for _, ch in pairs(state.controls or {}) do + if ch and type(ch.close) == 'function' then ch:close(reason or 'terminated') end + end + state.controls = {} + if state.scope then + local scope = state.scope + state.scope = nil + scope:cancel(reason or 'terminated') + end + state.started = false + state.logger = nil + state.dev_ev_ch = nil + state.cap_emit_ch = nil + return true, nil +end + +function M.fault_op() + return strict.fault_op_for_state(state) +end + +return M diff --git a/src/services/hal/managers/platform.lua b/src/services/hal/managers/platform.lua new file mode 100644 index 00000000..5ee0f8ee --- /dev/null +++ b/src/services/hal/managers/platform.lua @@ -0,0 +1,147 @@ +-- services/hal/managers/platform.lua +-- +-- Platform HAL Manager. +-- Creates a single PlatformDriver, registers it with HAL, and holds its +-- scope until the manager is stopped. + +local platform_driver = require "services.hal.drivers.platform" +local hal_types = require "services.hal.types.core" + +---@type any +local platform_driver_any = platform_driver + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local cond = require "fibers.cond" + +local STOP_TIMEOUT = 5.0 + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +---@class PlatformManager +---@field scope Scope? +---@field started boolean +---@field logger Logger? +local PlatformManager = { + started = false, + scope = nil, + logger = nil, +} + +---- manager fiber ---- + +---@param scope Scope +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +local function manager(scope, dev_ev_ch, cap_emit_ch) + dlog(PlatformManager.logger, 'debug', { what = 'started' }) + + scope:finally(function() + dlog(PlatformManager.logger, 'debug', { what = 'closed' }) + end) + + local driver_logger = nil + if PlatformManager.logger and PlatformManager.logger.child then + driver_logger = PlatformManager.logger:child({ component = 'driver', driver = 'platform', id = '1' }) + end + + local driver, drv_err = platform_driver_any.new(driver_logger) + if not driver then + error("Platform Manager: failed to create platform driver: " .. tostring(drv_err)) + end + + local init_err = driver:init() + if init_err ~= "" then + error("Platform Manager: failed to init driver: " .. tostring(init_err)) + end + + local capabilities, cap_err = driver:capabilities(cap_emit_ch) + if cap_err ~= "" then + error("Platform Manager: failed to bind capabilities: " .. tostring(cap_err)) + end + + local ready_cond = cond.new() + local device_event, ev_err = hal_types.new.DeviceEvent( + "added", "platform", "1", {}, capabilities, ready_cond) + if not device_event then + error("Platform Manager: failed to create DeviceEvent: " .. tostring(ev_err)) + end + dev_ev_ch:put(device_event) + ready_cond:wait() + + local ok, start_err = driver:start() + if not ok then + error("Platform Manager: failed to start driver: " .. tostring(start_err)) + end + + dlog(PlatformManager.logger, 'debug', { what = 'device_registered' }) +end + +---- public interface ---- + +---@param logger Logger? +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +---@return string error +function PlatformManager.start(logger, dev_ev_ch, cap_emit_ch) + if PlatformManager.started then + return "Already started" + end + + local scope, err = fibers.current_scope():child() + if not scope then + return "Failed to create child scope: " .. tostring(err) + end + PlatformManager.scope = scope + PlatformManager.logger = logger + + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + dlog(PlatformManager.logger, 'error', { what = 'scope_failed', err = tostring(primary), status = st }) + end + dlog(PlatformManager.logger, 'debug', { what = 'stopped' }) + end) + + scope:spawn(manager, dev_ev_ch, cap_emit_ch) + + PlatformManager.started = true + dlog(PlatformManager.logger, 'debug', { what = 'start_called' }) + return "" +end + +---@param timeout number? +---@return boolean ok +---@return string error +function PlatformManager.stop(timeout) + if not PlatformManager.started then + return false, "Not started" + end + timeout = timeout or STOP_TIMEOUT + PlatformManager.scope:cancel('platform manager stopped') + + local source = fibers.perform(op.named_choice { + join = PlatformManager.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + + if source == 'timeout' then + return false, "platform manager stop timeout" + end + PlatformManager.started = false + return true, "" +end + +---@param namespaces table +---@return boolean ok +---@return string error +function PlatformManager.apply_config(namespaces) -- luacheck: ignore + return true, "" +end + +return PlatformManager diff --git a/src/services/hal/managers/power.lua b/src/services/hal/managers/power.lua new file mode 100644 index 00000000..07605928 --- /dev/null +++ b/src/services/hal/managers/power.lua @@ -0,0 +1,147 @@ +-- services/hal/managers/power.lua +-- +-- Power HAL Manager. +-- Creates a single PowerDriver, registers it with HAL, and holds its +-- scope until the manager is stopped. + +local power_driver = require "services.hal.drivers.power" +local hal_types = require "services.hal.types.core" + +---@type any +local power_driver_any = power_driver + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local cond = require "fibers.cond" + +local STOP_TIMEOUT = 5.0 + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +---@class PowerManager +---@field scope Scope? +---@field started boolean +---@field logger Logger? +local PowerManager = { + started = false, + scope = nil, + logger = nil, +} + +---- manager fiber ---- + +---@param scope Scope +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +local function manager(scope, dev_ev_ch, cap_emit_ch) + dlog(PowerManager.logger, 'debug', { what = 'started' }) + + scope:finally(function() + dlog(PowerManager.logger, 'debug', { what = 'closed' }) + end) + + local driver_logger = nil + if PowerManager.logger and PowerManager.logger.child then + driver_logger = PowerManager.logger:child({ component = 'driver', driver = 'power', id = '1' }) + end + + local driver, drv_err = power_driver_any.new(driver_logger) + if not driver then + error("Power Manager: failed to create power driver: " .. tostring(drv_err)) + end + + local init_err = driver:init() + if init_err ~= "" then + error("Power Manager: failed to init driver: " .. tostring(init_err)) + end + + local capabilities, cap_err = driver:capabilities(cap_emit_ch) + if cap_err ~= "" then + error("Power Manager: failed to bind capabilities: " .. tostring(cap_err)) + end + + local ready_cond = cond.new() + local device_event, ev_err = hal_types.new.DeviceEvent( + "added", "power", "1", {}, capabilities, ready_cond) + if not device_event then + error("Power Manager: failed to create DeviceEvent: " .. tostring(ev_err)) + end + dev_ev_ch:put(device_event) + ready_cond:wait() + + local ok, start_err = driver:start() + if not ok then + error("Power Manager: failed to start driver: " .. tostring(start_err)) + end + + dlog(PowerManager.logger, 'debug', { what = 'device_registered' }) +end + +---- public interface ---- + +---@param logger Logger? +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +---@return string error +function PowerManager.start(logger, dev_ev_ch, cap_emit_ch) + if PowerManager.started then + return "Already started" + end + + local scope, err = fibers.current_scope():child() + if not scope then + return "Failed to create child scope: " .. tostring(err) + end + PowerManager.scope = scope + PowerManager.logger = logger + + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + dlog(PowerManager.logger, 'error', { what = 'scope_failed', err = tostring(primary), status = st }) + end + dlog(PowerManager.logger, 'debug', { what = 'stopped' }) + end) + + scope:spawn(manager, dev_ev_ch, cap_emit_ch) + + PowerManager.started = true + dlog(PowerManager.logger, 'debug', { what = 'start_called' }) + return "" +end + +---@param timeout number? +---@return boolean ok +---@return string error +function PowerManager.stop(timeout) + if not PowerManager.started then + return false, "Not started" + end + timeout = timeout or STOP_TIMEOUT + PowerManager.scope:cancel('power manager stopped') + + local source = fibers.perform(op.named_choice { + join = PowerManager.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + + if source == 'timeout' then + return false, "power manager stop timeout" + end + PowerManager.started = false + return true, "" +end + +---@param namespaces table +---@return boolean ok +---@return string error +function PowerManager.apply_config(namespaces) -- luacheck: ignore + return true, "" +end + +return PowerManager diff --git a/src/services/hal/managers/signature_verify.lua b/src/services/hal/managers/signature_verify.lua new file mode 100644 index 00000000..94b1a9d3 --- /dev/null +++ b/src/services/hal/managers/signature_verify.lua @@ -0,0 +1,413 @@ +---@module 'services.hal.managers.signature_verify' + +local fibers = require 'fibers' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local channel = require 'fibers.channel' + +local device_events = require 'services.hal.support.device_events' +local driver_reconcile = require 'services.hal.support.driver_reconcile' +local strict_manager = require 'services.hal.support.strict_manager' +local tablex = require 'shared.table' +local resource = require 'devicecode.support.resource' +local driver_mod = require 'services.hal.drivers.signature_verify_provider' + +local M = { + api_mode = 'op_only', +} + + +local STOP_TIMEOUT = 5.0 + +---@class SignatureVerifyManagerState +---@field started boolean +---@field scope Scope|nil +---@field logger table|nil +---@field dev_ev_ch Channel|nil +---@field cap_emit_ch Channel|nil +---@field cfg_ch Channel|nil +---@field generation integer +---@field drivers table +local S = { + started = false, + scope = nil, + logger = nil, + dev_ev_ch = nil, + cap_emit_ch = nil, + cfg_ch = nil, + generation = 0, + drivers = {}, +} + +local function finalise_manager_scope(scope, generation) + if S.scope ~= scope or S.generation ~= generation then + return + end + + for _, driver in pairs(S.drivers or {}) do + resource.terminate_checked(driver, 'manager finalised', 'HAL manager driver cleanup failed') + end + + S.started = false + S.scope = nil + S.logger = nil + S.dev_ev_ch = nil + S.cap_emit_ch = nil + S.cfg_ch = nil + S.drivers = {} +end + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +local function child_logger(id) + if S.logger and S.logger.child then + return S.logger:child({ + component = 'driver', + driver = 'signature_verify', + id = id, + }) + end + return S.logger +end + +local function validate_config(cfg) + cfg = cfg or {} + + if type(cfg) ~= 'table' then + return false, 'signature_verify config must be a table' + end + + local specs = cfg.providers + if specs == nil and cfg[1] == nil then + specs = { { id = 'main' } } + elseif specs == nil then + specs = cfg + end + + if type(specs) ~= 'table' then + return false, 'signature_verify.providers must be a table' + end + + local seen = {} + for i, rec in ipairs(specs) do + if type(rec) ~= 'table' then + return false, ('signature_verify.providers[%d] must be a table'):format(i) + end + + local id = rec.id or rec.name or 'main' + if type(id) ~= 'string' or id == '' then + return false, ('signature_verify.providers[%d].id must be a non-empty string'):format(i) + end + + if seen[id] then + return false, 'duplicate signature_verify provider id: ' .. id + end + seen[id] = true + end + + return true, nil +end + +local function normalise_config(cfg) + local specs = cfg.providers + if specs == nil and cfg[1] == nil then + specs = { { id = 'main' } } + elseif specs == nil then + specs = cfg + end + + local out = {} + for i = 1, #specs do + local rec = specs[i] + local id = rec.id or rec.name or 'main' + local opts = {} + for k, v in pairs(rec) do + if k ~= 'id' and k ~= 'name' then + opts[k] = v + end + end + out[id] = { id = id, opts = opts } + end + return out +end + +local deep_equal = tablex.deep_equal + +local function driver_matches_spec(driver, spec) + if not driver or not spec then + return false + end + + return deep_equal(driver.opts or {}, spec.opts or {}) +end + +local function emit_device_added_op(driver, caps) + return device_events.added_op(S.dev_ev_ch, 'signature_verify', driver.id, { provider = 'hal.signature_verify' }, caps) +end + +local function emit_device_removed_op(driver) + return device_events.removed_op(S.dev_ev_ch, 'signature_verify', driver.id, { provider = 'hal.signature_verify' }) +end + +local function start_driver_op(id, opts) + return fibers.run_scope_op(function () + local driver = driver_mod.new(id, opts, child_logger(id)) + + local ok_caps, caps_or_err = fibers.perform(driver:capabilities_op(S.cap_emit_ch)) + if not ok_caps then + return false, tostring(caps_or_err) + end + local caps = caps_or_err + + local started = false + local handed_off = false + + local function cleanup_started_driver() + if started and not handed_off then + resource.terminate_checked(driver, 'manager cleanup', 'HAL manager driver rollback failed') + end + end + + fibers.current_scope():finally(function () + cleanup_started_driver() + end) + + local ok_start, start_err = + fibers.perform(driver:start_op(assert(S.scope, 'signature_verify manager scope missing'))) + if not ok_start then + return false, tostring(start_err) + end + started = true + + local ok_emit, emit_err = fibers.perform(emit_device_added_op(driver, caps)) + if not ok_emit then + return false, tostring(emit_err) + end + + S.drivers[id] = driver + handed_off = true + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +local function stop_driver_op(id, driver) + return fibers.run_scope_op(function () + local ok_emit, emit_err = fibers.perform(emit_device_removed_op(driver)) + if not ok_emit then + return false, tostring(emit_err) + end + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + if not ok_stop then + return false, tostring(stop_err) + end + + S.drivers[id] = nil + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +local function reconcile_op(cfg) + local desired = normalise_config(cfg) + return driver_reconcile.reconcile_op { + current = S.drivers, + desired = desired, + same = driver_matches_spec, + stop = stop_driver_op, + start = function (id, spec) return start_driver_op(id, spec.opts) end, + } +end + +local function reply_config_op(req, reply) + if type(req) ~= 'table' + or type(req.reply_ch) ~= 'table' + or type(req.reply_ch.put_op) ~= 'function' + then + return op.always(false, 'config reply channel missing') + end + + return req.reply_ch:put_op(reply):wrap(function () + return true, nil + end):or_else(function () + return false, 'reply_not_ready' + end) +end + +local function shell_loop(generation, cfg_ch) + assert(S.scope, 'signature_verify shell without scope') + + while true do + local req = fibers.perform(cfg_ch:get_op()) + if not req then + return + end + + if req.generation ~= generation or S.generation ~= generation then + fibers.perform(reply_config_op(req, { + ok = false, + err = 'stale manager generation', + })) + else + local ok_reconcile, reconcile_err = fibers.perform(reconcile_op(req.config)) + local replied, reply_err = fibers.perform(reply_config_op(req, { + ok = ok_reconcile, + err = reconcile_err, + })) + + if replied ~= true and S.logger and S.logger.warn then + S.logger:warn({ + what = 'config_reply_failed', + err = tostring(reply_err), + }) + end + end + end +end + +function M.start_op(logger, dev_ev_ch, cap_emit_ch) + local owner_scope = fibers.current_scope() + assert(owner_scope ~= nil, 'signature_verify.start_op must be called from inside a fiber') + + return op.guard(function () + if S.started then + return op.always(false, 'already started') + end + + local scope, err = owner_scope:child() + if not scope then + return op.always(false, tostring(err)) + end + + S.scope = scope + S.logger = logger + S.dev_ev_ch = dev_ev_ch + S.cap_emit_ch = cap_emit_ch + + S.cfg_ch = channel.new(8) + S.generation = S.generation + 1 + local generation = S.generation + local cfg_ch = S.cfg_ch + + local detach_finaliser = scope:finally(function () + finalise_manager_scope(scope, generation) + end) + + local ok, serr = scope:spawn(function () shell_loop(generation, cfg_ch) end) + if not ok then + detach_finaliser() + finalise_manager_scope(scope, generation) + scope:cancel(tostring(serr or 'manager shell spawn failed')) + return op.always(false, tostring(serr)) + end + + S.started = true + return op.always(true, nil) + end) +end + +function M.apply_config_op(cfg) + return fibers.run_scope_op(function () + local ok, err = validate_config(cfg) + if not ok then + return false, err + end + if not S.started then + return false, 'signature_verify manager not started' + end + + local cfg_ch = S.cfg_ch + local generation = S.generation + if not cfg_ch then + return false, 'signature_verify manager not started' + end + + local reply_ch = channel.new(1) + local admitted, admit_err = fibers.perform(cfg_ch:put_op({ + generation = generation, + config = cfg, + reply_ch = reply_ch, + }):wrap(function () + return true, nil + end):or_else(function () + return false, 'signature_verify_manager_config_busy' + end)) + + if admitted ~= true then + return false, tostring(admit_err or 'signature_verify_manager_config_busy') + end + + local reply, recv_err = fibers.perform(reply_ch:get_op()) + if not reply then + return false, tostring(recv_err or 'config reply missing') + end + + return reply.ok, reply.err + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +function M.shutdown_op(timeout) + timeout = timeout or STOP_TIMEOUT + + return op.guard(function () + if not S.started or not S.scope then + return op.always(true, nil) + end + + local scope = S.scope + local generation = S.generation + scope:cancel('signature_verify manager stopped') + + return fibers.boolean_choice( + scope:join_op():wrap(function () + finalise_manager_scope(scope, generation) + return true, nil + end), + sleep.sleep_op(timeout):wrap(function () + return false, 'signature_verify manager stop timeout' + end) + ):wrap(function (completed, _a, b) + if completed then + return true, nil + end + return false, b + end) + end) +end + +function M.terminate(reason) + reason = tostring(reason or 'signature_verify manager terminated') + + local scope = S.scope + local generation = S.generation + if scope then + scope:cancel(reason) + end + + finalise_manager_scope(scope, generation) + return true, nil +end + +function M.fault_op() + return strict_manager.fault_op_for_state(S) +end + +return M diff --git a/src/services/hal/managers/sysmon.lua b/src/services/hal/managers/sysmon.lua new file mode 100644 index 00000000..3e9550c6 --- /dev/null +++ b/src/services/hal/managers/sysmon.lua @@ -0,0 +1,243 @@ +-- services/hal/managers/sysmon.lua +-- +-- System Monitor HAL Manager. +-- Discovers thermal zones at startup via /sys/class/thermal/ and creates: +-- • one CpuDriver → capability class 'cpu', id '1' +-- • one MemoryDriver → capability class 'memory', id '1' +-- • one ThermalDriver per thermal_zone* sysfs entry +-- +-- All devices are emitted once as HAL DeviceEvents, then the manager +-- sleeps until its scope is cancelled. + +local cpu_driver = require "services.hal.drivers.cpu" +local memory_driver = require "services.hal.drivers.memory" +local thermal_driver = require "services.hal.drivers.thermal" +local hal_types = require "services.hal.types.core" + +---@type any +local cpu_driver_any = cpu_driver +---@type any +local memory_driver_any = memory_driver +---@type any +local thermal_driver_any = thermal_driver + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local exec = require "fibers.io.exec" +local cond = require "fibers.cond" + +local STOP_TIMEOUT = 5.0 + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +---@class SysmonManager +---@field scope Scope? +---@field started boolean +---@field logger Logger? +local SysmonManager = { + started = false, + scope = nil, + logger = nil, +} + +---@param driver string +---@param id string +---@return Logger? +local function child_logger(driver, id) + if SysmonManager.logger and SysmonManager.logger.child then + return SysmonManager.logger:child({ component = 'driver', driver = driver, id = id }) + end + return nil +end + +---- helpers ---- + +--- List /sys/class/thermal/ and return table of {zone_id, sysfs_dir} entries. +---@return table zones list of {zone_id: string, sysfs_dir: string} +local function discover_thermal_zones() + local cmd = exec.command { 'ls', '/sys/class/thermal/', stdin = 'null', stdout = 'pipe', stderr = 'stdout' } + local out, status, code = fibers.perform(cmd:output_op()) + if status ~= 'exited' or code ~= 0 then + dlog(SysmonManager.logger, 'warn', { what = 'thermal_discovery_failed', code = tostring(code) }) + return {} + end + + local zones = {} + for entry in (out or ''):gmatch('[^\n]+') do + local n = entry:match('^thermal_zone(%d+)$') + if n then + zones[#zones + 1] = { + zone_id = 'zone' .. n, + sysfs_dir = '/sys/class/thermal/thermal_zone' .. n, + } + end + end + return zones +end + +--- Create, bind, start a driver and emit a HAL DeviceEvent. +---@param driver any +---@param class string +---@param id string +---@param meta table +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +---@return boolean ok +local function register_driver(driver, class, id, meta, dev_ev_ch, cap_emit_ch) + local init_err = driver:init() + if init_err ~= "" then + dlog(SysmonManager.logger, 'error', { what = 'driver_init_failed', class = class, id = id, err = init_err }) + return false + end + + local capabilities, cap_err = driver:capabilities(cap_emit_ch) + if cap_err ~= "" then + dlog(SysmonManager.logger, 'error', { + what = 'bind_capabilities_failed', class = class, id = id, err = cap_err, + }) + return false + end + + local ready_cond = cond.new() + local device_event, ev_err = hal_types.new.DeviceEvent("added", class, id, meta, capabilities, ready_cond) + if not device_event then + dlog(SysmonManager.logger, 'error', { + what = 'device_event_create_failed', class = class, id = id, err = ev_err, + }) + return false + end + dev_ev_ch:put(device_event) + ready_cond:wait() + + local ok, start_err = driver:start() + if not ok then + dlog(SysmonManager.logger, 'error', { what = 'driver_start_failed', class = class, id = id, err = start_err }) + return false + end + return true +end + +---- manager fiber ---- + +---@param scope Scope +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +local function manager(scope, dev_ev_ch, cap_emit_ch) + dlog(SysmonManager.logger, 'debug', { what = 'started' }) + + scope:finally(function() + dlog(SysmonManager.logger, 'debug', { what = 'closed' }) + end) + + -- ── CPU ── + local cpu_drv, cpu_err = cpu_driver_any.new(child_logger('cpu', '1')) + if not cpu_drv then + dlog(SysmonManager.logger, 'error', { + what = 'driver_create_failed', class = 'cpu', id = '1', err = cpu_err, + }) + else + register_driver(cpu_drv, 'cpu', '1', {}, dev_ev_ch, cap_emit_ch) + end + + -- ── Memory ── + local mem_drv, mem_err = memory_driver_any.new(child_logger('memory', '1')) + if not mem_drv then + dlog(SysmonManager.logger, 'error', { + what = 'driver_create_failed', class = 'memory', id = '1', err = mem_err, + }) + else + register_driver(mem_drv, 'memory', '1', {}, dev_ev_ch, cap_emit_ch) + end + + -- ── Thermal zones ── + local zones = discover_thermal_zones() + if #zones == 0 then + dlog(SysmonManager.logger, 'info', { what = 'no_thermal_zones_discovered' }) + end + for _, zone in ipairs(zones) do + local therm_drv, therm_err = thermal_driver_any.new( + zone.zone_id, + zone.sysfs_dir, + child_logger('thermal', zone.zone_id) + ) + if not therm_drv then + dlog(SysmonManager.logger, 'error', { + what = 'driver_create_failed', class = 'thermal', id = zone.zone_id, err = therm_err, + }) + else + register_driver(therm_drv, 'thermal', zone.zone_id, + { zone = zone.zone_id, path = zone.sysfs_dir }, dev_ev_ch, cap_emit_ch) + end + end + + dlog(SysmonManager.logger, 'debug', { what = 'all_devices_registered' }) +end + +---- public interface ---- + +---@param logger Logger? +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +---@return string error +function SysmonManager.start(logger, dev_ev_ch, cap_emit_ch) + if SysmonManager.started then + return "Already started" + end + + local scope, err = fibers.current_scope():child() + if not scope then + return "Failed to create child scope: " .. tostring(err) + end + SysmonManager.scope = scope + SysmonManager.logger = logger + + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + dlog(SysmonManager.logger, 'error', { what = 'scope_failed', err = tostring(primary), status = st }) + end + dlog(SysmonManager.logger, 'debug', { what = 'stopped' }) + end) + + scope:spawn(manager, dev_ev_ch, cap_emit_ch) + + SysmonManager.started = true + dlog(SysmonManager.logger, 'debug', { what = 'start_called' }) + return "" +end + +---@param timeout number? +---@return boolean ok +---@return string error +function SysmonManager.stop(timeout) + if not SysmonManager.started then + return false, "Not started" + end + timeout = timeout or STOP_TIMEOUT + SysmonManager.scope:cancel('sysmon manager stopped') + + local source = fibers.perform(op.named_choice { + join = SysmonManager.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + + if source == 'timeout' then + return false, "sysmon manager stop timeout" + end + SysmonManager.started = false + return true, "" +end + +---@param namespaces table +---@return boolean ok +---@return string error +function SysmonManager.apply_config(namespaces) -- luacheck: ignore + return true, "" +end + +return SysmonManager diff --git a/src/services/hal/managers/time.lua b/src/services/hal/managers/time.lua new file mode 100644 index 00000000..f431a86c --- /dev/null +++ b/src/services/hal/managers/time.lua @@ -0,0 +1,265 @@ +-- HAL modules +local time_driver = require "services.hal.drivers.time" +local hal_types = require "services.hal.types.core" + +-- Fibers modules +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local exec = require "fibers.io.exec" + +-- Other modules + +-- Constants + +local STOP_TIMEOUT = 5.0 -- seconds +local HOTPLUG_DIR = "/etc/hotplug.d/ntp" +local HOTPLUG_SCRIPT_NAME = "ntp" + +---@alias TimeDriverHandle table + +---@class TimeManager +---@field scope Scope +---@field logger Logger? +---@field started boolean +---@field driver TimeDriverHandle? +---@field dev_ev_ch Channel? +---@field cap_emit_ch Channel? +local TimeManager = { + started = false, + driver = nil, + dev_ev_ch = nil, + cap_emit_ch = nil, + logger = nil, +} + +---- Internal Utilities ---- + +---Emit a HAL device-added event for the time capability provider. +---@param driver TimeDriverHandle +---@param capabilities Capability[] +local function emit_device_added(driver, capabilities) + local device_event, ev_err = hal_types.new.DeviceEvent( + "added", + "time", + driver.id, + { source = "ntp" }, + capabilities + ) + if not device_event then + TimeManager.logger:error({ what = 'device_added_event_failed', err = tostring(ev_err) }) + return + end + TimeManager.dev_ev_ch:put(device_event) +end + +---Emit a HAL device-removed event for the time capability provider. +---@param driver TimeDriverHandle +local function emit_device_removed(driver) + local device_event, ev_err = hal_types.new.DeviceEvent( + "removed", + "time", + driver.id, + {} + ) + if not device_event then + TimeManager.logger:error({ what = 'device_removed_event_failed', err = tostring(ev_err) }) + return + end + TimeManager.dev_ev_ch:put(device_event) +end + +---Stop the currently running driver (if any) and notify HAL that the device was +---removed. Safe to call when no driver is running. +---@return nil +local function stop_existing_driver() + local prev = TimeManager.driver + if not prev then return end + + TimeManager.logger:debug({ what = 'stopping_existing_driver' }) + local ok, stop_err = prev:stop(STOP_TIMEOUT) + if not ok then + TimeManager.logger:warn({ what = 'driver_stop_failed', err = tostring(stop_err) }) + end + + emit_device_removed(prev) + TimeManager.driver = nil +end + +---Run a command and require a zero exit status. +---@param ... string argv +---@return boolean ok +---@return string? error +local function run_checked(...) + local argv = { ... } + local spec = { stdin = 'null', stdout = 'null', stderr = 'null' } + for i = 1, #argv do spec[i] = argv[i] end + local status, code, _, err = fibers.perform(exec.command(spec):run_op()) + if status ~= 'exited' or code ~= 0 then + return false, tostring(err or ("exit code " .. tostring(code))) + end + return true, nil +end + +---Resolve the directory that contains this manager file. +---@return string dir +local function manager_dir() + local source = debug.getinfo(1, 'S').source or '' + source = source:gsub('^@', '') + return source:match('^(.*)/[^/]+$') or '.' +end + +---Install the NTP hotplug script into /etc/hotplug.d/ntp. +---@return boolean ok +---@return string? error +local function install_ntp_hotplug_script() + local src = manager_dir() .. "/time/" .. HOTPLUG_SCRIPT_NAME + local dst = HOTPLUG_DIR .. "/" .. HOTPLUG_SCRIPT_NAME + + local ok, err = run_checked("mkdir", "-p", HOTPLUG_DIR) + if not ok then + return false, "failed to create hotplug directory: " .. tostring(err) + end + + ok, err = run_checked("cp", src, dst) + if not ok then + return false, "failed to copy hotplug script from " .. src .. ": " .. tostring(err) + end + + ok, err = run_checked("chmod", "+x", dst) + if not ok then + return false, "failed to chmod hotplug script: " .. tostring(err) + end + + return true, nil +end + +---Initialise, apply capabilities, and start a new TimeDriver. Stops any previously +---running driver first. Called from within a manager-scope fiber so exec and channel +---operations are safe. +---@return nil +local function bring_up_driver() + stop_existing_driver() + + local installed, install_err = install_ntp_hotplug_script() + if not installed then + TimeManager.logger:error({ what = 'hotplug_script_install_failed', err = tostring(install_err) }) + return + end + + local driver, new_err = time_driver.new(TimeManager.logger:child({ component = 'driver' })) + if not driver then + TimeManager.logger:error({ what = 'driver_create_failed', err = tostring(new_err) }) + return + end + + local init_err = driver:init() + if init_err ~= "" then + TimeManager.logger:error({ what = 'driver_init_failed', err = tostring(init_err) }) + return + end + + local capabilities, cap_err = driver:capabilities(TimeManager.cap_emit_ch) + if not capabilities then + TimeManager.logger:error({ what = 'driver_capabilities_failed', err = tostring(cap_err) }) + return + end + + local ok, start_err = driver:start() + if not ok then + TimeManager.logger:error({ what = 'driver_start_failed', err = tostring(start_err) }) + return + end + + TimeManager.driver = driver + emit_device_added(driver, capabilities) + TimeManager.logger:debug({ what = 'driver_started', cap_id = tostring(driver.id) }) +end + +---- Manager Lifecycle ---- + +---Start the Time Manager. Creates a child scope for managing the driver lifetime. +---@param dev_ev_ch Channel Device event channel (DeviceEvent messages to HAL) +---@param cap_emit_ch Channel Capability emit channel (Emit messages to HAL) +---@return string error Empty string on success. +function TimeManager.start(logger, dev_ev_ch, cap_emit_ch) + if TimeManager.started then + return "already started" + end + + local scope, sc_err = fibers.current_scope():child() + if not scope then + return "failed to create child scope: " .. tostring(sc_err) + end + + TimeManager.scope = scope + TimeManager.logger = logger + TimeManager.dev_ev_ch = dev_ev_ch + TimeManager.cap_emit_ch = cap_emit_ch + + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + logger:error({ what = 'scope_failed', err = tostring(primary), status = st }) + end + logger:debug({ what = 'stopped' }) + end) + + TimeManager.started = true + logger:debug({ what = 'started' }) + return "" +end + +---Stop the Time Manager and its driver. Cancels the manager scope which will +---propagate cancellation to any running driver scope. +---@param timeout number? Timeout in seconds. Defaults to 5. +---@return boolean ok +---@return string error +function TimeManager.stop(timeout) + if not TimeManager.started then + return false, "not started" + end + + timeout = timeout or STOP_TIMEOUT + TimeManager.scope:cancel() + + local source = fibers.perform(op.named_choice { + join = TimeManager.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + + if source == 'timeout' then + return false, "time manager stop timeout" + end + + TimeManager.started = false + return true, "" +end + +---Apply time manager configuration. Spawns a fiber to create and start the time +---driver. The time driver requires no user-supplied configuration (sysntpd path is +---fixed), so the config table is accepted for interface consistency but ignored. +---@param config table +---@return boolean ok +---@return string error +function TimeManager.apply_config(config) -- luacheck: ignore config + if not TimeManager.started then + return false, "time manager not started" + end + if TimeManager.dev_ev_ch == nil or TimeManager.cap_emit_ch == nil then + return false, "channels not initialized (start must be called first)" + end + + TimeManager.logger:debug({ what = 'config_received' }) + + local ok, spawn_err = TimeManager.scope:spawn(function() + bring_up_driver() + end) + if not ok then + return false, "failed to spawn driver initialization: " .. tostring(spawn_err) + end + + return true, "" +end + +return TimeManager diff --git a/src/ubus_scripts/etc/hotplug.d/ntp/ntp b/src/services/hal/managers/time/ntp similarity index 100% rename from src/ubus_scripts/etc/hotplug.d/ntp/ntp rename to src/services/hal/managers/time/ntp diff --git a/src/services/hal/managers/uart.lua b/src/services/hal/managers/uart.lua index ee1467fb..046aaf16 100644 --- a/src/services/hal/managers/uart.lua +++ b/src/services/hal/managers/uart.lua @@ -1,118 +1,416 @@ -local context = require "fibers.context" -local queue = require "fibers.queue" -local op = require "fibers.op" -local log = require "services.log" -local service = require "service" -local uart_driver = require "services.hal.drivers.uart" - -local UARTManagement = {} -UARTManagement.__index = UARTManagement - -function UARTManagement:apply_config(config) - self._config_apply_q:put(config) -end - -function UARTManagement:_apply_config(ctx, conn, device_event_q, capability_info_q, config) - log.trace(string.format( - "%s - %s: Applying configuration", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - - if not config.serial_ports or type(config.serial_ports) ~= "table" then - log.error(string.format( - "%s - %s: No serial ports configured", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - return +---@module 'services.hal.managers.uart' + +local fibers = require 'fibers' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local channel = require 'fibers.channel' + +local device_events = require 'services.hal.support.device_events' +local driver_reconcile = require 'services.hal.support.driver_reconcile' +local strict_manager = require 'services.hal.support.strict_manager' +local resource = require 'devicecode.support.resource' +local driver_mod = require 'services.hal.drivers.uart' + +local M = { + api_mode = 'op_only', +} + + +local STOP_TIMEOUT = 5.0 + +---@class UARTManagerState +---@field started boolean +---@field scope Scope|nil +---@field logger table|nil +---@field dev_ev_ch Channel|nil +---@field cap_emit_ch Channel|nil +---@field cfg_ch Channel|nil +---@field generation integer +---@field drivers table +local S = { + started = false, + scope = nil, + logger = nil, + dev_ev_ch = nil, + cap_emit_ch = nil, + cfg_ch = nil, + generation = 0, + drivers = {}, +} + +local function finalise_manager_scope(scope, generation) + if S.scope ~= scope or S.generation ~= generation then + return + end + + for _, driver in pairs(S.drivers or {}) do + resource.terminate_checked(driver, 'manager finalised', 'HAL manager driver cleanup failed') + end + + S.started = false + S.scope = nil + S.logger = nil + S.dev_ev_ch = nil + S.cap_emit_ch = nil + S.cfg_ch = nil + S.drivers = {} +end + +local function valid_mode(mode) + return mode == nil + or mode == '8N1' + or mode == '7E1' + or mode == '8O1' +end + +local function is_sequence(t) + if type(t) ~= 'table' then + return false end - for name, driver in pairs(self._uart_drivers) do - if not config.serial_ports[name] then - driver.ctx:cancel() - self._uart_drivers[name] = nil - device_event_q:put({ - connected = false, - type = "uart", - id_field = "name", - data = { - name = name - } - }) + local n = 0 + for k in pairs(t) do + if type(k) ~= 'number' or k < 1 or k % 1 ~= 0 then + return false end + n = n + 1 end - for _, port_cfg in ipairs(config.serial_ports) do - local driver = self._uart_drivers[port_cfg.name] - if driver and driver:get_port() ~= port_cfg.path then - driver.ctx:cancel() - self._uart_drivers[port_cfg.name] = nil - device_event_q:put({ - connected = false, - type = "uart", - id_field = "name", - data = { - name = port_cfg.name - } - }) - driver = nil - end - - if not driver then - local driver_ctx = context.with_cancel(ctx) - driver = uart_driver.new(driver_ctx, port_cfg.name, port_cfg.path) - self._uart_drivers[port_cfg.name] = driver - driver:spawn(conn) - local capabilities = driver:apply_capabilities(capability_info_q) - device_event_q:put({ - connected = true, - type = "uart", - id_field = "name", - capabilities = capabilities, - data = { - name = port_cfg.name - } - }) + return n == #t +end + +local function normalise_config(raw) + if type(raw) ~= 'table' then + return nil, 'uart config must be a table with serial_ports list' + end + + for k in pairs(raw) do + if k ~= 'serial_ports' then + return nil, 'uart config only supports serial_ports' end end -end -function UARTManagement:_manager(ctx, conn, device_event_q, capability_info_q) - log.trace(string.format( - "%s - %s: Starting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) + if not is_sequence(raw.serial_ports) then + return nil, 'uart serial_ports must be a list' + end + + return raw.serial_ports, nil +end - while not ctx:err() do - op.choice( - self._config_apply_q:get_op():wrap(function(config) - self:_apply_config(ctx, conn, device_event_q, capability_info_q, config) - end), - ctx:done_op() - ):perform() +local function validate_config(entries) + for _, entry in ipairs(entries) do + if type(entry) ~= 'table' then + return false, 'each uart entry must be a table' + end + if type(entry.id) ~= 'string' or entry.id == '' then + return false, 'uart entry id must be a non-empty string' + end + if type(entry.path) ~= 'string' or entry.path == '' then + return false, 'uart entry path must be a non-empty string' + end + if entry.baud ~= nil and (type(entry.baud) ~= 'number' or entry.baud <= 0 or entry.baud % 1 ~= 0) then + return false, 'uart entry baud must be a positive integer' + end + if not valid_mode(entry.mode) then + return false, 'uart entry mode is invalid' + end end - log.trace(string.format( - "%s - %s: Stopping", - ctx:value("service_name"), - ctx:value("fiber_name") - )) + return true, nil +end + +local UART_MANAGER_SOURCE_ID = 'uart_manager' + +local function uart_device_meta(driver) + return { + provider = 'hal.uart', + source_id = UART_MANAGER_SOURCE_ID, + path = driver.path, + baud = driver.default_baud, + mode = driver.default_mode, + } +end + +local function emit_device_added_op(driver, caps) + return device_events.added_op(S.dev_ev_ch, 'uart', driver.id, uart_device_meta(driver), caps) end -function UARTManagement:spawn(ctx, conn, device_event_q, capability_info_q) - service.spawn_fiber("UART Manager", conn, ctx, function(fctx) - self:_manager(fctx, conn, device_event_q, capability_info_q) +local function emit_device_removed_op(driver) + return device_events.removed_op(S.dev_ev_ch, 'uart', driver.id, uart_device_meta(driver)) +end + +local function same_driver_config(driver, entry) + return driver.path == entry.path + and driver.default_baud == entry.baud + and driver.default_mode == entry.mode +end + +local function start_driver_op(entry) + return fibers.run_scope_op(function () + local driver = driver_mod.new( + entry.id, + entry.path, + entry.baud, + entry.mode, + S.logger + ) + + local ok_caps, caps_or_err = fibers.perform(driver:capabilities_op(S.cap_emit_ch)) + if not ok_caps then + return false, tostring(caps_or_err) + end + local caps = caps_or_err + + local started = false + local handed_off = false + + local function cleanup_started_driver() + if started and not handed_off then + resource.terminate_checked(driver, 'manager cleanup', 'HAL manager driver rollback failed') + end + end + + fibers.current_scope():finally(function () + cleanup_started_driver() + end) + + local ok_start, start_err = + fibers.perform(driver:start_op(assert(S.scope, 'uart manager scope missing'))) + if not ok_start then + return false, tostring(start_err) + end + started = true + + local ok_emit, emit_err = fibers.perform(emit_device_added_op(driver, caps)) + if not ok_emit then + return false, tostring(emit_err) + end + + S.drivers[entry.id] = driver + handed_off = true + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err end) end -local function new() - local uart_management = { - _config_apply_q = queue.new(10), - _uart_drivers = {} - } - return setmetatable(uart_management, UARTManagement) +local function stop_driver_op(id, driver) + return fibers.run_scope_op(function () + local ok_emit, emit_err = fibers.perform(emit_device_removed_op(driver)) + if not ok_emit then + return false, tostring(emit_err) + end + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + if not ok_stop then + return false, tostring(stop_err) + end + + S.drivers[id] = nil + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +local function reconcile_op(entries) + return driver_reconcile.reconcile_op { + current = S.drivers, + entries = entries, + id_of = function (entry) return entry.id end, + same = same_driver_config, + stop = stop_driver_op, + start = function (_, entry) return start_driver_op(entry) end, + } +end + +local function reply_config_op(req, reply) + if type(req) ~= 'table' + or type(req.reply_ch) ~= 'table' + or type(req.reply_ch.put_op) ~= 'function' + then + return op.always(false, 'config reply channel missing') + end + + return req.reply_ch:put_op(reply):wrap(function () + return true, nil + end):or_else(function () + return false, 'reply_not_ready' + end) +end + +local function shell_loop(generation, cfg_ch) + assert(S.scope, 'uart shell without scope') + + while true do + local req = fibers.perform(cfg_ch:get_op()) + if not req then + return + end + + if req.generation ~= generation or S.generation ~= generation then + fibers.perform(reply_config_op(req, { + ok = false, + err = 'stale manager generation', + })) + else + local ok_reconcile, reconcile_err = fibers.perform(reconcile_op(req.config)) + local replied, reply_err = fibers.perform(reply_config_op(req, { + ok = ok_reconcile, + err = reconcile_err, + })) + + if replied ~= true and S.logger and S.logger.warn then + S.logger:warn({ + what = 'config_reply_failed', + err = tostring(reply_err), + }) + end + end + end +end + +function M.start_op(logger, dev_ev_ch, cap_emit_ch) + local owner_scope = fibers.current_scope() + assert(owner_scope ~= nil, 'uart.start_op must be called from inside a fiber') + + return op.guard(function () + if S.started then + return op.always(false, 'already started') + end + + local scope, err = owner_scope:child() + if not scope then + return op.always(false, tostring(err)) + end + + S.scope = scope + S.logger = logger + S.dev_ev_ch = dev_ev_ch + S.cap_emit_ch = cap_emit_ch + + S.cfg_ch = channel.new(8) + S.generation = S.generation + 1 + local generation = S.generation + local cfg_ch = S.cfg_ch + + local detach_finaliser = scope:finally(function () + finalise_manager_scope(scope, generation) + end) + + local ok, serr = scope:spawn(function () shell_loop(generation, cfg_ch) end) + if not ok then + detach_finaliser() + finalise_manager_scope(scope, generation) + scope:cancel(tostring(serr or 'manager shell spawn failed')) + return op.always(false, tostring(serr)) + end + + S.started = true + return op.always(true, nil) + end) +end + +function M.apply_config_op(entries) + return fibers.run_scope_op(function () + local normalised, norm_err = normalise_config(entries) + if not normalised then + return false, norm_err + end + + local ok, err = validate_config(normalised) + if not ok then + return false, err + end + if not S.started then + return false, 'uart manager not started' + end + + local cfg_ch = S.cfg_ch + local generation = S.generation + if not cfg_ch then + return false, 'uart manager not started' + end + + local reply_ch = channel.new(1) + local admitted, admit_err = fibers.perform(cfg_ch:put_op({ + generation = generation, + config = normalised, + reply_ch = reply_ch, + }):wrap(function () + return true, nil + end):or_else(function () + return false, 'uart_manager_config_busy' + end)) + + if admitted ~= true then + return false, tostring(admit_err or 'uart_manager_config_busy') + end + + local reply, recv_err = fibers.perform(reply_ch:get_op()) + if not reply then + return false, tostring(recv_err or 'config reply missing') + end + + return reply.ok, reply.err + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then + return false, tostring(err or rep) + end + return ok, err + end) +end + +function M.shutdown_op(timeout) + timeout = timeout or STOP_TIMEOUT + + return op.guard(function () + if not S.started or not S.scope then + return op.always(true, nil) + end + + local scope = S.scope + local generation = S.generation + scope:cancel() + + return fibers.boolean_choice( + scope:join_op():wrap(function () + finalise_manager_scope(scope, generation) + return true, nil + end), + sleep.sleep_op(timeout):wrap(function () + return false, 'uart manager stop timeout' + end) + ):wrap(function (completed, _a, b) + if completed then + return true, nil + end + return false, b + end) + end) +end + +function M.terminate(reason) + reason = tostring(reason or 'uart manager terminated') + + local scope = S.scope + local generation = S.generation + if scope then + scope:cancel(reason) + end + + finalise_manager_scope(scope, generation) + return true, nil +end + +function M.fault_op() + return strict_manager.fault_op_for_state(S) end -return { new = new } +return M diff --git a/src/services/hal/managers/ubus.lua b/src/services/hal/managers/ubus.lua deleted file mode 100644 index 7d57e79d..00000000 --- a/src/services/hal/managers/ubus.lua +++ /dev/null @@ -1,69 +0,0 @@ -local exec = require "fibers.exec" -local context = require "fibers.context" -local log = require "services.log" -local ubus_driver = require "services.hal.drivers.ubus" -local service = require "service" - -local UBusManagement = {} -UBusManagement.__index = UBusManagement - -local function new() - local ubus_management = {} - return setmetatable(ubus_management, UBusManagement) -end - -function UBusManagement:apply_config(config) - -- Currently no config options -end - -function UBusManagement:_manager(ctx, conn, device_event_q, capability_info_q) - log.trace(string.format( - "%s - %s: Starting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - - -- Check that ubus command responds - local _, err = exec.command_context(ctx, 'ubus', '-v'):output() - - if err and err ~= 1 then - log.error(string.format( - "%s - %s: ubus driver cannot be started, reason: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - err - )) - return - end - - local ubus_instance = ubus_driver.new(context.with_cancel(ctx)) - - local capabilities, cap_err = ubus_instance:apply_capabilities(capability_info_q) - if cap_err then - log.error(cap_err) - return - end - - ubus_instance:spawn(conn) - - local device_event = { - connected = true, - type = 'bus', - capabilities = capabilities, - device_control = {}, - id_field = "id", - data = { - id = "ubus" - } - } - - device_event_q:put(device_event) -end - -function UBusManagement:spawn(ctx, conn, device_event_q, capability_info_q) - service.spawn_fiber("UBus Manager", conn, ctx, function (fctx) - self:_manager(fctx, conn, device_event_q, capability_info_q) - end) -end - -return { new = new } diff --git a/src/services/hal/managers/uci.lua b/src/services/hal/managers/uci.lua deleted file mode 100644 index e39f0ada..00000000 --- a/src/services/hal/managers/uci.lua +++ /dev/null @@ -1,68 +0,0 @@ -local context = require "fibers.context" -local log = require "services.log" -local uci_driver = require "services.hal.drivers.uci" -local service = require "service" - -local UCIManagement = {} -UCIManagement.__index = UCIManagement - -local function new() - local ubus_management = {} - local uci_management = {} - return setmetatable(uci_management, UCIManagement) -end - -function UCIManagement:apply_config(config) - -- Currently no config options -end - -function UCIManagement:_manager(ctx, conn, device_event_q, capability_info_q) - log.trace(string.format( - "%s - %s: Starting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - - -- Check that uci package exists - local uci = require "uci" - - if not uci then - log.error(string.format( - "%s - %s: uci package cannot be loaded", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - return - end - - local uci_instance = uci_driver.new(context.with_cancel(ctx)) - - local capabilities, cap_err = uci_instance:apply_capabilities(capability_info_q) - if cap_err then - log.error(cap_err) - return - end - - uci_instance:spawn(conn) - - local device_event = { - connected = true, - type = 'uci', - capabilities = capabilities, - device_control = {}, - id_field = "id", - data = { - id = "uci" - } - } - - device_event_q:put(device_event) -end - -function UCIManagement:spawn(ctx, conn, device_event_q, capability_info_q) - service.spawn_fiber("UCI Manager", conn, ctx, function (fctx) - self:_manager(fctx, conn, device_event_q, capability_info_q) - end) -end - -return { new = new } diff --git a/src/services/hal/managers/usb.lua b/src/services/hal/managers/usb.lua new file mode 100644 index 00000000..fad60cae --- /dev/null +++ b/src/services/hal/managers/usb.lua @@ -0,0 +1,147 @@ +-- services/hal/managers/usb.lua +-- +-- USB HAL Manager. +-- Creates a single UsbDriver for the USB 3.0 bus, registers it with HAL, +-- and holds its scope until the manager is stopped. + +local usb_driver = require "services.hal.drivers.usb" +local hal_types = require "services.hal.types.core" + +---@type any +local usb_driver_any = usb_driver + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local cond = require "fibers.cond" + +local STOP_TIMEOUT = 5.0 + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +---@class UsbManager +---@field scope Scope? +---@field started boolean +---@field logger Logger? +local UsbManager = { + started = false, + scope = nil, + logger = nil, +} + +---- manager fiber ---- + +---@param scope Scope +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +local function manager(scope, dev_ev_ch, cap_emit_ch) + dlog(UsbManager.logger, 'debug', { what = 'started' }) + + scope:finally(function() + dlog(UsbManager.logger, 'debug', { what = 'closed' }) + end) + + local driver_logger = nil + if UsbManager.logger and UsbManager.logger.child then + driver_logger = UsbManager.logger:child({ component = 'driver', driver = 'usb', id = 'usb3' }) + end + + local driver, drv_err = usb_driver_any.new('usb3', driver_logger) + if not driver then + error("USB Manager: failed to create USB driver: " .. tostring(drv_err)) + end + + local init_err = driver:init() + if init_err ~= "" then + error("USB Manager: failed to init driver: " .. tostring(init_err)) + end + + local capabilities, cap_err = driver:capabilities(cap_emit_ch) + if cap_err ~= "" then + error("USB Manager: failed to bind capabilities: " .. tostring(cap_err)) + end + + local ready_cond = cond.new() + local device_event, ev_err = hal_types.new.DeviceEvent( + "added", "usb", "usb3", {}, capabilities, ready_cond) + if not device_event then + error("USB Manager: failed to create DeviceEvent: " .. tostring(ev_err)) + end + dev_ev_ch:put(device_event) + ready_cond:wait() + + local ok, start_err = driver:start() + if not ok then + error("USB Manager: failed to start driver: " .. tostring(start_err)) + end + + dlog(UsbManager.logger, 'debug', { what = 'device_registered' }) +end + +---- public interface ---- + +---@param logger Logger? +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +---@return string error +function UsbManager.start(logger, dev_ev_ch, cap_emit_ch) + if UsbManager.started then + return "Already started" + end + + local scope, err = fibers.current_scope():child() + if not scope then + return "Failed to create child scope: " .. tostring(err) + end + UsbManager.scope = scope + UsbManager.logger = logger + + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + dlog(UsbManager.logger, 'error', { what = 'scope_failed', err = tostring(primary), status = st }) + end + dlog(UsbManager.logger, 'debug', { what = 'stopped' }) + end) + + scope:spawn(manager, dev_ev_ch, cap_emit_ch) + + UsbManager.started = true + dlog(UsbManager.logger, 'debug', { what = 'start_called' }) + return "" +end + +---@param timeout number? +---@return boolean ok +---@return string error +function UsbManager.stop(timeout) + if not UsbManager.started then + return false, "Not started" + end + timeout = timeout or STOP_TIMEOUT + UsbManager.scope:cancel('usb manager stopped') + + local source = fibers.perform(op.named_choice { + join = UsbManager.scope:join_op(), + timeout = sleep.sleep_op(timeout), + }) + + if source == 'timeout' then + return false, "usb manager stop timeout" + end + UsbManager.started = false + return true, "" +end + +---@param namespaces table +---@return boolean ok +---@return string error +function UsbManager.apply_config(namespaces) -- luacheck: ignore + return true, "" +end + +return UsbManager diff --git a/src/services/hal/managers/wired.lua b/src/services/hal/managers/wired.lua new file mode 100644 index 00000000..cc7c80a7 --- /dev/null +++ b/src/services/hal/managers/wired.lua @@ -0,0 +1,405 @@ +-- services/hal/managers/wired.lua +-- Strict op-only HAL manager for semantic wired-provider capabilities. +-- +-- Provider capabilities are derived from HAL configuration. The manager +-- deliberately keeps wired-provider capabilities raw at the HAL boundary; the +-- Wired service combines them with Device assembly into public state/wired/... surfaces. + +local fibers = require 'fibers' +local safe = require 'coxpcall' +local op = require 'fibers.op' +local channel = require 'fibers.channel' +local cond = require 'fibers.cond' + +local strict = require 'services.hal.support.strict_manager' +local hal_types = require 'services.hal.types.core' +local cap_types = require 'services.hal.types.capabilities' +local backend_mod = require 'services.hal.drivers.wired' +local provider_runner = require 'services.hal.managers.wired.provider_runner' + +local M = strict.api_table() + +local state = { + started = false, + scope = nil, + logger = nil, + dev_ev_ch = nil, + cap_emit_ch = nil, + http_client_for = nil, + runners = {}, -- provider_id -> provider runner handle + controls = {}, + provider_ids = {}, + device_registered = false, +} + +local function log(level, payload) + if state.logger and type(state.logger[level]) == 'function' then state.logger[level](state.logger, payload) end +end + +local function new_reply(ok, payload) + return assert(hal_types.new.Reply(ok == true, payload)) +end + +local function reply(req, ok, payload) + if not req or not req.reply_ch then return end + fibers.perform(req.reply_ch:put_op(new_reply(ok, payload))) +end + +local function sorted_keys(t) + local keys = {} + for k in pairs(t or {}) do keys[#keys + 1] = k end + table.sort(keys, function(a, b) return tostring(a) < tostring(b) end) + return keys +end + +local function shallow_copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +local function list_signature(list) + return table.concat(list or {}, '\0') +end + +local function normalise_groups(groups, path) + if type(groups) ~= 'table' then return nil, path .. '.groups must be a non-empty array' end + local out = {} + for i = 1, #groups do + local group = groups[i] + if type(group) ~= 'string' or group == '' then return nil, path .. '.groups[' .. tostring(i) .. '] must be a non-empty string' end + out[#out + 1] = group + end + if #out == 0 then return nil, path .. '.groups must be a non-empty array' end + return out, nil +end + +local function provider_poll_plan(config) + config = config or {} + if config.poll_interval_s ~= nil then return nil, 'use poll, not poll_interval_s, for grouped wired polling' end + if config.poll == nil then return nil, 'poll is required' end + if type(config.poll) ~= 'table' then return nil, 'poll must be a table' end + local out = {} + for _, name in ipairs(sorted_keys(config.poll)) do + local rec = config.poll[name] + local path = 'poll.' .. tostring(name) + if type(rec) ~= 'table' then return nil, path .. ' must be a table' end + local interval_s = tonumber(rec.interval_s) + if interval_s == nil or interval_s <= 0 then return nil, path .. '.interval_s must be a positive number' end + local groups, gerr = normalise_groups(rec.groups, path) + if not groups then return nil, gerr end + out[#out + 1] = { name = tostring(name), interval_s = interval_s, groups = groups } + end + if #out == 0 then return nil, 'poll must contain at least one poll group' end + return out, nil +end + +local function emit_state(class, id, key, payload) + local ev = assert(hal_types.new.Emit(class, id, 'state', key, payload)) + return state.cap_emit_ch:put_op(ev):wrap(function () return true, nil end) +end + +local function emit_provider_state(provider_id, key, payload) + local ok, err = fibers.perform(emit_state('wired-provider', provider_id, key, payload or {})) + if ok == false or ok == nil then return nil, err end + return true, nil +end + +local function runner_result(provider_id, method, opts) + local runner = state.runners[provider_id] + if not runner then return { ok = false, err = 'wired provider not configured', code = 'not_configured' } end + local opname = tostring(method) .. '_op' + local fn = runner[opname] + if type(fn) ~= 'function' then return { ok = false, err = 'wired runner missing ' .. opname } end + local ok, runner_op = safe.pcall(function () return fn(runner, opts or {}) end) + if not ok then return { ok = false, err = tostring(runner_op) } end + if type(runner_op) ~= 'table' then return { ok = false, err = opname .. ' did not return an Op' } end + local ok2, result = safe.pcall(function () return fibers.perform(runner_op) end) + if not ok2 then return { ok = false, err = tostring(result) } end + if type(result) == 'table' then return result end + return { ok = result == true, result = result } +end + +local function handle_request(provider_id, req) + local verb = req and req.verb + local opts = req and req.opts or {} + local result + if verb == 'snapshot' or verb == 'watch' or verb == 'apply_attachments' or verb == 'set_poe' or verb == 'bounce' then + result = runner_result(provider_id, verb, opts) + else + result = { ok = false, err = 'unsupported wired-provider verb: ' .. tostring(verb) } + end + reply(req, result and result.ok == true, result) +end + +local function control_loop(provider_id, ch) + while true do + local req = fibers.perform(ch:get_op()) + if req == nil then return end + handle_request(provider_id, req) + end +end + +local function make_capability(provider_id) + local ch = channel.new(16) + state.controls[provider_id] = ch + return assert(cap_types.new.WiredProviderCapability(provider_id, ch)) +end + +local function make_caps(provider_ids) + local caps = {} + for i = 1, #provider_ids do caps[#caps + 1] = make_capability(provider_ids[i]) end + return caps +end + +local function device_event_op(event_type, caps, ready_cond) + local ev = assert(hal_types.new.DeviceEvent(event_type, 'wired', 'main', { + source = 'host', + source_id = 'wired', + manager = 'wired', + }, caps or {}, ready_cond)) + return state.dev_ev_ch:put_op(ev):wrap(function () return true, nil end) +end + +local function close_control_channels() + for _, ch in pairs(state.controls or {}) do if ch and type(ch.close) == 'function' then ch:close('reconfigured') end end + state.controls = {} +end + +local function stop_runners(reason) + for _, runner in pairs(state.runners or {}) do + if runner and type(runner.terminate) == 'function' then runner:terminate(reason or 'reconfigured') end + end + state.runners = {} +end + +local function spawn_control_loops(provider_ids) + for i = 1, #provider_ids do + local id = provider_ids[i] + state.scope:spawn(function () control_loop(id, state.controls[id]) end) + end +end + +local function normalise_provider_ids(config) + config = config or {} + local providers = config.providers + if type(providers) ~= 'table' then return nil, 'wired providers must be declared in providers map' end + local ids = {} + local keys = sorted_keys(providers) + for i = 1, #keys do + local key = tostring(keys[i]) + local rec = providers[key] + if key == '' then return nil, 'wired provider id must be a non-empty map key' end + if type(rec) ~= 'table' then return nil, ('wired provider %s must be a table'):format(key) end + if rec.id ~= nil then return nil, ('wired provider %s must use the map key as its id'):format(key) end + ids[#ids + 1] = key + end + return ids, nil +end + +local function configured_provider(config, provider_id) + config = config or {} + local providers = config.providers + if type(providers) ~= 'table' then return nil end + local rec = providers[provider_id] + if type(rec) ~= 'table' then return nil end + return shallow_copy(rec) +end + +local function reconcile_device_caps(provider_ids, ready_cond) + local new_sig = list_signature(provider_ids) + local old_sig = list_signature(state.provider_ids) + if new_sig == old_sig then return true, nil, ready_cond, true end + + if state.device_registered then + local ok, err = fibers.perform(device_event_op('removed', {})) + if ok == false or ok == nil then return nil, err or 'wired device remove event failed' end + state.device_registered = false + end + + close_control_channels() + state.provider_ids = {} + + if #provider_ids == 0 then return true, nil, ready_cond, true end + + local caps = make_caps(provider_ids) + spawn_control_loops(provider_ids) + local ok, err = fibers.perform(device_event_op('added', caps, ready_cond)) + if ok == false or ok == nil then + close_control_channels() + return nil, err or 'wired device add event failed' + end + + state.provider_ids = provider_ids + state.device_registered = true + return true, nil, ready_cond, false +end + +function M.start_op(logger, dev_ev_ch, cap_emit_ch, opts) + return op.guard(function () + if state.started then return op.always(true, nil) end + local parent = fibers.current_scope() + local child, cerr = parent:child() + if not child then return op.always(false, cerr or 'wired manager scope create failed') end + + state.scope = child + state.logger = logger + state.dev_ev_ch = dev_ev_ch + state.cap_emit_ch = cap_emit_ch + state.http_client_for = opts and opts.http_client_for or nil + state.controls = {} + state.runners = {} + state.provider_ids = {} + state.device_registered = false + + child:finally(function (_, status, primary) M.terminate(primary or status or 'wired manager closed') end) + state.started = true + log('debug', { what = 'wired_manager_started' }) + return op.always(true, nil) + end) +end + +local function terminate_prepared(prepared, reason) + for _, rec in pairs(prepared or {}) do + local backend_handle = rec and rec.backend_handle + if backend_handle and not rec.owned_by_runner and type(backend_handle.terminate) == 'function' then + backend_handle:terminate(reason or 'discarded') + end + end +end + +local function terminate_runners(runners, reason) + for _, runner in pairs(runners or {}) do + if runner and type(runner.terminate) == 'function' then runner:terminate(reason or 'discarded') end + end +end + +local function prepare_providers(config, provider_ids) + local prepared = {} + for i = 1, #provider_ids do + local id = provider_ids[i] + local pcfg = configured_provider(config or {}, id) + if not pcfg then + terminate_prepared(prepared, 'prepare failed') + return nil, ('wired provider %s missing configuration'):format(id) + end + + local driver_config = {} + for k, v in pairs(pcfg) do driver_config[k] = v end + local poll_plan, poll_err = provider_poll_plan(driver_config) + if not poll_plan then + terminate_prepared(prepared, 'prepare failed') + return nil, ('wired provider %s poll config failed: %s'):format(id, tostring(poll_err)) + end + driver_config.poll = nil + + local driver_opts = { logger = state.logger, cap_emit_ch = state.cap_emit_ch, provider_id = id } + if driver_config.provider == 'rtl8380m_http' then driver_opts.http_client_for = state.http_client_for end + local backend_handle, err = backend_mod.new(driver_config, driver_opts) + if not backend_handle then + terminate_prepared(prepared, 'prepare failed') + return nil, ('wired provider %s create failed: %s'):format(id, tostring(err)) + end + prepared[id] = { backend_handle = backend_handle, provider_name = backend_handle.provider_name, poll_plan = poll_plan } + end + return prepared, nil +end + +function M.apply_config_op(config) + return op.guard(function () + if not state.started then return op.always(false, 'wired manager not started') end + return fibers.run_scope_op(function () + local provider_ids, perr = normalise_provider_ids(config or {}) + if not provider_ids then return false, perr end + + local prepared, prep_err = prepare_providers(config or {}, provider_ids) + if not prepared then return false, prep_err end + + local caps_ready_cond = cond.new() + local runners = {} + for i = 1, #provider_ids do + local id = provider_ids[i] + local rec = prepared[id] + local runner, rerr = provider_runner.new({ + provider_id = id, + provider_name = rec.provider_name, + backend = rec.backend_handle, + poll_plan = rec.poll_plan, + ready_cond = caps_ready_cond, + parent_scope = state.scope, + emit_state = emit_provider_state, + log = log, + }) + if not runner then + terminate_runners(runners, 'runner create failed') + terminate_prepared(prepared, 'runner create failed') + return false, ('wired provider %s runner failed: %s'):format(id, tostring(rerr)) + end + rec.owned_by_runner = true + runners[id] = runner + end + + for i = 1, #provider_ids do + local id = provider_ids[i] + local spawned, spawn_err = runners[id]:start() + if spawned ~= true then + terminate_runners(runners, 'runner spawn failed') + terminate_prepared(prepared, 'runner spawn failed') + return false, ('wired provider %s runner failed: %s'):format(id, tostring(spawn_err)) + end + end + + stop_runners('reconfigured') + local ok, cerr, _, ready_now = reconcile_device_caps(provider_ids, caps_ready_cond) + if ok ~= true then + terminate_runners(runners, 'capability reconcile failed') + return false, cerr + end + + state.runners = {} + for i = 1, #provider_ids do + local id = provider_ids[i] + state.runners[id] = runners[id] + end + if ready_now and caps_ready_cond then caps_ready_cond:signal() end + log('debug', { what = 'wired_manager_configured', providers = provider_ids }) + return true, nil + end):wrap(function (status, report, ok_or_primary, err) + if status == 'ok' then + if ok_or_primary == true then return true, nil end + return false, err or ok_or_primary or 'wired manager configuration failed' + end + return false, ok_or_primary or (report and report.primary) or status or 'wired manager configuration failed' + end) + end) +end + +function M.shutdown_op(_timeout_s) + return op.guard(function () M.terminate('shutdown'); return op.always(true, nil) end) +end + +function M.terminate(reason) + stop_runners(reason or 'terminated') + close_control_channels() + state.provider_ids = {} + state.device_registered = false + if state.scope then local scope = state.scope; state.scope = nil; scope:cancel(reason or 'terminated') end + state.started = false + state.logger = nil + state.dev_ev_ch = nil + state.cap_emit_ch = nil + state.http_client_for = nil + return true, nil +end + +function M.fault_op() + return strict.fault_op_for_state(state) +end + +M._test = { + normalise_provider_ids = normalise_provider_ids, + provider_poll_plan = provider_poll_plan, + groups_for_plans = provider_runner._test.groups_for_plans, +} + +return M diff --git a/src/services/hal/managers/wired/provider_runner.lua b/src/services/hal/managers/wired/provider_runner.lua new file mode 100644 index 00000000..de87415b --- /dev/null +++ b/src/services/hal/managers/wired/provider_runner.lua @@ -0,0 +1,364 @@ +-- services/hal/managers/wired/provider_runner.lua +-- +-- Owned runner for one HAL wired-provider backend. The runner is the sole +-- owner of the backend/session object: polling, snapshots and future controls +-- all pass through this mailbox, giving CML-style serialisation without locks. + +local fibers = require 'fibers' +local safe = require 'coxpcall' +local op = require 'fibers.op' +local channel = require 'fibers.channel' +local sleep = require 'fibers.sleep' +local runtime = require 'fibers.runtime' +local tablex = require 'shared.table' + +local M = {} +local Runner = {} +Runner.__index = Runner + +local function max(a, b) if a > b then return a end return b end + +local function copy(v) return tablex.deep_copy(v) end + +local function stable_signature(v) + local tv = type(v) + if tv == 'nil' or tv == 'boolean' or tv == 'number' or tv == 'string' then + return tv .. ':' .. tostring(v) + end + if tv ~= 'table' then return tv .. ':' .. tostring(v) end + local keys = {} + for k in pairs(v) do keys[#keys + 1] = k end + table.sort(keys, function(a, b) return tostring(a) < tostring(b) end) + local out = { 'table{' } + for i = 1, #keys do + local k = keys[i] + out[#out + 1] = stable_signature(k) + out[#out + 1] = '=' + out[#out + 1] = stable_signature(v[k]) + out[#out + 1] = ';' + end + out[#out + 1] = '}' + return table.concat(out) +end + +local function merge_table(dst, src) + dst = dst or {} + if type(src) ~= 'table' then return dst end + for k, v in pairs(src) do + if v ~= nil then + if type(v) == 'table' and type(dst[k]) == 'table' then + merge_table(dst[k], v) + else + dst[k] = copy(v) + end + end + end + return dst +end + +local function merge_observation(cache, snapshot) + cache = cache or { + status = {}, + identity = {}, + runtime = {}, + power = {}, + surfaces = {}, + counters = {}, + topology = {}, + } + snapshot = snapshot or {} + if type(snapshot.status) == 'table' then merge_table(cache.status, snapshot.status) end + for _, key in ipairs({ 'identity', 'runtime', 'power', 'topology', 'counters' }) do + if type(snapshot[key]) == 'table' then merge_table(cache[key], snapshot[key]) end + end + if type(snapshot.surfaces) == 'table' then + for surface_id, surface in pairs(snapshot.surfaces) do + local id = tostring(surface_id or '') + if id ~= '' and type(surface) == 'table' then + cache.surfaces[id] = merge_table(cache.surfaces[id] or {}, surface) + end + end + end + return cache +end + +local function emit_state_changed(self, key, payload) + local sig = stable_signature(payload or {}) + if self.emitted[key] == sig then return true, nil, false end + self.emitted[key] = sig + local ok, err = self.emit_state(self.provider_id, key, payload or {}) + if ok == false or ok == nil then + self.emitted[key] = nil + return nil, err, false + end + return true, nil, true +end + +local function emit_snapshot(self, snapshot, present) + snapshot = snapshot or {} + present = present or snapshot + local ok, err + if present.status ~= nil then + ok, err = emit_state_changed(self, 'status', snapshot.status or { state = 'available', available = snapshot.ok == true }) + if ok ~= true then return nil, err end + end + if present.identity ~= nil then + ok, err = emit_state_changed(self, 'identity', snapshot.identity or {}) + if ok == false or ok == nil then return nil, err end + end + if present.runtime ~= nil then + ok, err = emit_state_changed(self, 'runtime', snapshot.runtime or {}) + if ok == false or ok == nil then return nil, err end + end + if present.power ~= nil then + ok, err = emit_state_changed(self, 'power', snapshot.power or {}) + if ok == false or ok == nil then return nil, err end + end + if present.surfaces ~= nil then + ok, err = emit_state_changed(self, 'surfaces', { surfaces = snapshot.surfaces or {} }) + if ok == false or ok == nil then return nil, err end + end + if present.counters ~= nil then + ok, err = emit_state_changed(self, 'counters', { counters = snapshot.counters or {} }) + if ok == false or ok == nil then return nil, err end + end + if present.topology ~= nil then + ok, err = emit_state_changed(self, 'topology', snapshot.topology or {}) + if ok == false or ok == nil then return nil, err end + end + return true, nil +end + +local function publish_observation(self, snapshot) + self.observation = merge_observation(self.observation, snapshot) + return emit_snapshot(self, self.observation, snapshot or {}) +end + +local function emit_status(self, status) + local ok, err = emit_state_changed(self, 'status', status or { state = 'available', available = true }) + if ok == false or ok == nil then return nil, err end + return true, nil +end + +local function copy_list(list) + local out = {} + for i = 1, #(list or {}) do out[i] = list[i] end + return out +end + +local function append_unique(out, seen, value) + value = tostring(value or '') + if value ~= '' and not seen[value] then + seen[value] = true + out[#out + 1] = value + end +end + +local function perform_backend_method(backend, method, opts) + local opname = tostring(method) .. '_op' + local fn = backend and backend[opname] + if type(fn) ~= 'function' then return { ok = false, err = 'wired backend missing ' .. opname } end + local ok, backend_op = safe.pcall(function () return fn(backend, opts or {}) end) + if not ok then return { ok = false, err = tostring(backend_op) } end + if type(backend_op) ~= 'table' then return { ok = false, err = opname .. ' did not return an Op' } end + local ok2, result = safe.pcall(function () return fibers.perform(backend_op) end) + if not ok2 then return { ok = false, err = tostring(result) } end + if type(result) == 'table' then return result end + return { ok = result == true, result = result } +end + +local function failure_status(provider_id, groups, result) + local gs = groups or {} + local err = result and result.err or (result and result.status and result.status.err) or 'wired provider observation failed' + local unavailable = false + for _, group in ipairs(gs) do + if group == 'panel' or group == 'snapshot' then unavailable = true end + end + if #gs == 0 then unavailable = true end + return { + state = unavailable and 'unavailable' or 'degraded', + available = not unavailable, + err = err, + groups = copy_list(gs), + provider_id = provider_id, + polling = true, + } +end + +local function groups_for_plans(plans) + local groups, seen = {}, {} + for _, plan in ipairs(plans or {}) do + for _, group in ipairs(plan.groups or {}) do append_unique(groups, seen, group) end + end + return groups +end + +local function initialise_due_times(self) + local now = runtime.now() + for i, plan in ipairs(self.poll_plan or {}) do + self.next_due[plan.name] = now + math.min(1.0, (i - 1) * 0.25) + end +end + +local function next_due_time(self) + local due = nil + for _, plan in ipairs(self.poll_plan or {}) do + local t = self.next_due[plan.name] + if t ~= nil and (due == nil or t < due) then due = t end + end + if due == nil then due = runtime.now() + 3600 end + return max(due, self.idle_until or 0) +end + +local function due_plans(self, now) + local plans = {} + for _, plan in ipairs(self.poll_plan or {}) do + local t = self.next_due[plan.name] + if t ~= nil and t <= now then plans[#plans + 1] = plan end + end + return plans +end + +local function mark_plans_attempted(self, plans) + local now = runtime.now() + for _, plan in ipairs(plans or {}) do self.next_due[plan.name] = now + plan.interval_s end + self.idle_until = now + self.min_idle_s +end + +local function runner_request_op(self, verb, opts) + return op.guard(function () + if self.closed then return op.always({ ok = false, err = 'wired provider runner closed', code = 'closed' }) end + local reply_ch = channel.new(1) + local msg = { kind = 'request', verb = verb, opts = opts or {}, reply_ch = reply_ch } + return fibers.run_scope_op(function () + fibers.perform(self.request_ch:put_op(msg)) + return fibers.perform(reply_ch:get_op()) + end):wrap(function (status, _report, result_or_primary, err) + if status == 'ok' then return result_or_primary, err end + return { ok = false, err = tostring(result_or_primary or status or 'wired provider request failed') }, nil + end) + end) +end + +function Runner:snapshot_op(req) return runner_request_op(self, 'snapshot', req) end +function Runner:watch_op(req) return runner_request_op(self, 'snapshot', req) end +function Runner:observe_groups_op(req) return runner_request_op(self, 'observe_groups', req) end +function Runner:apply_attachments_op(req) return runner_request_op(self, 'apply_attachments', req) end +function Runner:set_poe_op(req) return runner_request_op(self, 'set_poe', req) end +function Runner:bounce_op(req) return runner_request_op(self, 'bounce', req) end + +function Runner:terminate(reason) + self.closed = true + if self.scope then local scope = self.scope; self.scope = nil; scope:cancel(reason or 'wired provider runner terminated') end + if self.backend and type(self.backend.terminate) == 'function' then self.backend:terminate(reason or 'wired provider runner terminated') end + return true, nil +end + +function Runner:emit_observing() + if self.observing_emitted then return true, nil end + local ok, err = emit_status(self, { + state = 'observing', + available = false, + driver = self.provider_name, + polling = true, + }) + if ok == true then self.observing_emitted = true end + return ok, err +end + +function Runner:poll_due() + local now = runtime.now() + local plans = due_plans(self, now) + if #plans == 0 then return true, nil end + local groups = groups_for_plans(plans) + local result = perform_backend_method(self.backend, 'observe_groups', { groups = groups }) + mark_plans_attempted(self, plans) + if result and result.ok == true then return publish_observation(self, result) end + return emit_status(self, failure_status(self.provider_id, groups, result)) +end + +function Runner:handle_request(msg) + if type(msg) ~= 'table' then return end + local verb = msg.verb + local opts = msg.opts or {} + local result + if verb == 'snapshot' or verb == 'watch' then + result = perform_backend_method(self.backend, 'snapshot', opts) + if result and result.ok == true then publish_observation(self, result) end + elseif verb == 'observe_groups' then + result = perform_backend_method(self.backend, 'observe_groups', opts) + if result and result.ok == true then publish_observation(self, result) end + elseif verb == 'apply_attachments' or verb == 'set_poe' or verb == 'bounce' then + result = perform_backend_method(self.backend, verb, opts) + else + result = { ok = false, err = 'unsupported wired-provider verb: ' .. tostring(verb) } + end + if msg.reply_ch then fibers.perform(msg.reply_ch:put_op(result)) end +end + +function Runner:run() + if self.ready_cond ~= nil then fibers.perform(self.ready_cond:wait_op()) end + if self.closed then return end + local ok, err = self:emit_observing() + if ok ~= true then self.log('error', { what = 'wired_provider_initial_status_emit_failed', provider = self.provider_id, err = err }) end + initialise_due_times(self) + while not self.closed do + local which, msg = fibers.perform(op.named_choice({ + request = self.request_ch:get_op(), + due = sleep.sleep_until_op(next_due_time(self)), + })) + if which == 'request' then + self:handle_request(msg) + elseif which == 'due' then + local pok, perr = self:poll_due() + if pok ~= true then self.log('error', { what = 'wired_provider_poll_emit_failed', provider = self.provider_id, err = perr }) end + end + end +end + +function Runner:start() + local ok, err = self.scope:spawn(function () self:run() end) + if not ok then + self:terminate(tostring(err or 'wired provider runner spawn failed')) + return nil, err or 'wired provider runner spawn failed' + end + return true, nil +end + +function M.new(opts) + opts = opts or {} + if type(opts.provider_id) ~= 'string' or opts.provider_id == '' then return nil, 'provider_id is required' end + if type(opts.provider_name) ~= 'string' or opts.provider_name == '' then return nil, 'provider_name is required' end + if type(opts.backend) ~= 'table' then return nil, 'backend is required' end + if type(opts.poll_plan) ~= 'table' then return nil, 'poll_plan is required' end + if type(opts.parent_scope) ~= 'table' then return nil, 'parent_scope is required' end + if type(opts.emit_state) ~= 'function' then return nil, 'emit_state callback is required' end + + local child, err = opts.parent_scope:child() + if not child then return nil, err or 'wired provider runner scope create failed' end + local self = setmetatable({ + provider_id = opts.provider_id, + provider_name = opts.provider_name, + backend = opts.backend, + poll_plan = opts.poll_plan, + ready_cond = opts.ready_cond, + emit_state = opts.emit_state, + observation = nil, + emitted = {}, + log = opts.log or function () end, + request_ch = channel.new(opts.request_queue_size or 32), + scope = child, + closed = false, + next_due = {}, + idle_until = 0, + min_idle_s = tonumber(opts.min_idle_s) or 0.05, + observing_emitted = false, + }, Runner) + return self, nil +end + +M._test = { + groups_for_plans = groups_for_plans, +} + +return M diff --git a/src/services/hal/managers/wlan.lua b/src/services/hal/managers/wlan.lua index 9e7de94f..f3f19cc9 100644 --- a/src/services/hal/managers/wlan.lua +++ b/src/services/hal/managers/wlan.lua @@ -1,320 +1,325 @@ -local context = require "fibers.context" -local fiber = require "fibers.fiber" -local op = require "fibers.op" -local queue = require "fibers.queue" -local log = require "services.log" -local wireless_driver = require "services.hal.drivers.wireless" -local band_driver = require "services.hal.drivers.band" -local service = require "service" -local new_msg = require "bus".new_msg - -local WLANManagement = {} -WLANManagement.__index = WLANManagement - -local function new() - local wlan_management = { - _config_apply_q = queue.new(10), - _wlan_devices = {}, - _wlan_add_queue = queue.new(10), - _band_add_queue = queue.new(10) +local hal_types = require "services.hal.types.core" +local radio_driver = require "services.hal.drivers.radio" +local band_driver = require "services.hal.drivers.band" + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local channel = require "fibers.channel" + +local STOP_TIMEOUT = 5.0 + +local log -- set in start() + +---@class WLANManager +---@field scope Scope? +---@field started boolean +---@field radios table +---@field band BandDriver? +---@field config_ch Channel +local WLANManager = { + started = false, + radios = {}, + band = nil, + scope = nil, + config_ch = channel.new(4), +} + +------------------------------------------------------------------------ +-- Internal helpers +------------------------------------------------------------------------ + +---Start a single radio driver and emit device-added event. +---Must be called from within the manager scope's context. +---@param name string +---@param radio_cfg table { name, path, type } from device config +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +local function start_radio(name, radio_cfg, dev_ev_ch, cap_emit_ch) + local driver, drv_err = radio_driver.new(name, log:child({ radio = name })) + if not driver then + log:error({ what = 'create_radio_driver_failed', name = name, err = drv_err }) + return + end + + local init_err = driver:init(radio_cfg.path or '', radio_cfg.type or '') + if init_err ~= '' then + log:error({ what = 'init_radio_driver_failed', name = name, err = init_err }) + return + end + + local capabilities, cap_err = driver:capabilities(cap_emit_ch) + if not capabilities then + log:error({ what = 'radio_capabilities_failed', name = name, err = cap_err }) + return + end + + local ok, start_err = driver:start() + if not ok then + log:error({ what = 'start_radio_driver_failed', name = name, err = start_err }) + return + end + + WLANManager.radios[name] = { + driver = driver, + path = driver.staged.path, + type = driver.staged.type, } - return setmetatable(wlan_management, WLANManagement) + + local device_event, ev_err = hal_types.new.DeviceEvent( + 'added', + 'radio', + name, + { + provider = 'hal', + version = 1, + name = name, + path = driver.staged.path, + type = driver.staged.type, + }, + capabilities + ) + if not device_event then + log:error({ what = 'create_radio_device_event_failed', name = name, err = ev_err }) + return + end + + dev_ev_ch:put(device_event) + log:debug({ what = 'radio_driver_started', name = name }) end -function WLANManagement:apply_config(config) - self._config_apply_q:put(config) +---Stop a single radio driver and emit device-removed event. +---@param name string +---@param dev_ev_ch Channel +local function stop_radio(name, dev_ev_ch) + local entry = WLANManager.radios[name] + if not entry then + log:warn({ what = 'stop_radio_not_found', name = name }) + return + end + + -- Cancel the driver's scope (structured concurrency handles cleanup) + entry.driver.scope:cancel('removed by manager') + WLANManager.radios[name] = nil + + -- HAL unregisters by class+id, so an empty capabilities list is fine here + local device_event, ev_err = hal_types.new.DeviceEvent( + 'removed', + 'radio', + name, + {}, + {} + ) + if not device_event then + log:error({ what = 'create_radio_remove_event_failed', name = name, err = ev_err }) + return + end + dev_ev_ch:put(device_event) + log:debug({ what = 'radio_driver_stopped', name = name }) end -function WLANManagement:_apply_config(ctx, conn, device_event_q, capability_info_q, config) - log.trace(string.format( - "%s - %s: Applying WLAN config", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - if not config.radios or type(config.radios) ~= "table" then - log.error(string.format( - "%s - %s: Invalid WLAN config, 'radios' field missing or not a table", - ctx:value("service_name"), - ctx:value("fiber_name") - )) +---Start the band driver and emit device-added event. +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +local function start_band(dev_ev_ch, cap_emit_ch) + local driver, drv_err = band_driver.new(log:child({ driver = 'band' })) + if not driver then + log:warn({ what = 'create_band_driver_failed', err = drv_err }) return end - local radio_configs = config.radios - for radio_name, _ in pairs(self._wlan_devices) do - local device_event = self:_remove_wlan(ctx, radio_name) - if device_event then - device_event_q:put(device_event) - end + + local init_err = driver:init() + if init_err ~= '' then + log:warn({ what = 'init_band_driver_failed', err = init_err }) + return end - for radio_name, radio_config in pairs(radio_configs) do - self:_create_wlan(ctx, conn, radio_name, radio_config, capability_info_q) + local capabilities, cap_err = driver:capabilities(cap_emit_ch) + if not capabilities then + log:warn({ what = 'band_capabilities_failed', err = cap_err }) + return end -end -function WLANManagement:_create_wlan(ctx, conn, radio_name, radio_config, capability_info_q) - if self._wlan_devices[radio_name] then + local ok, start_err = driver:start() + if not ok then + log:warn({ what = 'start_band_driver_failed', err = start_err }) return end - fiber.spawn(function() - local wireless_instance = wireless_driver.new( - context.with_cancel(ctx), - radio_name, - radio_config.path, - radio_config.type - ) - local err = wireless_instance:init(conn) - if err then - log.error( - string.format("%s - %s: Failed to initialize wireless driver for %s: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - radio_name, - err - ) - ) - return - end - local capabilities, cap_err = wireless_instance:apply_capabilities(capability_info_q) - if cap_err then - log.error(cap_err) - return - end - wireless_instance:spawn(conn) - self._wlan_add_queue:put({ - capabilities = capabilities, - driver = wireless_instance - }) - end) -end -function WLANManagement:_remove_wlan(ctx, radio_name) - log.trace(string.format( - "%s - %s: Removed WLAN of name %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - radio_name - )) - local wireless_instance = self._wlan_devices[radio_name] - if not wireless_instance then - return nil + WLANManager.band = driver + + local device_event, ev_err = hal_types.new.DeviceEvent( + 'added', + 'band', + '1', + { provider = 'hal', version = 1 }, + capabilities + ) + if not device_event then + log:warn({ what = 'create_band_device_event_failed', err = ev_err }) + return end - wireless_instance.ctx:cancel() - self._wlan_devices[radio_name] = nil - local device_event = { - connected = false, - type = 'wlan', - id_field = "radioname", - data = { - interface = wireless_instance:get_phy(), - radioname = radio_name, - devpath = wireless_instance:get_path() - } - } - return device_event + + dev_ev_ch:put(device_event) + log:debug({ what = 'band_driver_started' }) end -function WLANManagement:_get_radios(ctx, conn) - local status_req = conn:request(new_msg( - { 'hal', 'capability', 'ubus', '1', 'control', 'call' }, - { 'network.wireless', 'status' } - )) - local status_msg, ctx_err = status_req:next_msg_with_context(ctx) - status_req:unsubscribe() - if status_msg and status_msg.payload and status_msg.payload.err or ctx_err then - return nil, status_msg.payload.err or ctx_err +---Reconcile running radio drivers against a new config. +---@param config table +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +local function reconcile_radios(config, dev_ev_ch, cap_emit_ch) + local new_radios = {} + if type(config.radios) == 'table' then + for _, r in ipairs(config.radios) do + if type(r.name) == 'string' and r.name ~= '' then + new_radios[r.name] = r + end + end end - if status_msg and status_msg.payload and status_msg.payload.result then - return status_msg.payload.result + + -- Stop drivers no longer in config or whose path/type changed + local to_stop = {} + for name, entry in pairs(WLANManager.radios) do + local new_r = new_radios[name] + if not new_r then + table.insert(to_stop, { name = name, reason = 'removed' }) + elseif (new_r.path or '') ~= entry.path or (new_r.type or '') ~= entry.type then + table.insert(to_stop, { name = name, reason = 'changed' }) + end + end + for _, s in ipairs(to_stop) do + log:debug({ what = 'stopping_radio', name = s.name, reason = s.reason }) + stop_radio(s.name, dev_ev_ch) end - return nil, "Failed to get response from ubus for radios" -end -function WLANManagement:_create_band_driver(ctx, conn, capability_info_q, device_event_q) - fiber.spawn(function() - local band_instance = band_driver.new(context.with_cancel(ctx)) - local band_init_ok, band_init_err = band_instance:init(ctx, conn) - if band_init_err then - log.error(string.format( - "%s - %s: band driver cannot be started, reason: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - band_init_err - )) - return nil, band_init_err + -- Start new or replacement radios + for name, radio_cfg in pairs(new_radios) do + if not WLANManager.radios[name] then + start_radio(name, radio_cfg, dev_ev_ch, cap_emit_ch) end - band_instance:spawn(conn) - local capabilities, cap_err = band_instance:apply_capabilities(capability_info_q) - if cap_err then - log.error(string.format( - "%s - %s: band driver cannot apply capabilities, reason: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - cap_err - )) - return nil, cap_err + end +end + +------------------------------------------------------------------------ +-- Manager fiber +------------------------------------------------------------------------ + +local function manager_fiber(scope, dev_ev_ch, cap_emit_ch) + scope:finally(function() + local st, primary = scope:status() + if st == 'failed' then + log:error({ what = 'wlan_manager_scope_error', err = tostring(primary) }) end - local device_event = { - connected = true, - type = 'band', - capabilities = capabilities, - device_control = {}, - id_field = "id", - data = { - id = "1", - } - } - device_event_q:put(device_event) + log:debug({ what = 'wlan_manager_stopped' }) end) + + -- Start the band driver immediately (always, regardless of radio count). + -- The band driver is persistent; it is not restarted on config updates. + start_band(dev_ev_ch, cap_emit_ch) + + -- Watch for config updates and driver faults + while true do + -- Build fault ops for each running radio driver + local fault_ops = {} + for name, entry in pairs(WLANManager.radios) do + table.insert(fault_ops, entry.driver.scope:fault_op():wrap(function(_, err) return name, err end)) + end + + local fault_op = op.never() + if #fault_ops > 0 then + fault_op = op.choice(unpack(fault_ops)) + end + + local source, val, fault_err = fibers.perform(fibers.named_choice({ + config = WLANManager.config_ch:get_op(), + driver_fault = fault_op, + cancel = scope:cancel_op(), + })) + + if source == 'cancel' then + break + elseif source == 'config' then + log:debug({ what = 'apply_config_received' }) + -- reconcile_radios handles both initial setup and subsequent updates + reconcile_radios(val, dev_ev_ch, cap_emit_ch) + elseif source == 'driver_fault' then + local name = val + log:error({ what = 'radio_driver_fault', name = tostring(name), err = tostring(fault_err) }) + -- Remove from registry (scope already failed); emit device-removed + if WLANManager.radios[name] then + WLANManager.radios[name] = nil + local device_event = hal_types.new.DeviceEvent('removed', 'radio', name, {}, {}) + if device_event then dev_ev_ch:put(device_event) end + end + end + end end -function WLANManagement:_manager(ctx, conn, device_event_q, capability_info_q) - local ubus_sub = conn:subscribe({ 'hal', 'capability', 'ubus', '1' }) - local _, ctx_err = ubus_sub:next_msg_with_context(ctx) -- wait for ubus capability to be available - ubus_sub:unsubscribe() - if ctx_err then return end - local uci_sub = conn:subscribe({ 'hal', 'capability', 'uci', '1' }) - local _, ctx_err = uci_sub:next_msg_with_context(ctx) - uci_sub:unsubscribe() - if ctx_err then return end - log.trace(string.format( - "%s - %s: Starting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - - -- Set dawn restart policy to immediate so that changes are applied immediately - conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set_restart_actions' }, - { 'dawn', { { '/etc/init.d/dawn', 'restart' } } } - )) - - self:_create_band_driver(ctx, conn, capability_info_q, device_event_q) - - - -- Set wireless restart policy to immediate so that changes are applied immediately - conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set_restart_actions' }, - { 'wireless', { { 'wifi', 'reload' } } } - )) - - local hotplug_req = conn:request(new_msg( - { 'hal', 'capability', 'ubus', '1', 'control', 'listen' }, - { 'hotplug.net' } - )) - - local msg, ctx_err = hotplug_req:next_msg_with_context(ctx) - hotplug_req:unsubscribe() - if ctx_err or msg and msg.payload and msg.payload.err then - log.error(string.format( - "%s - %s: ubus driver cannot be started, reason: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - ctx_err or msg.payload.err - )) - return +------------------------------------------------------------------------ +-- Public manager interface +------------------------------------------------------------------------ + +---Start the WLAN Manager. +---Creates a child scope and launches the manager and driver fibers. +---@param logger table +---@param dev_ev_ch Channel +---@param cap_emit_ch Channel +---@return string err "" on success +function WLANManager.start(logger, dev_ev_ch, cap_emit_ch) + log = logger + + if WLANManager.started then + return "already started" + end + + local scope, scope_err = fibers.current_scope():child() + if not scope then + return "failed to create child scope: " .. tostring(scope_err) end + WLANManager.scope = scope - local stream_id = msg.payload.result.stream_id + scope:spawn(manager_fiber, dev_ev_ch, cap_emit_ch) - local hotplug_sub = conn:subscribe( - { 'hal', 'capability', 'ubus', '1', 'info', 'stream', stream_id } - ) - local stream_end_sub = conn:subscribe( - { 'hal', 'capability', 'ubus', '1', 'info', 'stream', stream_id, 'closed' } - ) + WLANManager.started = true + log:debug({ what = 'wlan_manager_started' }) + return "" +end - local process_ended = false - - while not process_ended and not ctx:err() do - op.choice( - self._config_apply_q:get_op():wrap(function(config) - self:_apply_config(ctx, conn, device_event_q, capability_info_q, config) - end), - self._wlan_add_queue:get_op():wrap(function(wlan_device) - local driver = wlan_device.driver - local capabilities = wlan_device.capabilities - self._wlan_devices[driver:get_name()] = driver - device_event_q:put({ - connected = true, - type = 'wlan', - capabilities = capabilities, - device_control = {}, - id_field = "radioname", - data = { - phy = driver:get_phy(), - radioname = driver:get_name(), - devpath = driver:get_path(), - } - }) - end), - hotplug_sub:next_msg_op():wrap(function(msg) - local event = msg.payload - if not event then return end - local data = event['hotplug.net'] - if data.devtype ~= 'wlan' then return end - local phy = data.interface:match("^(phy%d+)") - local phy_found = false - -- First check if we already have a driver with this phy assigned - -- route interface event to that driver - for _, radio in pairs(self._wlan_devices) do - if radio:get_phy() == phy then - phy_found = true - if data.action == "add" then - radio:attach_interface(data.interface) - else - radio:detach_interface(data.interface) - end - end - end - -- If we don't have a driver with this phy assigned, it means it's a new phy - -- We must scan all radios to find the one without a phy assigned but with an interface - -- matching the phy of the event and assign it - -- this is not very efficient but we don't have a better way to do it for now - if not phy_found then - radios = self:_get_radios(ctx, conn) - for radio_name, radio_metadata in pairs(radios or {}) do - if radio_metadata.interfaces and - radio_metadata.interfaces[1] and - radio_metadata.interfaces[1].ifname and - self._wlan_devices[radio_name] and - not self._wlan_devices[radio_name]:get_phy() and - radio_metadata.interfaces[1].ifname:match("^(phy%d+)") == phy - then - if data.action == "add" then - self._wlan_devices[radio_name]:attach_interface(data.interface) - else - self._wlan_devices[radio_name]:detach_interface(data.interface) - end - end - end - end - end), - stream_end_sub:next_msg_op():wrap(function(msg) - process_ended = msg.payload - end), - ctx:done_op():wrap(function() - conn:publish(new_msg( - { 'hal', 'capability', 'ubus', '1', 'control', 'stop_stream' }, - { stream_id } - )) - end) - ):perform() +---Apply a new device config to the WLAN Manager. +---The manager reconciles radio drivers against the new config. +---@param config table +---@return boolean ok +---@return string err +function WLANManager.apply_config(config) + if not WLANManager.started then + return false, "not started" end - hotplug_sub:unsubscribe() - stream_end_sub:unsubscribe() - - log.trace(string.format( - "%s - %s: Stopping", - ctx:value("service_name"), - ctx:value("fiber_name") - )) + WLANManager.config_ch:put(config) + return true, "" end -function WLANManagement:spawn(ctx, conn, device_event_q, capability_info_q) - service.spawn_fiber("WLAN Manager", conn, ctx, function(fctx) - self:_manager(fctx, conn, device_event_q, capability_info_q) - end) +---Stop the WLAN Manager and all its child drivers. +---@param timeout number? +---@return boolean ok +---@return string err +function WLANManager.stop(timeout) + if not WLANManager.started then + return false, "not started" + end + timeout = timeout or STOP_TIMEOUT + WLANManager.scope:cancel() + + local source = fibers.perform(op.named_choice({ + join = WLANManager.scope:join_op(), + timeout = sleep.sleep_op(timeout), + })) + + if source == 'timeout' then + return false, "wlan manager stop timeout" + end + WLANManager.started = false + return true, "" end -return { new = new } +return WLANManager diff --git a/src/services/hal/sdk/cap.lua b/src/services/hal/sdk/cap.lua new file mode 100644 index 00000000..1f499d21 --- /dev/null +++ b/src/services/hal/sdk/cap.lua @@ -0,0 +1,536 @@ +local cap_args = require 'services.hal.types.capability_args' + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local wait = require 'fibers.wait' + +local perform = fibers.perform + +---------------------------------------------------------------------- +-- Topic helpers +---------------------------------------------------------------------- + +-- Legacy public capability discovery surface. +local function t_cap_listen(class, id) + return { 'cap', class, id, 'state' } +end + +local function t_cap_state(class, id, state) + return { 'cap', class, id, 'state', state } +end + +local function t_cap_event(class, id, event) + return { 'cap', class, id, 'event', event } +end + +local function t_cap_control(class, id, method) + return { 'cap', class, id, 'rpc', method } +end + +-- New curated public capability surface. +local function t_cap_meta(class, id) + return { 'cap', class, id, 'meta' } +end + +local function t_cap_status(class, id) + return { 'cap', class, id, 'status' } +end + +---------------------------------------------------------------------- +-- Raw host capability surface +---------------------------------------------------------------------- + +local function t_raw_host_cap_meta(source, class, id) + return { 'raw', 'host', source, 'cap', class, id, 'meta' } +end + +local function t_raw_host_cap_status(source, class, id) + return { 'raw', 'host', source, 'cap', class, id, 'status' } +end + +local function t_raw_host_cap_state(source, class, id, state) + return { 'raw', 'host', source, 'cap', class, id, 'state', state } +end + +local function t_raw_host_cap_event(source, class, id, event) + return { 'raw', 'host', source, 'cap', class, id, 'event', event } +end + +local function t_raw_host_cap_control(source, class, id, method) + return { 'raw', 'host', source, 'cap', class, id, 'rpc', method } +end + +---------------------------------------------------------------------- +-- Raw member capability surface +---------------------------------------------------------------------- + +local function t_raw_member_cap_meta(source, class, id) + return { 'raw', 'member', source, 'cap', class, id, 'meta' } +end + +local function t_raw_member_cap_status(source, class, id) + return { 'raw', 'member', source, 'cap', class, id, 'status' } +end + +local function t_raw_member_cap_state(source, class, id, state) + return { 'raw', 'member', source, 'cap', class, id, 'state', state } +end + +local function t_raw_member_cap_event(source, class, id, event) + return { 'raw', 'member', source, 'cap', class, id, 'event', event } +end + +local function t_raw_member_cap_control(source, class, id, method) + return { 'raw', 'member', source, 'cap', class, id, 'rpc', method } +end + +---------------------------------------------------------------------- +-- Validation helpers +---------------------------------------------------------------------- + +local function assert_source(source, level) + if type(source) ~= 'string' or source == '' then + error('source must be a non-empty string', (level or 1) + 1) + end + return source +end + +---------------------------------------------------------------------- +-- Topic resolution helpers +---------------------------------------------------------------------- + +local function control_topic(self, method) + if self.raw_kind == 'host' then + return t_raw_host_cap_control(self.source, self.class, self.id, method) + elseif self.raw_kind == 'member' then + return t_raw_member_cap_control(self.source, self.class, self.id, method) + else + return t_cap_control(self.class, self.id, method) + end +end + +local function state_topic(self, field) + if self.raw_kind == 'host' then + return t_raw_host_cap_state(self.source, self.class, self.id, field) + elseif self.raw_kind == 'member' then + return t_raw_member_cap_state(self.source, self.class, self.id, field) + else + return t_cap_state(self.class, self.id, field) + end +end + +local function event_topic(self, name) + if self.raw_kind == 'host' then + return t_raw_host_cap_event(self.source, self.class, self.id, name) + elseif self.raw_kind == 'member' then + return t_raw_member_cap_event(self.source, self.class, self.id, name) + else + return t_cap_event(self.class, self.id, name) + end +end + +local function meta_topic(self) + if self.raw_kind == 'host' then + return t_raw_host_cap_meta(self.source, self.class, self.id) + elseif self.raw_kind == 'member' then + return t_raw_member_cap_meta(self.source, self.class, self.id) + else + return t_cap_meta(self.class, self.id) + end +end + +local function status_topic(self) + if self.raw_kind == 'host' then + return t_raw_host_cap_status(self.source, self.class, self.id) + elseif self.raw_kind == 'member' then + return t_raw_member_cap_status(self.source, self.class, self.id) + else + return t_cap_status(self.class, self.id) + end +end + +---------------------------------------------------------------------- +-- Core op-only lane +-- All non-legacy constructors return these types. +---------------------------------------------------------------------- + +---@class CoreCapabilityReference +---@field conn Connection +---@field class CapabilityClass +---@field id CapabilityId +---@field raw_kind '"host"'|'"member"'|nil +---@field source string|nil +local CoreCapabilityReference = {} +CoreCapabilityReference.__index = CoreCapabilityReference + +function CoreCapabilityReference:call_control_op(method, args, opts) + local out = {} + for k, v in pairs(opts or {}) do out[k] = v end + if out.timeout == nil and out.deadline == nil then out.timeout = false end + return self.conn:call_op(control_topic(self, method), args, out) +end + +---@param field string +---@param opts table? +---@return Subscription +function CoreCapabilityReference:get_state_sub(field, opts) + return self.conn:subscribe(state_topic(self, field), opts) +end + +---@param name string +---@param opts table? +---@return Subscription +function CoreCapabilityReference:get_event_sub(name, opts) + return self.conn:subscribe(event_topic(self, name), opts) +end + +---@param opts table? +---@return Subscription +function CoreCapabilityReference:get_meta_sub(opts) + return self.conn:subscribe(meta_topic(self), opts) +end + +---@param opts table? +---@return Subscription +function CoreCapabilityReference:get_status_sub(opts) + return self.conn:subscribe(status_topic(self), opts) +end + +---------------------------------------------------------------------- +-- Legacy compatibility lane +-- Sync convenience wrappers live here, and only here. +---------------------------------------------------------------------- + +---@class LegacyCapabilityReference : CoreCapabilityReference +local LegacyCapabilityReference = setmetatable({}, { __index = CoreCapabilityReference }) +LegacyCapabilityReference.__index = LegacyCapabilityReference + +---@param method string +---@param args any +---@return Reply? +---@return string error +function LegacyCapabilityReference:call_control(method, args) + return perform(self:call_control_op(method, args)) +end + +---------------------------------------------------------------------- +-- Listener helpers +---------------------------------------------------------------------- + +local function new_core_cap_ref(conn, class, id, raw_kind, source) + return setmetatable({ + conn = conn, + class = class, + id = id, + raw_kind = raw_kind, + source = source, + }, CoreCapabilityReference) +end + +local function new_legacy_cap_ref(conn, class, id) + return setmetatable({ + conn = conn, + class = class, + id = id, + }, LegacyCapabilityReference) +end + +local function capability_ref_for_listener(self, class, id) + if self.mode == 'legacy-public' then + return new_legacy_cap_ref(self.conn, class, id) + elseif self.mode == 'raw-host' then + assert(self.source, 'raw-host listener missing source') + return new_core_cap_ref(self.conn, class, id, 'host', self.source) + elseif self.mode == 'raw-member' then + assert(self.source, 'raw-member listener missing source') + return new_core_cap_ref(self.conn, class, id, 'member', self.source) + else + return new_core_cap_ref(self.conn, class, id) + end +end + +local function listener_payload_is_ready(self, payload) + if self.mode == 'legacy-public' then + return payload == 'added' + end + + if type(payload) ~= 'table' then + return false + end + + if self.mode == 'curated-public' + or self.mode == 'raw-host' + or self.mode == 'raw-member' then + return payload.state == 'available' + or payload.available == true + or payload.state == 'running' + end + + return false +end + +local function listener_extract_class_and_id(self, msg) + if self.mode == 'raw-host' or self.mode == 'raw-member' then + return msg.topic[5], msg.topic[6] + end + return msg.topic[2], msg.topic[3] +end + +local function noop_token() + return { + unlink = function () + return false + end, + } +end + +---------------------------------------------------------------------- +-- Core op-only listener +---------------------------------------------------------------------- + +---@class CoreCapListener +---@field conn Connection +---@field sub Subscription +---@field topic Topic +---@field mode '"legacy-public"'|'"curated-public"'|'"raw-host"'|'"raw-member"' +---@field source string|nil +local CoreCapListener = {} +CoreCapListener.__index = CoreCapListener + +--- Wait for a capability to become ready and return a capability reference. +--- +--- This uses wait.waitable2(...) so the wait remains properly op-first: +--- * probe_step is pure +--- * run_step drains already-buffered messages +--- * mailbox wakeup is provided by rx:on_message(...) +--- +--- Note: this relies on Subscription._rx exposing the underlying mailbox +--- receiver. That coupling is intentionally boxed inside the SDK. +---@return Op +function CoreCapListener:wait_for_cap_op() + local sub = assert(self.sub, 'cap listener has no subscription') + local rx = assert(sub._rx, 'CoreCapListener requires mailbox-backed subscription') + assert(type(rx.recv_op) == 'function', 'CoreCapListener requires MailboxRx:recv_op()') + assert(type(rx.on_message) == 'function', 'CoreCapListener requires MailboxRx:on_message()') + + local function probe_step() + return false, 'capmsg' + end + + local function run_step() + while true do + local recv_op = rx:recv_op() + local ready, msg = recv_op.try_fn() + + if not ready then + return false, 'capmsg' + end + + if msg == nil then + return true, nil, tostring(rx:why() or 'subscription closed') + end + + if listener_payload_is_ready(self, msg.payload) then + local class, id = listener_extract_class_and_id(self, msg) + return true, capability_ref_for_listener(self, class, id), '' + end + end + end + + local function register(task, waker, want) + if want == 'run' then + waker:wakeup(task) + return noop_token() + end + return rx:on_message(task, waker) + end + + return wait.waitable2(register, probe_step, run_step) +end + +function CoreCapListener:close() + self.sub:unsubscribe() +end + +function CoreCapListener:terminate(_) + self:close() + return true, nil +end + +function CoreCapListener:close_on_scope(scope) + assert(scope ~= nil, 'CoreCapListener:close_on_scope requires scope') + return scope:finally(function () + self:terminate('scope closed') + end) +end + +---------------------------------------------------------------------- +-- Legacy compatibility listener +---------------------------------------------------------------------- + +---@class LegacyCapListener : CoreCapListener +local LegacyCapListener = setmetatable({}, { __index = CoreCapListener }) +LegacyCapListener.__index = LegacyCapListener + +---@param opts { timeout?: number }|nil +---@return LegacyCapabilityReference|CoreCapabilityReference|nil +---@return string error +function LegacyCapListener:wait_for_cap(opts) + opts = opts or {} + + local ops = { + cap = self:wait_for_cap_op(), + } + + if opts.timeout then + ops.timeout = sleep.sleep_op(opts.timeout) + end + + local which, a, b = perform(op.named_choice(ops)) + if which == 'cap' then + return a, b + elseif which == 'timeout' then + return nil, 'timeout' + end + + return nil, 'unknown error' +end + +---------------------------------------------------------------------- +-- SDK +-- +-- Constructor boundary: +-- * legacy constructors return sync-capable compatibility types +-- * all other constructors return strict op-only types +---------------------------------------------------------------------- + +---@class CapSDK +local CapSDK = { + args = cap_args, +} +CapSDK.__index = CapSDK + +---------------------------------------------------------------------- +-- Legacy public helpers +-- For unchanged config/system callers. +---------------------------------------------------------------------- + +---@param conn Connection +---@param class CapabilityClass +---@param id CapabilityId +---@return LegacyCapListener +function CapSDK.new_cap_listener(conn, class, id) + local topic = t_cap_listen(class, id) + local sub = conn:subscribe(topic) + return setmetatable({ + conn = conn, + sub = sub, + topic = topic, + mode = 'legacy-public', + }, LegacyCapListener) +end + +---@param conn Connection +---@param class CapabilityClass +---@param id CapabilityId +---@return LegacyCapabilityReference +function CapSDK.new_cap_ref(conn, class, id) + return new_legacy_cap_ref(conn, class, id) +end + +---------------------------------------------------------------------- +-- New curated public helpers +-- Op-only. +---------------------------------------------------------------------- + +---@param conn Connection +---@param class CapabilityClass +---@param id CapabilityId +---@return CoreCapListener +function CapSDK.new_curated_cap_listener(conn, class, id) + local topic = t_cap_status(class, id) + local sub = conn:subscribe(topic) + return setmetatable({ + conn = conn, + sub = sub, + topic = topic, + mode = 'curated-public', + }, CoreCapListener) +end + +---@param conn Connection +---@param class CapabilityClass +---@param id CapabilityId +---@return CoreCapabilityReference +function CapSDK.new_curated_cap_ref(conn, class, id) + return new_core_cap_ref(conn, class, id) +end + +---------------------------------------------------------------------- +-- New raw host helpers +-- Op-only. +---------------------------------------------------------------------- + +---@param conn Connection +---@param source string +---@param class CapabilityClass +---@param id CapabilityId +---@return CoreCapListener +function CapSDK.new_raw_host_cap_listener(conn, source, class, id) + source = assert_source(source, 1) + local topic = t_raw_host_cap_status(source, class, id) + local sub = conn:subscribe(topic) + return setmetatable({ + conn = conn, + sub = sub, + topic = topic, + mode = 'raw-host', + source = source, + }, CoreCapListener) +end + +---@param conn Connection +---@param source string +---@param class CapabilityClass +---@param id CapabilityId +---@return CoreCapabilityReference +function CapSDK.new_raw_host_cap_ref(conn, source, class, id) + source = assert_source(source, 1) + return new_core_cap_ref(conn, class, id, 'host', source) +end + +---------------------------------------------------------------------- +-- New raw member helpers +-- Op-only. +---------------------------------------------------------------------- + +---@param conn Connection +---@param source string +---@param class CapabilityClass +---@param id CapabilityId +---@return CoreCapListener +function CapSDK.new_raw_member_cap_listener(conn, source, class, id) + source = assert_source(source, 1) + local topic = t_raw_member_cap_status(source, class, id) + local sub = conn:subscribe(topic) + return setmetatable({ + conn = conn, + sub = sub, + topic = topic, + mode = 'raw-member', + source = source, + }, CoreCapListener) +end + +---@param conn Connection +---@param source string +---@param class CapabilityClass +---@param id CapabilityId +---@return CoreCapabilityReference +function CapSDK.new_raw_member_cap_ref(conn, source, class, id) + source = assert_source(source, 1) + return new_core_cap_ref(conn, class, id, 'member', source) +end + +return CapSDK diff --git a/src/services/hal/structures.lua b/src/services/hal/structures.lua deleted file mode 100644 index d366a7c9..00000000 --- a/src/services/hal/structures.lua +++ /dev/null @@ -1,80 +0,0 @@ ----@alias event_key [string, number] ----@class EventList A list of get operations ----@field events BaseOp[] ----@field event_ids event_key[] -local EventList = {} -EventList.__index = EventList ----Creates an empty EventList ----@return EventList -function EventList.new() - local self = setmetatable({}, EventList) - self.events = {} - self.event_ids = {} - return self -end - ----Adds an operation to the event list ----@param event_id event_key ----@param event_op BaseOp -function EventList:add(event_id, event_op) - -- add event to events list - table.insert(self.events, event_op) - table.insert(self.event_ids, event_id) -end - ----removes an operation from the event list ----@param event_id event_key -function EventList:remove(event_id) - local i = 1 - while i <= #self.events do - if self.event_ids[i][1] == event_id[1] and self.event_ids[i][2] == event_id[2] then - table.remove(self.events, i) - table.remove(self.event_ids, i) - else - i = i + 1 - end - end -end - ----get all operations ----@return BaseOp[] -function EventList:get_events() - return self.events -end - -Tracker = {} -Tracker.__index = Tracker - -function Tracker.new() - local self = setmetatable({}, Tracker) - self.current_idx = 1 - self.devices = {} - return self -end - -function Tracker:get(index) - return self.devices[index] -end - -function Tracker:add(device) - local dev_idx = self.current_idx - self.devices[dev_idx] = device - self.current_idx = self.current_idx + 1 - return dev_idx -end - -function Tracker:next_index() - return self.current_idx -end -function Tracker:remove(index) - if self.devices[index] ~= nil then - self.devices[index] = nil - return true - end - return false -end - -return { - new_event_list = EventList.new, - new_tracker = Tracker.new, -} diff --git a/src/services/hal/support/control_loop.lua b/src/services/hal/support/control_loop.lua new file mode 100644 index 00000000..171f088d --- /dev/null +++ b/src/services/hal/support/control_loop.lua @@ -0,0 +1,264 @@ +local fibers = require 'fibers' +local op = require 'fibers.op' +local scope_mod = require 'fibers.scope' + +local hal_types = require 'services.hal.types.core' + +local perform = fibers.perform + +local M = {} + +local function elapsed_ms(t0) + if not t0 then return nil end + return math.floor(((fibers.now() - t0) * 1000) + 0.5) +end + +local function dlog(logger, level, payload) + if logger and logger[level] then + logger[level](logger, payload) + end +end + +local function err_text(v) + if type(v) == 'table' then + return tostring(v.err or v.reason or v.message or v.code or 'structured_error') + end + if v ~= nil then return tostring(v) end + return nil +end + +local function method_cancel_policy(methods, verb) + local policies = type(methods) == 'table' and rawget(methods, '__cancel_policy') or nil + if type(policies) == 'table' then return policies[verb] end + return nil +end + +local function caller_abandoned(reason) + return reason ~= nil and reason ~= false +end + +---@param ev any +---@param verb string +---@return Op +local function assert_op_result(ev, verb) + if type(ev) ~= 'table' or getmetatable(ev) ~= op.Op then + error( + ('control method %q must return an Op, got %s (%s)'):format( + tostring(verb), + type(ev), + tostring(ev) + ), + 3 + ) + end + return ev +end + +--- Evaluate a control request against an op-returning method table. +--- +--- Contract: +--- methods[verb](opts, request) -> Op +--- +--- The returned Op must resolve to: +--- ok:boolean, value_or_err:any +--- +---@param methods table +---@param request ControlRequest +---@return Op +function M.evaluate_request_op(methods, request) + return op.guard(function () + local fn = methods[request.verb] + if type(fn) ~= 'function' then + return op.always(false, 'unsupported verb: ' .. tostring(request.verb)) + end + + return assert_op_result(fn(request.opts, request), tostring(request.verb)) + end) +end + +--- Reply to a control request via its reply channel. +--- +--- Returns: +--- true, nil on successful delivery +--- false, reason on validation/delivery failure +--- +---@param reply_ch Channel +---@param ok boolean +---@param value_or_err any +---@return Op +function M.reply_op(reply_ch, ok, value_or_err) + return op.guard(function () + local reply, err = hal_types.new.Reply(ok, value_or_err) + if not reply then + return op.always(false, 'invalid reply: ' .. tostring(err)) + end + + return reply_ch:put_op(reply):wrap(function (sent, send_err) + if sent ~= false then + return true, nil + end + return false, tostring(send_err or 'reply delivery failed') + end) + end) +end + +--- Canonical shell loop for HAL request/reply control channels. +--- +--- This is intentionally the semantic boundary where blocking policy is visible: +--- * wait for the next request or scope shutdown/failure +--- * perform exactly one request Op +--- * perform exactly one reply Op +--- +--- Request handlers must return Ops and must not hide blocking behind +--- immediate helper methods. +--- +---@param ch Channel +---@param methods table +---@param logger table|nil +---@param what string|nil +function M.run_request_loop(ch, methods, logger, what) + local scope = scope_mod.current() + + scope:finally(function (_, status, primary) + dlog(logger, 'debug', { + what = tostring(what or 'control_loop') .. '_exiting', + status = tostring(status), + primary = primary and tostring(primary) or nil, + }) + end) + + while true do + local request, req_err = perform(ch:get_op()) + if not request then + dlog(logger, 'debug', { + what = tostring(what or 'control_loop') .. '_closed', + err = tostring(req_err or 'control channel closed'), + }) + return + end + + local loop_name = tostring(what or 'control_loop') + local trace_control = loop_name:match('^network_') ~= nil + local req_t0 = fibers.now() + if trace_control then + dlog(logger, 'debug', { + what = loop_name .. '_request_begin', + verb = tostring(request.verb), + }) + end + + local ok, value_or_err + local caller_cancelled = false + local cancel_policy = method_cancel_policy(methods, request.verb) + if request.cancel_op ~= nil and cancel_policy ~= 'detach_after_admission' then + local which, a, b = perform(op.named_choice({ + work = M.evaluate_request_op(methods, request), + cancel = request.cancel_op, + })) + if which == 'cancel' and caller_abandoned(a) then + caller_cancelled = true + dlog(logger, 'debug', { + what = loop_name .. '_request_cancelled', + verb = tostring(request.verb), + reason = tostring(a), + elapsed_ms = elapsed_ms(req_t0), + }) + else + ok, value_or_err = a, b + if trace_control then + local method_level = ok == true and 'debug' or 'warn' + if loop_name == 'network_config' and (request.verb == 'apply' or request.verb == 'apply_live_weights') then method_level = 'debug' end + dlog(logger, method_level, { + what = loop_name .. '_method_done', + verb = tostring(request.verb), + ok = ok == true, + err = ok == true and nil or err_text(value_or_err), + elapsed_ms = elapsed_ms(req_t0), + }) + end + end + else + ok, value_or_err = perform(M.evaluate_request_op(methods, request)) + if trace_control then + local method_level = ok == true and 'debug' or 'warn' + if loop_name == 'network_config' and (request.verb == 'apply' or request.verb == 'apply_live_weights') then method_level = 'debug' end + dlog(logger, method_level, { + what = loop_name .. '_method_done', + verb = tostring(request.verb), + ok = ok == true, + err = ok == true and nil or err_text(value_or_err), + elapsed_ms = elapsed_ms(req_t0), + }) + end + end + + if not caller_cancelled then + local reply_t0 = fibers.now() + local replied, reply_err + if request.cancel_op ~= nil then + -- Detach-protected operations are deliberately allowed to drain after + -- admission. If the caller abandoned while the operation was running, + -- prefer recording detachment over delivering a late reply. Do not race + -- reply and cancellation here: both may be immediately ready and + -- named_choice has no semantic priority. + if cancel_policy == 'detach_after_admission' then + local pre_reply_cancel = perform(request.cancel_op:or_else(function () return nil end)) + if caller_abandoned(pre_reply_cancel) then + caller_cancelled = true + if trace_control then + dlog(logger, 'debug', { + what = loop_name .. '_request_detached', + verb = tostring(request.verb), + reason = tostring(pre_reply_cancel), + admitted = true, + elapsed_ms = elapsed_ms(req_t0), + }) + end + else + replied, reply_err = perform(M.reply_op(request.reply_ch, ok, value_or_err):or_else(function () + return false, 'reply_not_ready' + end)) + end + else + local which, a, b = perform(op.named_choice({ + reply = M.reply_op(request.reply_ch, ok, value_or_err), + cancel = request.cancel_op, + })) + if which == 'cancel' and caller_abandoned(a) then + caller_cancelled = true + if trace_control then + dlog(logger, 'debug', { + what = loop_name .. '_reply_cancelled', + verb = tostring(request.verb), + reason = tostring(a), + elapsed_ms = elapsed_ms(req_t0), + }) + end + else + replied, reply_err = a, b + end + end + else + replied, reply_err = perform(M.reply_op(request.reply_ch, ok, value_or_err)) + end + if not caller_cancelled and not replied then + dlog(logger, 'warn', { + what = loop_name .. '_reply_failed', + verb = tostring(request.verb), + err = tostring(reply_err), + elapsed_ms = elapsed_ms(req_t0), + reply_elapsed_ms = elapsed_ms(reply_t0), + }) + elseif trace_control and not caller_cancelled then + dlog(logger, 'debug', { + what = loop_name .. '_reply_done', + verb = tostring(request.verb), + elapsed_ms = elapsed_ms(req_t0), + reply_elapsed_ms = elapsed_ms(reply_t0), + }) + end + end + end +end + +return M diff --git a/src/services/hal/support/device_events.lua b/src/services/hal/support/device_events.lua new file mode 100644 index 00000000..5358ae98 --- /dev/null +++ b/src/services/hal/support/device_events.lua @@ -0,0 +1,41 @@ +-- services/hal/support/device_events.lua +-- +-- Shared HAL DeviceEvent construction and emission helpers. + +local op = require 'fibers.op' +local hal_types = require 'services.hal.types.core' + +local M = {} + +function M.emit_op(dev_ev_ch, action, class, id, metadata, caps) + return op.guard(function () + if type(dev_ev_ch) ~= 'table' or type(dev_ev_ch.put_op) ~= 'function' then + return op.always(false, 'device event channel missing') + end + + local ev, err = hal_types.new.DeviceEvent( + action, + class, + id, + metadata or {}, + caps or {} + ) + if not ev then + return op.always(false, tostring(err)) + end + + return dev_ev_ch:put_op(ev):wrap(function () + return true, nil + end) + end) +end + +function M.added_op(dev_ev_ch, class, id, metadata, caps) + return M.emit_op(dev_ev_ch, 'added', class, id, metadata, caps) +end + +function M.removed_op(dev_ev_ch, class, id, metadata) + return M.emit_op(dev_ev_ch, 'removed', class, id, metadata or {}, {}) +end + +return M diff --git a/src/services/hal/support/driver_reconcile.lua b/src/services/hal/support/driver_reconcile.lua new file mode 100644 index 00000000..8ff173bb --- /dev/null +++ b/src/services/hal/support/driver_reconcile.lua @@ -0,0 +1,61 @@ +-- services/hal/support/driver_reconcile.lua +-- +-- Generic sequential reconcile helper for strict HAL managers. The caller owns +-- driver construction, event emission and shutdown semantics; this helper owns +-- only the repeated desired/current traversal. + +local fibers = require 'fibers' + +local M = {} + +function M.desired_by_id(entries, id_of) + local out = {} + id_of = id_of or function (entry) return entry and entry.id end + for _, entry in ipairs(entries or {}) do + out[id_of(entry)] = entry + end + return out +end + +function M.reconcile_op(spec) + return fibers.run_scope_op(function () + if type(spec) ~= 'table' then return false, 'driver_reconcile: spec table required' end + local current = spec.current + local entries = spec.entries + local desired = spec.desired + local id_of = spec.id_of or function (entry) return entry and entry.id end + local same = spec.same + local stop = spec.stop + local start = spec.start + if type(current) ~= 'table' then return false, 'driver_reconcile: current table required' end + if desired == nil and type(entries) ~= 'table' then return false, 'driver_reconcile: entries table required' end + if desired ~= nil and type(desired) ~= 'table' then return false, 'driver_reconcile: desired table required' end + if type(same) ~= 'function' then return false, 'driver_reconcile: same function required' end + if type(stop) ~= 'function' then return false, 'driver_reconcile: stop function required' end + if type(start) ~= 'function' then return false, 'driver_reconcile: start function required' end + + desired = desired or M.desired_by_id(entries, id_of) + + for id, driver in pairs(current) do + local want = desired[id] + if want == nil or not same(driver, want) then + local ok_stop, stop_err = fibers.perform(stop(id, driver)) + if not ok_stop then return false, stop_err end + end + end + + for id, entry in pairs(desired) do + if not current[id] then + local ok_start, start_err = fibers.perform(start(id, entry)) + if not ok_start then return false, start_err end + end + end + + return true, nil + end):wrap(function (st, rep, ok, err) + if st ~= 'ok' then return false, tostring(err or rep) end + return ok, err + end) +end + +return M diff --git a/src/services/hal/support/strict_manager.lua b/src/services/hal/support/strict_manager.lua new file mode 100644 index 00000000..7e59205a --- /dev/null +++ b/src/services/hal/support/strict_manager.lua @@ -0,0 +1,46 @@ +-- services/hal/support/strict_manager.lua +-- +-- Small shared helpers for callback-free HAL managers exposing the strict +-- op-only manager surface. + +local op = require 'fibers.op' +local resource = require 'devicecode.support.resource' + +local M = {} + +function M.api_table(extra) + local out = { api_mode = 'op_only' } + for k, v in pairs(extra or {}) do out[k] = v end + return out +end + +function M.terminate_drivers(drivers, reason, label) + for _, driver in pairs(drivers or {}) do + resource.terminate_checked(driver, reason or 'manager finalised', label or 'HAL manager driver cleanup failed') + end + return true, nil +end + +function M.fault_op_for_state(state) + if state and state.scope and state.started then + return state.scope:fault_op() + end + return op.never() +end + +function M.finaliser(state, generation, opts) + opts = opts or {} + return function (scope) + if state.scope ~= scope or state.generation ~= generation then return end + M.terminate_drivers(state.drivers, opts.reason or 'manager finalised', opts.label) + state.started = false + state.scope = nil + state.logger = nil + state.dev_ev_ch = nil + state.cap_emit_ch = nil + state.cfg_ch = nil + state.drivers = {} + end +end + +return M diff --git a/src/services/hal/sysinfo.lua b/src/services/hal/sysinfo.lua deleted file mode 100644 index 11b5f0c9..00000000 --- a/src/services/hal/sysinfo.lua +++ /dev/null @@ -1,194 +0,0 @@ -local file = require "fibers.stream.file" -local sleep = require "fibers.sleep" -local op = require "fibers.op" -local exec = require "fibers.exec" -local utils = require "services.hal.utils" -local bit = rawget(_G, "bit") or require "bit32" - ----@return string? ----@return string? Error -local function get_cpu_info(_) - local cpuinfo, cpuinfo_err = utils.read_file("/proc/cpuinfo") - if cpuinfo_err or not cpuinfo then return nil, cpuinfo_err end - local model - - if cpuinfo:match("Qualcomm Atheros") then - model = cpuinfo:match("cpu model%s+:%s+(.+)\n") - elseif cpuinfo:match("Raspberry Pi") then - model = "Raspberry Pi 4 Model B" - else - model = "Unknown" - end - - return model, nil -end - ----@param ctx Context ----@return table? cpu_utilisation_and_freq ----@return string? Error -local function get_cpu_utilisation_and_freq(ctx) - local function extract_cpu_times(stat, core) - local pattern = core .. "%s+(%d+)%s+(%d+)%s+(%d+)%s+(%d+)" - local user, nice, system, idle = stat:match(pattern) - return tonumber(user), tonumber(nice), tonumber(system), tonumber(idle) - end - - local function compute_utilisation(user_prev, nice_prev, system_prev, idle_prev, user_curr, nice_curr, system_curr, idle_curr) - local total_prev = user_prev + nice_prev + system_prev + idle_prev - local total_curr = user_curr + nice_curr + system_curr + idle_curr - local active_diff = (user_curr - user_prev) + (nice_curr - nice_prev) + (system_curr - system_prev) - local total_diff = total_curr - total_prev - return total_diff == 0 and 0 or (active_diff / total_diff) * 100 - end - - local function get_scaling_cur_freq(core) - local path = "/sys/devices/system/cpu/" .. core .. "/cpufreq/scaling_cur_freq" - local freq, read_err = utils.read_file(path) - if not freq or read_err then return nil, read_err end - return tonumber(freq), nil - end - - local stat_prev, prev_err = utils.read_file("/proc/stat") - if prev_err then return nil, prev_err end - local ctx_err = op.choice( - sleep.sleep_op(1), - ctx:done_op():wrap(function () - return ctx:err() - end) - ):perform() - if ctx_err then return nil, ctx_err end - local stat_curr, curr_err = utils.read_file("/proc/stat") - if curr_err then return nil, curr_err end - - local core_utilisations = {} - local core_frequencies = {} - local core_id = 0 - local overall_utilisation_sum = 0 - local overall_freq_sum = 0 - - while true do - local core = "cpu" .. core_id - local user_prev, nice_prev, system_prev, idle_prev = extract_cpu_times(stat_prev, core) - if not user_prev then - break - end - local user_curr, nice_curr, system_curr, idle_curr = extract_cpu_times(stat_curr, core) - local utilisation = compute_utilisation(user_prev, nice_prev, system_prev, idle_prev, user_curr, nice_curr, system_curr, idle_curr) - core_utilisations[core] = utilisation - overall_utilisation_sum = overall_utilisation_sum + utilisation - - local freq = get_scaling_cur_freq(core) - if freq then - core_frequencies[core] = freq - overall_freq_sum = overall_freq_sum + freq - end - - core_id = core_id + 1 - end - - local overall_utilisation = overall_utilisation_sum / (core_id > 0 and core_id or 1) - local average_frequency = overall_freq_sum / (core_id > 0 and core_id or 1) - - return { - overall_utilisation = overall_utilisation, - core_utilisations = core_utilisations, - average_frequency = average_frequency, - core_frequencies = core_frequencies - } -end - ---- Get total, used, and free RAM ----@return table? ram_info ----@return string? Error -local function get_ram_info(_) - local meminfo, err = utils.read_file("/proc/meminfo") - if not meminfo or err then return nil, err end - local total = meminfo:match("MemTotal:%s*(%d+)") or 0 - local free = meminfo:match("MemFree:%s*(%d+)") or 0 - local buffers = meminfo:match("Buffers:%s*(%d+)") or 0 - local cached = meminfo:match("Cached:%s*(%d+)") or 0 - - local used = total - (free + buffers + cached) - return { - total = tonumber(total), - used = tonumber(used), - free = tonumber(free) + tonumber(buffers) + tonumber(cached) - }, nil -end - ----Gets the modem and version of hardware ----@return string? model ----@return string? version ----@return string? error -local function get_hw_revision(_) - local revision, err = utils.read_file("/etc/hwrevision") - if err or not revision then return nil, nil, err end - return revision, nil -end - ----@return string? version ----@return string? error -local function get_fw_version(_) - local version, err = utils.read_file("/etc/fwversion") - if err or not version then return nil, err end - return version, nil -end - -local function get_serial(_) - local serial, err = utils.read_file("/data/serial") - if err or not serial then return nil, err end - return serial, nil -end - -local function get_temperature(_) - local temperature, err = utils.read_file("/sys/class/thermal/thermal_zone0/temp") - if err or not temperature then return nil, err end - return tonumber(temperature) / 1000, nil -end - -local function get_uptime(_) - local uptime, err = utils.read_file("/proc/uptime") - if err or not uptime then return nil, err end - local up = string.match(uptime, "(%S+)%s") - if not up then return nil, "Failed to parse uptime" end - return tonumber(up), nil -end - -local function get_board_revision(_) - local board_revision_data, err = exec.command("fw_printenv"):output() - if err then - return nil, err - end - local board_revision = string.match(board_revision_data, "board_revision=([^\n]+)") - return board_revision, nil -end - -local function get_power_state(_) - local throttled, err = utils.read_file("/sys/devices/platform/soc/soc:firmware/get_throttled") - if err or not throttled then return nil, nil end - local hex_str = throttled:match("0x(%x+)") or throttled:match("(%x+)") - if not hex_str then return nil, "Failed to parse throttled state" end - local value = tonumber(hex_str, 16) - if not value then return nil, "Failed to parse throttled state" end - - return { - under_voltage_detected = bit.band(value, 0x1) ~= 0, - arm_frequency_capped = bit.band(value, 0x2) ~= 0, - currently_throttled = bit.band(value, 0x4) ~= 0, - under_voltage_occurred = bit.band(value, 0x10000) ~= 0, - raw = value - }, nil -end - -return { - get_hw_revision = get_hw_revision, - get_fw_version = get_fw_version, - get_cpu_model = get_cpu_info, - get_cpu_utilisation_and_freq = get_cpu_utilisation_and_freq, - get_ram_info = get_ram_info, - get_serial = get_serial, - get_temperature = get_temperature, - get_uptime = get_uptime, - get_board_revision = get_board_revision, - get_power_state = get_power_state -} diff --git a/src/services/hal/templates/driver_template.lua b/src/services/hal/templates/driver_template.lua new file mode 100644 index 00000000..512cf92e --- /dev/null +++ b/src/services/hal/templates/driver_template.lua @@ -0,0 +1,254 @@ +-- services/hal/templates/driver_template.lua +-- +-- Strict HAL driver template. +-- +-- New drivers should expose Op-returning operations, graceful component +-- shutdown through shutdown_op(), and immediate finaliser-safe cleanup through +-- terminate(reason). Ordinary resources owned by a driver should use +-- close_op() for graceful close and terminate(reason) for finaliser cleanup. + +local hal_types = require "services.hal.types.core" +local cap_types = require "services.hal.types.capabilities" + +local fibers = require "fibers" +local op = require "fibers.op" +local channel = require "fibers.channel" +local sleep = require "fibers.sleep" +local safe = require "coxpcall" + +---@class TemplateDriver +---@field id string +---@field scope Scope|nil +---@field control_ch Channel +---@field cap_emit_ch Channel|nil +---@field initialised boolean +---@field caps_applied boolean +---@field started boolean +---@field log Logger +local TemplateDriver = {} +TemplateDriver.__index = TemplateDriver + +local CONTROL_Q_LEN = 8 +local DEFAULT_SHUTDOWN_TIMEOUT = 5.0 + +local function return_error(err, code) + return false, err or "unknown error", code +end + +local function emit_op(emit_ch, class, id, mode, key, data) + return op.guard(function () + local payload, err = hal_types.new.Emit(class, id, mode, key, data) + if not payload then + return op.always(false, err) + end + + return emit_ch:put_op(payload):wrap(function (sent, send_err) + if sent ~= false and sent ~= nil then + return true, nil + end + return false, tostring(send_err or "emit channel closed") + end) + end) +end + +local function validate_fn(fn) + if type(fn) ~= "function" then + return false, "verb handler is unimplemented" + end + return true, nil +end + +function TemplateDriver:init_op() + return op.guard(function () + self.initialised = true + return op.always(true, nil) + end) +end + +function TemplateDriver:get_status_op(opts) + return op.guard(function () + if opts ~= nil and type(opts) ~= "table" then + return op.always(return_error("invalid options", 1)) + end + + return op.always(true, { + id = self.id, + state = "ready", + }) + end) +end + +function TemplateDriver:reset_op(opts) + return op.guard(function () + if opts ~= nil and type(opts) ~= "table" then + return op.always(return_error("invalid options", 1)) + end + + if self.cap_emit_ch then + return emit_op(self.cap_emit_ch, "template", self.id, "event", "reset", { + at = os.time(), + }) + end + + return op.always(true, nil) + end) +end + +local function control_loop(self, scope) + scope:finally(function () + self.log:debug({ what = "template_driver_control_loop_stopped", id = self.id }) + end) + + while true do + local request, b = fibers.perform(self.control_ch:get_op()) + if not request then + self.log:error({ what = "control_channel_read_failed", id = self.id, err = tostring(b) }) + return + end + + local fn = self[request.verb .. "_op"] + local valid, validation_err = validate_fn(fn) + + local ok, reason, code + if not valid then + ok, reason, code = false, validation_err, 1 + else + local call_ok, fn_ok, fn_reason, fn_code = safe.pcall(function () + return fibers.perform(fn(self, request.opts)) + end) + + if not call_ok then + ok, reason, code = false, tostring(fn_ok), 1 + else + ok, reason, code = fn_ok, fn_reason, fn_code + end + end + + local reply, reply_err = hal_types.new.Reply(ok, reason, code) + if not reply then + self.log:error({ what = "reply_create_failed", id = self.id, err = tostring(reply_err) }) + else + local sent, send_err = fibers.perform(request.reply_ch:put_op(reply)) + if sent == false or sent == nil then + self.log:error({ what = "reply_send_failed", id = self.id, err = tostring(send_err) }) + end + end + end +end + +function TemplateDriver:start_op(owner_scope) + return op.guard(function () + if self.started then + return op.always(false, "already started") + end + if not self.initialised then + return op.always(false, "driver not initialised") + end + if not self.caps_applied then + return op.always(false, "capabilities not applied") + end + + local scope, err = owner_scope:child() + if not scope then + return op.always(false, tostring(err)) + end + + self.scope = scope + + local ok, spawn_err = scope:spawn(function () + return control_loop(self, scope) + end) + if not ok then + self.scope = nil + scope:cancel(tostring(spawn_err or "template driver spawn failed")) + return op.always(false, tostring(spawn_err)) + end + + self.started = true + return op.always(true, nil) + end) +end + +function TemplateDriver:shutdown_op(timeout) + timeout = timeout or DEFAULT_SHUTDOWN_TIMEOUT + + return op.guard(function () + local scope = self.scope + if not self.started or not scope then + return op.always(true, nil) + end + + scope:cancel("template driver shutdown") + + return fibers.boolean_choice( + scope:join_op():wrap(function () + self.started = false + self.scope = nil + return true, nil + end), + sleep.sleep_op(timeout):wrap(function () + return false, "template driver shutdown timeout" + end) + ):wrap(function (completed, _a, b) + if completed then return true, nil end + return false, b + end) + end) +end + +function TemplateDriver:terminate(reason) + if self.scope then + self.scope:cancel(reason or "template driver terminated") + end + self.started = false + self.scope = nil + return true, nil +end + +function TemplateDriver:capabilities_op(emit_ch) + return op.guard(function () + if not self.initialised then + return op.always(false, "driver not initialised") + end + if self.caps_applied then + return op.always(false, "capabilities already applied") + end + + self.cap_emit_ch = emit_ch + + local cap, cap_err = cap_types.new.Capability( + "template", + self.id, + self.control_ch, + { "get_status", "reset" } + ) + if not cap then + return op.always(false, cap_err) + end + + self.caps_applied = true + return op.always(true, { cap }) + end) +end + +local function new(id, logger) + if type(id) ~= "string" or id == "" then + return nil, "invalid id" + end + + return setmetatable({ + id = id, + scope = nil, + control_ch = channel.new(CONTROL_Q_LEN), + cap_emit_ch = nil, + initialised = false, + caps_applied = false, + started = false, + log = logger, + }, TemplateDriver), nil +end + +return { + new = new, + Driver = TemplateDriver, +} diff --git a/src/services/hal/templates/manager_template.lua b/src/services/hal/templates/manager_template.lua new file mode 100644 index 00000000..739fd7d6 --- /dev/null +++ b/src/services/hal/templates/manager_template.lua @@ -0,0 +1,216 @@ +-- services/hal/templates/manager_template.lua +-- +-- Strict HAL manager template. +-- +-- New managers advertise api_mode = 'op_only'. HAL will call start_op(), +-- apply_config_op(), shutdown_op(), terminate(reason), and fault_op(). Legacy stop/stop_op +-- belong only on the compatibility side of hal.lua. + +local hal_types = require "services.hal.types.core" +local driver_template = require "services.hal.templates.driver_template" +local resource = require "devicecode.support.resource" + +local fibers = require "fibers" +local op = require "fibers.op" +local channel = require "fibers.channel" +local sleep = require "fibers.sleep" + +---@class TemplateManager +---@field api_mode string +---@field scope Scope|nil +---@field started boolean +---@field detect_ch Channel +---@field remove_ch Channel +---@field ready_driver_ch Channel +---@field drivers table +---@field log Logger|nil +local TemplateManager = { + api_mode = 'op_only', + scope = nil, + started = false, + detect_ch = channel.new(), + remove_ch = channel.new(), + ready_driver_ch = channel.new(), + drivers = {}, + log = nil, +} + +local DEFAULT_SHUTDOWN_TIMEOUT = 5.0 + +local function detector(scope) + scope:finally(function () + TemplateManager.log:debug({ what = "template_detector_stopped" }) + end) + + while true do + local which, payload = fibers.perform(fibers.named_choice{ + detect = TemplateManager.detect_ch:get_op(), + remove = TemplateManager.remove_ch:get_op(), + }) + + if which == "detect" then + local id = payload + local driver, err = driver_template.new(id, TemplateManager.log:child({ template = id })) + if not driver then + TemplateManager.log:error({ what = "template_driver_create_failed", id = id, err = tostring(err) }) + else + local ok, spawn_err = fibers.spawn(function () + local ok_init, init_err = fibers.perform(driver:init_op()) + if not ok_init then + TemplateManager.log:error({ what = "template_driver_init_failed", id = id, err = tostring(init_err) }) + return + end + fibers.perform(TemplateManager.ready_driver_ch:put_op(driver)) + end) + if not ok then + resource.terminate_checked(driver, tostring(spawn_err or "driver init spawn failed"), "template driver init cleanup failed") + end + end + elseif which == "remove" then + local id = payload + local driver = TemplateManager.drivers[id] + if driver then + TemplateManager.drivers[id] = nil + local ok, err = fibers.perform(driver:shutdown_op(DEFAULT_SHUTDOWN_TIMEOUT)) + if not ok then + resource.terminate_checked(driver, tostring(err or "driver shutdown failed"), "template driver shutdown cleanup failed") + end + end + end + end +end + +local function manager_loop(scope, dev_ev_ch, cap_emit_ch) + scope:finally(function () + TemplateManager.log:debug({ what = "template_manager_loop_stopped" }) + end) + + while true do + local driver = fibers.perform(TemplateManager.ready_driver_ch:get_op()) + if not driver then + return + end + + repeat + local ok_caps, caps_or_err = fibers.perform(driver:capabilities_op(cap_emit_ch)) + if not ok_caps then + TemplateManager.log:error({ what = "template_caps_failed", err = tostring(caps_or_err) }) + resource.terminate_checked(driver, "capabilities failed", "template driver capabilities cleanup failed") + break + end + + local ok_start, start_err = fibers.perform(driver:start_op(scope)) + if not ok_start then + TemplateManager.log:error({ what = "template_start_failed", err = tostring(start_err) }) + resource.terminate_checked(driver, "start failed", "template driver start cleanup failed") + break + end + + TemplateManager.drivers[driver.id] = driver + + local ev, ev_err = hal_types.new.DeviceEvent( + "added", + "template_device", + driver.id, + { source = "template" }, + caps_or_err + ) + if not ev then + TemplateManager.log:error({ what = "template_device_event_failed", err = tostring(ev_err) }) + break + end + + fibers.perform(dev_ev_ch:put_op(ev)) + until true + end +end + +function TemplateManager.start_op(logger, dev_ev_ch, cap_emit_ch) + return op.guard(function () + if TemplateManager.started then + return op.always(false, "already started") + end + + local owner_scope = fibers.current_scope() + local scope, err = owner_scope:child() + if not scope then + return op.always(false, tostring(err)) + end + + TemplateManager.log = logger + TemplateManager.scope = scope + + scope:finally(function () + TemplateManager.terminate("template manager scope finalised") + end) + + local ok1, err1 = scope:spawn(detector) + if not ok1 then + TemplateManager.terminate(tostring(err1 or "detector spawn failed")) + return op.always(false, tostring(err1)) + end + + local ok2, err2 = scope:spawn(manager_loop, dev_ev_ch, cap_emit_ch) + if not ok2 then + TemplateManager.terminate(tostring(err2 or "manager loop spawn failed")) + return op.always(false, tostring(err2)) + end + + TemplateManager.started = true + return op.always(true, nil) + end) +end + +function TemplateManager.shutdown_op(timeout) + timeout = timeout or DEFAULT_SHUTDOWN_TIMEOUT + + return op.guard(function () + local scope = TemplateManager.scope + if not TemplateManager.started or not scope then + return op.always(true, nil) + end + + scope:cancel("template manager shutdown") + + return fibers.boolean_choice( + scope:join_op():wrap(function () + TemplateManager.terminate("template manager joined") + return true, nil + end), + sleep.sleep_op(timeout):wrap(function () + return false, "template manager shutdown timeout" + end) + ):wrap(function (completed, _a, b) + if completed then return true, nil end + return false, b + end) + end) +end + +function TemplateManager.terminate(reason) + for id, driver in pairs(TemplateManager.drivers or {}) do + resource.terminate_checked(driver, reason or "template manager terminated", "template manager driver cleanup failed") + TemplateManager.drivers[id] = nil + end + + if TemplateManager.scope then + TemplateManager.scope:cancel(reason or "template manager terminated") + end + + TemplateManager.scope = nil + TemplateManager.started = false + return true, nil +end + +function TemplateManager.fault_op() + if TemplateManager.scope and TemplateManager.started then + return TemplateManager.scope:fault_op() + end + return op.never() +end + +function TemplateManager.apply_config_op(_namespaces) + return op.always(true, nil) +end + +return TemplateManager diff --git a/src/services/hal/types/capabilities.lua b/src/services/hal/types/capabilities.lua new file mode 100644 index 00000000..99d6bd52 --- /dev/null +++ b/src/services/hal/types/capabilities.lua @@ -0,0 +1,389 @@ +---@param list string[] +---@return table +local function list_to_map(list) + local map = {} + for _, v in ipairs(list) do + map[v] = true + end + return map +end + +local channel = require "fibers.channel" +local ChannelMT = getmetatable(channel.new()) + +local function valid_class(class) + return type(class) == 'string' and class ~= '' +end + +local function valid_id(id) + return (type(id) == 'string' and id ~= '') or (type(id) == 'number' and id >= 0) +end + +local function valid_offerings(offerings) + if type(offerings) ~= 'table' then + return false + end + for _, v in ipairs(offerings) do + if type(v) ~= 'string' or v == '' then + return false + end + end + return true +end + +---@alias CapabilityClass string +---@alias CapabilityId string|integer +---@alias CapabilityOffering string +---@alias CapabilityOfferingMap table + +---@class CapabilityConstructors +local new = {} + + +---@class Capability +---@field class CapabilityClass +---@field id CapabilityId +---@field offerings CapabilityOfferingMap +---@field control_ch Channel +local Capability = {} +Capability.__index = Capability + +---@param class CapabilityClass +---@param id CapabilityId +---@param control_ch Channel +---@param offerings CapabilityOffering[] +---@return Capability? +---@return string error +function new.Capability(class, id, control_ch, offerings) + if not valid_class(class) then + return nil, "invalid capability class" + end + + if not valid_id(id) then + return nil, "invalid capability id" + end + + if getmetatable(control_ch) ~= ChannelMT then + return nil, "invalid capability control_ch" + end + + if not valid_offerings(offerings) then + return nil, "invalid capability offerings" + end + + local offerings_map = list_to_map(offerings) + + local capability = setmetatable({ + class = class, + id = id, + offerings = offerings_map, + control_ch = control_ch, + }, Capability) + + return capability, "" +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.ModemCapability(id, control_ch) + local offerings = { + 'get', + 'enable', + 'disable', + 'restart', + 'connect', + 'disconnect', + 'listen_for_sim', + 'set_signal_update_freq', + } + return new.Capability('modem', id, control_ch, offerings) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.GeoCapability(id, control_ch) + local offerings = {} + return new.Capability('geo', id, control_ch, offerings) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.TimeCapability(id, control_ch) + local offerings = {} + return new.Capability('time', id, control_ch, offerings) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.NetworkCapability(id, control_ch) + local offerings = { + 'validate', + 'plan', + 'apply', + 'snapshot', + 'probe_link', + 'read_counters', + } + return new.Capability('network', id, control_ch, offerings) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.NetworkConfigCapability(id, control_ch) + return new.Capability('network-config', id, control_ch, { + 'validate', + 'plan', + 'apply', + 'apply_live_weights', + 'apply_shaping', + }) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.NetworkStateCapability(id, control_ch) + return new.Capability('network-state', id, control_ch, { + 'snapshot', + 'watch', + }) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.NetworkDiagnosticsCapability(id, control_ch) + return new.Capability('network-diagnostics', id, control_ch, { + 'probe_link', + 'read_counters', + 'speedtest', + }) +end + + + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.WiredProviderCapability(id, control_ch) + return new.Capability('wired-provider', id, control_ch, { + 'snapshot', + 'watch', + 'apply_attachments', + 'set_poe', + 'bounce', + }) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.RadioCapability(id, control_ch) + local offerings = { + 'set_channels', + 'set_txpower', + 'set_country', + 'set_enabled', + 'add_interface', + 'delete_interface', + 'clear_radio_config', + 'set_report_period', + 'apply', + 'rollback', + } + return new.Capability('radio', id, control_ch, offerings) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.BandCapability(id, control_ch) + local offerings = { + 'set_log_level', + 'set_kicking', + 'set_station_counting', + 'set_rrm_mode', + 'set_neighbour_reports', + 'set_legacy_options', + 'set_band_priority', + 'set_band_kicking', + 'set_support_bonus', + 'set_update_freq', + 'set_client_inactive_kickoff', + 'set_cleanup', + 'set_networking', + 'apply', + 'clear', + 'rollback', + } + return new.Capability('band', id, control_ch, offerings) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.SerialCapability(id, control_ch) + local offerings = { + 'open', 'close', 'write' + } + return new.Capability('serial', id, control_ch, offerings) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.FilesystemCapability(id, control_ch) + local offerings = { + 'read', + 'write' + } + return new.Capability('fs', id, control_ch, offerings) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.UARTCapability(id, control_ch) + local offerings = { + 'status', + 'open', + } + return new.Capability('uart', id, control_ch, offerings) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.CpuCapability(id, control_ch) + return new.Capability('cpu', id, control_ch, { 'get' }) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.MemoryCapability(id, control_ch) + return new.Capability('memory', id, control_ch, { 'get' }) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.ThermalCapability(id, control_ch) + return new.Capability('thermal', id, control_ch, { 'get' }) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.PlatformCapability(id, control_ch) + return new.Capability('platform', id, control_ch, { 'get' }) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.PowerCapability(id, control_ch) + return new.Capability('power', id, control_ch, { 'shutdown', 'reboot' }) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.UsbCapability(id, control_ch) + return new.Capability('usb', id, control_ch, { 'enable', 'disable' }) +end + +---@class ControlError +---@field reason string +---@field code integer +local ControlError = {} +ControlError.__index = ControlError + +---@param reason string +---@param code integer? +---@return ControlError +function new.ControlError(reason, code) + if type(reason) ~= 'string' then + reason = tostring(reason) + end + + if type(code) ~= 'number' or code < 0 then + code = 1 + end + + local control_error = setmetatable({ + reason = reason, + code = code, + }, ControlError) + return control_error +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.ControlStoreCapability(id, control_ch) + local offerings = { + 'get', + 'put', + 'delete', + 'list', + 'status', + } + return new.Capability('control-store', id, control_ch, offerings) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.SignatureVerifyCapability(id, control_ch) + return new.Capability('signature_verify', id, control_ch, { + 'verify_ed25519', + }) +end + +---@param id CapabilityId +---@param control_ch Channel +---@return Capability? +---@return string error +function new.ArtifactStoreCapability(id, control_ch) + return new.Capability('artifact-store', id, control_ch, { + 'create-sink', + 'import-path', + 'import-source', + 'open', + 'delete', + 'status', + }) +end + +return { + Capability = Capability, + ControlError = ControlError, + new = new, +} diff --git a/src/services/hal/types/capability_args.lua b/src/services/hal/types/capability_args.lua new file mode 100644 index 00000000..fe503c46 --- /dev/null +++ b/src/services/hal/types/capability_args.lua @@ -0,0 +1,1125 @@ +local new = {} + +---@class ModemGetOpts +---@field field string +---@field timescale? number +local ModemGetOpts = {} +ModemGetOpts.__index = ModemGetOpts + +---Create a new ModemGetOpts. +---@param field string +---@param timescale? number +---@return ModemGetOpts? +---@return string error +function new.ModemGetOpts(field, timescale) + if type(field) ~= 'string' or field == '' then + return nil, "invalid field" + end + + if timescale ~= nil and (type(timescale) ~= 'number' or timescale < 0) then + return nil, "invalid timescale" + end + + return setmetatable({ + field = field, + timescale = timescale, + }, ModemGetOpts), "" +end + +---@class ModemConnectOpts +---@field connection_string string +local ModemConnectOpts = {} +ModemConnectOpts.__index = ModemConnectOpts + +---Create a new ModemConnectOpts. +---@param connection_string string +---@return ModemConnectOpts? +---@return string error +function new.ModemConnectOpts(connection_string) + if type(connection_string) ~= 'string' or connection_string == '' then + return nil, "invalid connection string" + end + return setmetatable({ + connection_string = connection_string, + }, ModemConnectOpts), "" +end + +---@class ModemSignalUpdateOpts +---@field frequency number +local ModemSignalUpdateOpts = {} +ModemSignalUpdateOpts.__index = ModemSignalUpdateOpts + +---Create a new ModemSignalUpdateOpts. +---@param frequency number +---@return ModemSignalUpdateOpts? +---@return string error +function new.ModemSignalUpdateOpts(frequency) + if type(frequency) ~= 'number' or frequency <= 0 then + return nil, "invalid frequency" + end + return setmetatable({ + frequency = frequency, + }, ModemSignalUpdateOpts), "" +end + +---@class FilesystemReadOpts +---@field filename string +local FilesystemReadOpts = {} +FilesystemReadOpts.__index = FilesystemReadOpts + +--- Validate that a filename contains no path separators or .. segments +---@param filename string +---@return boolean valid +---@return string? error +local function validate_filename(filename) + if type(filename) ~= 'string' or filename == '' then + return false, "filename must be a non-empty string" + end + + if filename:find('/') or filename:find('\\') then + return false, "filename cannot contain path separators" + end + + if filename == '..' or filename:find('^%.%.') or filename:find('%.%.') then + return false, "filename cannot contain .. segments" + end + + return true, nil +end + +---Create a new FilesystemReadOpts +---@param filename string +---@return FilesystemReadOpts? +---@return string error +function new.FilesystemReadOpts(filename) + local valid, err = validate_filename(filename) + if not valid then + return nil, err + end + return setmetatable({ + filename = filename, + }, FilesystemReadOpts), "" +end + +---@class FilesystemWriteOpts +---@field filename string +---@field data string +local FilesystemWriteOpts = {} +FilesystemWriteOpts.__index = FilesystemWriteOpts + +---Create a new FilesystemWriteOpts +---@param filename string +---@param data string +---@return FilesystemWriteOpts? +---@return string error +function new.FilesystemWriteOpts(filename, data) + local valid, err = validate_filename(filename) + if not valid then + return nil, err + end + if type(data) ~= 'string' then + return nil, "invalid data" + end + return setmetatable({ + filename = filename, + data = data, + }, FilesystemWriteOpts), "" +end + +---------------------------------------------------------------------- +-- UART +---------------------------------------------------------------------- + +---@class UARTOpenOpts +local UARTOpenOpts = {} +UARTOpenOpts.__index = UARTOpenOpts + +---@param opts table|nil +---@return UARTOpenOpts?|nil +---@return string +function new.UARTOpenOpts(opts) + if opts ~= nil and type(opts) ~= 'table' then + return nil, 'invalid uart open opts' + end + + opts = opts or {} + + -- Deliberately empty for now. + -- On current OpenWrt targets UART line settings are assumed to come from + -- platform/devicetree configuration. Runtime termios configuration can be + -- added later without changing the capability surface. + for k in pairs(opts) do + return nil, 'unsupported uart open option: ' .. tostring(k) + end + + return setmetatable({}, UARTOpenOpts), '' +end + +---@class UARTWriteOpts +---@field data string +local UARTWriteOpts = {} +UARTWriteOpts.__index = UARTWriteOpts + +---Create a new UARTWriteOpts. +---@param data string +---@return UARTWriteOpts? +---@return string error +function new.UARTWriteOpts(data) + if type(data) ~= 'string' or data == '' then + return nil, "data must be a non-empty string" + end + return setmetatable({ + data = data, + }, UARTWriteOpts), "" +end + +---@class MemoryGetOpts +---@field field string +---@field max_age number +local MemoryGetOpts = {} +MemoryGetOpts.__index = MemoryGetOpts + +---Create a new MemoryGetOpts. +---@param field string +---@param max_age number +---@return MemoryGetOpts? +---@return string error +function new.MemoryGetOpts(field, max_age) + if type(field) ~= 'string' or field == '' then + return nil, "invalid field" + end + if type(max_age) ~= 'number' or max_age < 0 then + return nil, "invalid max_age" + end + return setmetatable({ field = field, max_age = max_age }, MemoryGetOpts), "" +end + +---@class CpuGetOpts +---@field field string +---@field max_age number +local CpuGetOpts = {} +CpuGetOpts.__index = CpuGetOpts + +---Create a new CpuGetOpts. +---@param field string +---@param max_age number +---@return CpuGetOpts? +---@return string error +function new.CpuGetOpts(field, max_age) + if type(field) ~= 'string' or field == '' then + return nil, "invalid field" + end + if type(max_age) ~= 'number' or max_age < 0 then + return nil, "invalid max_age" + end + return setmetatable({ field = field, max_age = max_age }, CpuGetOpts), "" +end + +---@class ThermalGetOpts +---@field max_age number +local ThermalGetOpts = {} +ThermalGetOpts.__index = ThermalGetOpts + +---Create a new ThermalGetOpts. +---@param max_age number +---@return ThermalGetOpts? +---@return string error +function new.ThermalGetOpts(max_age) + if type(max_age) ~= 'number' or max_age < 0 then + return nil, "invalid max_age" + end + return setmetatable({ max_age = max_age }, ThermalGetOpts), "" +end + +---@class PlatformGetOpts +---@field field string +---@field max_age number +local PlatformGetOpts = {} +PlatformGetOpts.__index = PlatformGetOpts + +---Create a new PlatformGetOpts. +---@param field string +---@param max_age number +---@return PlatformGetOpts? +---@return string error +function new.PlatformGetOpts(field, max_age) + if type(field) ~= 'string' or field == '' then + return nil, "invalid field" + end + if type(max_age) ~= 'number' or max_age < 0 then + return nil, "invalid max_age" + end + return setmetatable({ field = field, max_age = max_age }, PlatformGetOpts), "" +end + +---@class PowerActionOpts +---@field delay? number +local PowerActionOpts = {} +PowerActionOpts.__index = PowerActionOpts + +---Create a new PowerActionOpts. +---@param delay? number +---@return PowerActionOpts? +---@return string error +function new.PowerActionOpts(delay) + if delay ~= nil and (type(delay) ~= 'number' or delay < 0) then + return nil, "invalid delay" + end + return setmetatable({ delay = delay }, PowerActionOpts), "" +end + +---@class ControlStoreGetOpts +---@field key string +local ControlStoreGetOpts = {} +ControlStoreGetOpts.__index = ControlStoreGetOpts + +---@class ControlStorePutOpts +---@field key string +---@field data string +local ControlStorePutOpts = {} +ControlStorePutOpts.__index = ControlStorePutOpts + +---@class ControlStoreDeleteOpts +---@field key string +local ControlStoreDeleteOpts = {} +ControlStoreDeleteOpts.__index = ControlStoreDeleteOpts + +---@class ControlStoreListOpts +---@field prefix string|nil +local ControlStoreListOpts = {} +ControlStoreListOpts.__index = ControlStoreListOpts + +local function valid_store_key(key) + return type(key) == 'string' + and key ~= '' + and not key:find('[/\\]', 1) + and key:match('^[%w%._%-]+$') ~= nil +end + +---@param key string +---@return ControlStoreGetOpts?|nil +---@return string +function new.ControlStoreGetOpts(key) + if not valid_store_key(key) then + return nil, 'invalid key' + end + return setmetatable({ key = key }, ControlStoreGetOpts), '' +end + +---@param key string +---@param data string +---@return ControlStorePutOpts?|nil +---@return string +function new.ControlStorePutOpts(key, data) + if not valid_store_key(key) then + return nil, 'invalid key' + end + if type(data) ~= 'string' then + return nil, 'data must be a string' + end + return setmetatable({ key = key, data = data }, ControlStorePutOpts), '' +end + +---@param key string +---@return ControlStoreDeleteOpts?|nil +---@return string +function new.ControlStoreDeleteOpts(key) + if not valid_store_key(key) then + return nil, 'invalid key' + end + return setmetatable({ key = key }, ControlStoreDeleteOpts), '' +end + +---@param prefix string|nil +---@return ControlStoreListOpts?|nil +---@return string +function new.ControlStoreListOpts(prefix) + if prefix ~= nil and type(prefix) ~= 'string' then + return nil, 'invalid prefix' + end + return setmetatable({ prefix = prefix }, ControlStoreListOpts), '' +end + +---@class SignatureVerifyEd25519Opts +---@field pubkey_pem string +---@field message string +---@field signature string +local SignatureVerifyEd25519Opts = {} +SignatureVerifyEd25519Opts.__index = SignatureVerifyEd25519Opts + +---@param pubkey_pem string +---@param message string +---@param signature string +---@return SignatureVerifyEd25519Opts? +---@return string +function new.SignatureVerifyEd25519Opts(pubkey_pem, message, signature) + if type(pubkey_pem) ~= 'string' or pubkey_pem == '' then + return nil, 'invalid pubkey_pem' + end + if type(message) ~= 'string' then + return nil, 'invalid message' + end + if type(signature) ~= 'string' or signature == '' then + return nil, 'invalid signature' + end + + return setmetatable({ + pubkey_pem = pubkey_pem, + message = message, + signature = signature, + }, SignatureVerifyEd25519Opts), '' +end + +---------------------------------------------------------------------- +-- Artifact store +---------------------------------------------------------------------- + +---@class ArtifactStoreCreateSinkOpts +---@field meta table +---@field policy string|nil +local ArtifactStoreCreateSinkOpts = {} +ArtifactStoreCreateSinkOpts.__index = ArtifactStoreCreateSinkOpts + +---@class ArtifactStoreImportPathOpts +---@field path string +---@field meta table +---@field policy string|nil +local ArtifactStoreImportPathOpts = {} +ArtifactStoreImportPathOpts.__index = ArtifactStoreImportPathOpts + +---@class ArtifactStoreImportSourceOpts +---@field source any +---@field meta table +---@field policy string|nil +local ArtifactStoreImportSourceOpts = {} +ArtifactStoreImportSourceOpts.__index = ArtifactStoreImportSourceOpts + +---@class ArtifactStoreOpenOpts +---@field artifact_ref string +local ArtifactStoreOpenOpts = {} +ArtifactStoreOpenOpts.__index = ArtifactStoreOpenOpts + +---@class ArtifactStoreDeleteOpts +---@field artifact_ref string +local ArtifactStoreDeleteOpts = {} +ArtifactStoreDeleteOpts.__index = ArtifactStoreDeleteOpts + +---@class ArtifactStoreStatusOpts +local ArtifactStoreStatusOpts = {} +ArtifactStoreStatusOpts.__index = ArtifactStoreStatusOpts + +local function valid_artifact_ref(ref) + return type(ref) == 'string' and ref ~= '' and ref:match('^[A-Za-z0-9_.-]+$') ~= nil +end + +local function valid_artifact_policy(policy) + return policy == nil + or policy == 'transient_only' + or policy == 'prefer_durable' + or policy == 'require_durable' +end + +local function valid_blob_source(source) + return type(source) == 'table' + and type(source.read_chunk_op) == 'function' +end + +---@param meta table|nil +---@param policy string|nil +---@return ArtifactStoreCreateSinkOpts?|nil +---@return string +function new.ArtifactStoreCreateSinkOpts(meta, policy) + if meta ~= nil and type(meta) ~= 'table' then + return nil, 'invalid meta' + end + if not valid_artifact_policy(policy) then + return nil, 'invalid policy' + end + + return setmetatable({ + meta = meta or {}, + policy = policy, + }, ArtifactStoreCreateSinkOpts), '' +end + +---@param path string +---@param meta table|nil +---@param policy string|nil +---@return ArtifactStoreImportPathOpts?|nil +---@return string +function new.ArtifactStoreImportPathOpts(path, meta, policy) + if type(path) ~= 'string' or path == '' then + return nil, 'invalid path' + end + if meta ~= nil and type(meta) ~= 'table' then + return nil, 'invalid meta' + end + if not valid_artifact_policy(policy) then + return nil, 'invalid policy' + end + + return setmetatable({ + path = path, + meta = meta or {}, + policy = policy, + }, ArtifactStoreImportPathOpts), '' +end + +---@param source any +---@param meta table|nil +---@param policy string|nil +---@return ArtifactStoreImportSourceOpts?|nil +---@return string +function new.ArtifactStoreImportSourceOpts(source, meta, policy) + if not valid_blob_source(source) then + return nil, 'invalid source' + end + if meta ~= nil and type(meta) ~= 'table' then + return nil, 'invalid meta' + end + if not valid_artifact_policy(policy) then + return nil, 'invalid policy' + end + + return setmetatable({ + source = source, + meta = meta or {}, + policy = policy, + }, ArtifactStoreImportSourceOpts), '' +end + +---@param artifact_ref string +---@return ArtifactStoreOpenOpts?|nil +---@return string +function new.ArtifactStoreOpenOpts(artifact_ref) + if not valid_artifact_ref(artifact_ref) then + return nil, 'invalid artifact_ref' + end + return setmetatable({ + artifact_ref = artifact_ref, + }, ArtifactStoreOpenOpts), '' +end + +---@param artifact_ref string +---@return ArtifactStoreDeleteOpts?|nil +---@return string +function new.ArtifactStoreDeleteOpts(artifact_ref) + if not valid_artifact_ref(artifact_ref) then + return nil, 'invalid artifact_ref' + end + return setmetatable({ + artifact_ref = artifact_ref, + }, ArtifactStoreDeleteOpts), '' +end + +---@return ArtifactStoreStatusOpts +---@return string +function new.ArtifactStoreStatusOpts() + return setmetatable({}, ArtifactStoreStatusOpts), '' +end + +------------------------------------------------------------------------ +-- Radio capability arg types +------------------------------------------------------------------------ + +local RADIO_VALID_BANDS = { '2g', '5g' } +local RADIO_VALID_HTMODES = { + 'HE20', 'HE40+', 'HE40-', 'HE80', 'HE160', + 'HT20', 'HT40+', 'HT40-', + 'VHT20', 'VHT40+', 'VHT40-', 'VHT80', 'VHT160', +} +local RADIO_VALID_ENCRYPTIONS = { + 'none', 'wep', 'psk', 'psk2', 'psk-mixed', + 'sae', 'sae-mixed', 'owe', 'wpa', 'wpa2', 'wpa3', +} +local RADIO_VALID_IFACE_MODES = { 'ap', 'sta', 'adhoc', 'mesh', 'monitor' } + +local function is_in(value, list) + for _, v in ipairs(list) do + if v == value then return true end + end + return false +end + +---@alias RadioBand '2g'|'5g' +---@alias RadioHtmode 'HE20'|'HE40+'|'HE40-'|'HE80'|'HE160'|'HT20'|'HT40+'|'HT40-'|'VHT20'|'VHT40+'|'VHT40-'|'VHT80'|'VHT160' +---@alias RadioEncryption 'none'|'wep'|'psk'|'psk2'|'psk-mixed'|'sae'|'sae-mixed'|'owe'|'wpa'|'wpa2'|'wpa3' +---@alias RadioIfaceMode 'ap'|'sta'|'adhoc'|'mesh'|'monitor' + +---@class RadioSetChannelsOpts +---@field band RadioBand +---@field channel number|string -- channel number, string, or "auto" +---@field htmode RadioHtmode +---@field channels? (number|string)[] -- required when channel == "auto" +local RadioSetChannelsOpts = {} +RadioSetChannelsOpts.__index = RadioSetChannelsOpts + +---@param band RadioBand +---@param channel number|string +---@param htmode RadioHtmode +---@param channels? (number|string)[] +---@return RadioSetChannelsOpts? +---@return string error +function new.RadioSetChannelsOpts(band, channel, htmode, channels) + if not is_in(band, RADIO_VALID_BANDS) then + return nil, 'band must be one of: ' .. table.concat(RADIO_VALID_BANDS, ', ') + end + if not is_in(htmode, RADIO_VALID_HTMODES) then + return nil, 'htmode must be one of: ' .. table.concat(RADIO_VALID_HTMODES, ', ') + end + if channel == 'auto' then + if type(channels) ~= 'table' or #channels == 0 then + return nil, 'channels must be a non-empty list when channel is "auto"' + end + elseif type(channel) ~= 'number' and type(channel) ~= 'string' then + return nil, 'channel must be a number, string, or "auto"' + end + return setmetatable({ + band = band, channel = channel, htmode = htmode, channels = channels, + }, RadioSetChannelsOpts), "" +end + +---@class RadioSetTxpowerOpts +---@field txpower number|string +local RadioSetTxpowerOpts = {} +RadioSetTxpowerOpts.__index = RadioSetTxpowerOpts + +---@param txpower number|string +---@return RadioSetTxpowerOpts? +---@return string error +function new.RadioSetTxpowerOpts(txpower) + if type(txpower) ~= 'number' and type(txpower) ~= 'string' then + return nil, 'txpower must be a number or string' + end + return setmetatable({ txpower = txpower }, RadioSetTxpowerOpts), "" +end + +---@class RadioSetCountryOpts +---@field country string -- 2-letter ISO code, normalised to uppercase +local RadioSetCountryOpts = {} +RadioSetCountryOpts.__index = RadioSetCountryOpts + +---@param country string ISO-3166-1 alpha-2 code +---@return RadioSetCountryOpts? +---@return string error +function new.RadioSetCountryOpts(country) + if type(country) ~= 'string' or #country ~= 2 then + return nil, 'country must be a 2-character string' + end + return setmetatable({ country = country:upper() }, RadioSetCountryOpts), "" +end + +---@class RadioSetEnabledOpts +---@field enabled boolean +local RadioSetEnabledOpts = {} +RadioSetEnabledOpts.__index = RadioSetEnabledOpts + +---@param enabled boolean +---@return RadioSetEnabledOpts? +---@return string error +function new.RadioSetEnabledOpts(enabled) + if type(enabled) ~= 'boolean' then + return nil, 'enabled must be a boolean' + end + return setmetatable({ enabled = enabled }, RadioSetEnabledOpts), "" +end + +---@class RadioAddInterfaceOpts +---@field ssid string +---@field encryption RadioEncryption +---@field password string +---@field network string +---@field mode RadioIfaceMode +---@field enable_steering boolean +local RadioAddInterfaceOpts = {} +RadioAddInterfaceOpts.__index = RadioAddInterfaceOpts + +---@param ssid string +---@param encryption RadioEncryption +---@param password string +---@param network string +---@param mode RadioIfaceMode +---@param enable_steering boolean +---@return RadioAddInterfaceOpts? +---@return string error +function new.RadioAddInterfaceOpts(ssid, encryption, password, network, mode, enable_steering) + if type(ssid) ~= 'string' or ssid == '' then + return nil, 'ssid must be a non-empty string' + end + if not is_in(encryption, RADIO_VALID_ENCRYPTIONS) then + return nil, 'encryption must be one of: ' .. table.concat(RADIO_VALID_ENCRYPTIONS, ', ') + end + if type(password) ~= 'string' then + return nil, 'password must be a string' + end + if type(network) ~= 'string' or network == '' then + return nil, 'network must be a non-empty string' + end + if not is_in(mode, RADIO_VALID_IFACE_MODES) then + return nil, 'mode must be one of: ' .. table.concat(RADIO_VALID_IFACE_MODES, ', ') + end + if type(enable_steering) ~= 'boolean' then + return nil, 'enable_steering must be a boolean' + end + return setmetatable({ + ssid = ssid, encryption = encryption, password = password, + network = network, mode = mode, enable_steering = enable_steering, + }, RadioAddInterfaceOpts), "" +end + +---@class RadioDeleteInterfaceOpts +---@field interface string +local RadioDeleteInterfaceOpts = {} +RadioDeleteInterfaceOpts.__index = RadioDeleteInterfaceOpts + +---@param interface string +---@return RadioDeleteInterfaceOpts? +---@return string error +function new.RadioDeleteInterfaceOpts(interface) + if type(interface) ~= 'string' or interface == '' then + return nil, 'interface must be a non-empty string' + end + return setmetatable({ interface = interface }, RadioDeleteInterfaceOpts), "" +end + +---@class RadioSetReportPeriodOpts +---@field period number -- seconds, must be > 0 +local RadioSetReportPeriodOpts = {} +RadioSetReportPeriodOpts.__index = RadioSetReportPeriodOpts + +---@param period number seconds, must be > 0 +---@return RadioSetReportPeriodOpts? +---@return string error +function new.RadioSetReportPeriodOpts(period) + if type(period) ~= 'number' or period <= 0 then + return nil, 'period must be a positive number' + end + return setmetatable({ period = period }, RadioSetReportPeriodOpts), "" +end + +---@class RadioCapabilityReply +---@field ok boolean +---@field reason? string -- error message on failure; generated interface name on add_interface success + +------------------------------------------------------------------------ +-- Band capability arg types +------------------------------------------------------------------------ + +local BAND_VALID_KICK_MODES = { 'none', 'compare', 'absolute', 'both' } +local BAND_VALID_RRM_MODES = { 'PAT' } +local BAND_VALID_LEGACY_KEYS = { + 'eval_probe_req', 'eval_assoc_req', 'eval_auth_req', + 'min_probe_count', 'deny_assoc_reason', 'deny_auth_reason', +} +local BAND_VALID_KICKING_OPTS = { + 'rssi_center', 'rssi_reward_threshold', 'rssi_reward', + 'rssi_penalty_threshold', 'rssi_penalty', 'rssi_weight', + 'channel_util_reward_threshold', 'channel_util_reward', + 'channel_util_penalty_threshold', 'channel_util_penalty', +} +local BAND_VALID_UPDATE_KEYS = { 'client', 'chan_util', 'hostapd', 'beacon_reports', 'tcp_con' } +local BAND_VALID_CLEANUP_KEYS = { 'probe', 'client', 'ap' } +local BAND_VALID_NET_METHODS = { 'broadcast', 'tcp+umdns', 'multicast', 'tcp' } +local BAND_VALID_NET_OPTS = { 'ip', 'port', 'broadcast_port', 'enable_encryption' } +local BAND_VALID_BANDS = { '2G', '5G' } +local BAND_VALID_SUPPORTS = { 'ht', 'vht' } + +---@alias BandId '2G'|'5G' +---@alias BandKickMode 'none'|'compare'|'absolute'|'both' +---@alias BandRrmMode 'PAT' +---@alias BandNetworkingMethod 'broadcast'|'tcp+umdns'|'multicast'|'tcp' +---@alias BandSupportType 'ht'|'vht' + +---@class BandSetLogLevelOpts +---@field level number -- non-negative integer +local BandSetLogLevelOpts = {} +BandSetLogLevelOpts.__index = BandSetLogLevelOpts + +---@param level number non-negative integer +---@return BandSetLogLevelOpts? +---@return string error +function new.BandSetLogLevelOpts(level) + if type(level) ~= 'number' or level < 0 then + return nil, 'level must be a non-negative number' + end + return setmetatable({ level = level }, BandSetLogLevelOpts), "" +end + +---@class BandSetKickingOpts +---@field mode BandKickMode +---@field bandwidth_threshold number +---@field kicking_threshold number +---@field evals_before_kick number +local BandSetKickingOpts = {} +BandSetKickingOpts.__index = BandSetKickingOpts + +---@param mode BandKickMode +---@param bandwidth_threshold number +---@param kicking_threshold number +---@param evals_before_kick number +---@return BandSetKickingOpts? +---@return string error +function new.BandSetKickingOpts(mode, bandwidth_threshold, kicking_threshold, evals_before_kick) + if not is_in(mode, BAND_VALID_KICK_MODES) then + return nil, 'mode must be one of: ' .. table.concat(BAND_VALID_KICK_MODES, ', ') + end + if type(bandwidth_threshold) ~= 'number' or bandwidth_threshold < 0 then + return nil, 'bandwidth_threshold must be a non-negative number' + end + if type(kicking_threshold) ~= 'number' or kicking_threshold < 0 then + return nil, 'kicking_threshold must be a non-negative number' + end + if type(evals_before_kick) ~= 'number' or evals_before_kick < 0 then + return nil, 'evals_before_kick must be a non-negative integer' + end + return setmetatable({ + mode = mode, + bandwidth_threshold = bandwidth_threshold, + kicking_threshold = kicking_threshold, + evals_before_kick = evals_before_kick, + }, BandSetKickingOpts), "" +end + +---@class BandSetStationCountingOpts +---@field use_station_count boolean +---@field max_station_diff number +local BandSetStationCountingOpts = {} +BandSetStationCountingOpts.__index = BandSetStationCountingOpts + +---@param use_station_count boolean +---@param max_station_diff number +---@return BandSetStationCountingOpts? +---@return string error +function new.BandSetStationCountingOpts(use_station_count, max_station_diff) + if type(use_station_count) ~= 'boolean' then + return nil, 'use_station_count must be a boolean' + end + if type(max_station_diff) ~= 'number' or max_station_diff < 0 then + return nil, 'max_station_diff must be a non-negative integer' + end + return setmetatable({ + use_station_count = use_station_count, + max_station_diff = max_station_diff, + }, BandSetStationCountingOpts), "" +end + +---@class BandSetRrmModeOpts +---@field mode BandRrmMode +local BandSetRrmModeOpts = {} +BandSetRrmModeOpts.__index = BandSetRrmModeOpts + +---@param mode BandRrmMode +---@return BandSetRrmModeOpts? +---@return string error +function new.BandSetRrmModeOpts(mode) + if not is_in(mode, BAND_VALID_RRM_MODES) then + return nil, 'mode must be one of: ' .. table.concat(BAND_VALID_RRM_MODES, ', ') + end + return setmetatable({ mode = mode }, BandSetRrmModeOpts), "" +end + +---@class BandSetNeighbourReportsOpts +---@field dyn_report_num number +---@field disassoc_report_len number +local BandSetNeighbourReportsOpts = {} +BandSetNeighbourReportsOpts.__index = BandSetNeighbourReportsOpts + +---@param dyn_report_num number +---@param disassoc_report_len number +---@return BandSetNeighbourReportsOpts? +---@return string error +function new.BandSetNeighbourReportsOpts(dyn_report_num, disassoc_report_len) + local dyn = tonumber(dyn_report_num) + local dis = tonumber(disassoc_report_len) + if not dyn or dyn < 0 then + return nil, 'dyn_report_num must be a non-negative integer' + end + if not dis or dis < 0 then + return nil, 'disassoc_report_len must be a non-negative integer' + end + return setmetatable({ + dyn_report_num = dyn, + disassoc_report_len = dis, + }, BandSetNeighbourReportsOpts), "" +end + +---@class BandLegacyOpts +---@field eval_probe_req? boolean +---@field eval_assoc_req? boolean +---@field eval_auth_req? boolean +---@field min_probe_count? number +---@field deny_auth_reason? number +---@field deny_assoc_reason? number + +---@class BandSetLegacyOptionsOpts +---@field opts BandLegacyOpts +local BandSetLegacyOptionsOpts = {} +BandSetLegacyOptionsOpts.__index = BandSetLegacyOptionsOpts + +---@param opts BandLegacyOpts +---@return BandSetLegacyOptionsOpts? +---@return string error +function new.BandSetLegacyOptionsOpts(opts) + if type(opts) ~= 'table' then + return nil, 'opts must be a table' + end + for key in pairs(opts) do + if not is_in(key, BAND_VALID_LEGACY_KEYS) then + return nil, 'unknown legacy option key: ' .. tostring(key) + end + end + return setmetatable({ opts = opts }, BandSetLegacyOptionsOpts), "" +end + +---@class BandSetBandPriorityOpts +---@field band BandId +---@field priority number +local BandSetBandPriorityOpts = {} +BandSetBandPriorityOpts.__index = BandSetBandPriorityOpts + +---@param band BandId +---@param priority number +---@return BandSetBandPriorityOpts? +---@return string error +function new.BandSetBandPriorityOpts(band, priority) + local b = type(band) == 'string' and band:upper() or '' + if not is_in(b, BAND_VALID_BANDS) then + return nil, 'band must be "2G" or "5G"' + end + if type(priority) ~= 'number' or priority < 0 then + return nil, 'priority must be a non-negative number' + end + return setmetatable({ band = b, priority = priority }, BandSetBandPriorityOpts), "" +end + +---@class BandKickingOptions +---@field rssi_center? number +---@field rssi_reward_threshold? number +---@field rssi_reward? number +---@field rssi_penalty_threshold? number +---@field rssi_penalty? number +---@field rssi_weight? number +---@field channel_util_reward_threshold? number +---@field channel_util_reward? number +---@field channel_util_penalty_threshold? number +---@field channel_util_penalty? number + +---@class BandSetBandKickingOpts +---@field band BandId +---@field options BandKickingOptions +local BandSetBandKickingOpts = {} +BandSetBandKickingOpts.__index = BandSetBandKickingOpts + +---@param band BandId +---@param options BandKickingOptions +---@return BandSetBandKickingOpts? +---@return string error +function new.BandSetBandKickingOpts(band, options) + local b = type(band) == 'string' and band:upper() or '' + if not is_in(b, BAND_VALID_BANDS) then + return nil, 'band must be "2G" or "5G"' + end + if type(options) ~= 'table' then + return nil, 'options must be a table' + end + for key, value in pairs(options) do + if not is_in(key, BAND_VALID_KICKING_OPTS) then + return nil, 'unknown band kicking option: ' .. tostring(key) + end + if tonumber(value) == nil then + return nil, 'value for ' .. key .. ' must be a number' + end + end + return setmetatable({ band = b, options = options }, BandSetBandKickingOpts), "" +end + +---@class BandSetSupportBonusOpts +---@field band BandId +---@field support BandSupportType +---@field reward number +local BandSetSupportBonusOpts = {} +BandSetSupportBonusOpts.__index = BandSetSupportBonusOpts + +---@param band BandId +---@param support BandSupportType +---@param reward number +---@return BandSetSupportBonusOpts? +---@return string error +function new.BandSetSupportBonusOpts(band, support, reward) + local b = type(band) == 'string' and band:upper() or '' + if not is_in(b, BAND_VALID_BANDS) then + return nil, 'band must be "2G" or "5G"' + end + if not is_in(support, BAND_VALID_SUPPORTS) then + return nil, 'support must be "ht" or "vht"' + end + if type(reward) ~= 'number' then + return nil, 'reward must be a number' + end + return setmetatable({ band = b, support = support, reward = reward }, BandSetSupportBonusOpts), "" +end + +---@class BandUpdateFreqOptions +---@field client? number +---@field chan_util? number +---@field hostapd? number +---@field beacon_reports? number +---@field tcp_con? number + +---@class BandSetUpdateFreqOpts +---@field updates BandUpdateFreqOptions +local BandSetUpdateFreqOpts = {} +BandSetUpdateFreqOpts.__index = BandSetUpdateFreqOpts + +---@param updates BandUpdateFreqOptions +---@return BandSetUpdateFreqOpts? +---@return string error +function new.BandSetUpdateFreqOpts(updates) + if type(updates) ~= 'table' then + return nil, 'updates must be a table' + end + for key, value in pairs(updates) do + if not is_in(key, BAND_VALID_UPDATE_KEYS) then + return nil, 'unknown update key: ' .. tostring(key) + end + if type(value) ~= 'number' or value < 0 then + return nil, 'value for ' .. key .. ' must be a non-negative number' + end + end + return setmetatable({ updates = updates }, BandSetUpdateFreqOpts), "" +end + +---@class BandSetClientInactiveKickoffOpts +---@field timeout number +local BandSetClientInactiveKickoffOpts = {} +BandSetClientInactiveKickoffOpts.__index = BandSetClientInactiveKickoffOpts + +---@param timeout number non-negative integer +---@return BandSetClientInactiveKickoffOpts? +---@return string error +function new.BandSetClientInactiveKickoffOpts(timeout) + local t = tonumber(timeout) + if not t or t < 0 then + return nil, 'timeout must be a non-negative integer' + end + return setmetatable({ timeout = t }, BandSetClientInactiveKickoffOpts), "" +end + +---@class BandCleanupTimeouts +---@field probe? number +---@field client? number +---@field ap? number + +---@class BandSetCleanupOpts +---@field timeouts BandCleanupTimeouts +local BandSetCleanupOpts = {} +BandSetCleanupOpts.__index = BandSetCleanupOpts + +---@param timeouts BandCleanupTimeouts +---@return BandSetCleanupOpts? +---@return string error +function new.BandSetCleanupOpts(timeouts) + if type(timeouts) ~= 'table' then + return nil, 'timeouts must be a table' + end + for key, value in pairs(timeouts) do + if not is_in(key, BAND_VALID_CLEANUP_KEYS) then + return nil, 'unknown cleanup key: ' .. tostring(key) + end + if type(value) ~= 'number' or value < 0 then + return nil, 'value for cleanup.' .. key .. ' must be a non-negative number' + end + end + return setmetatable({ timeouts = timeouts }, BandSetCleanupOpts), "" +end + +---@class BandNetworkingOptions +---@field ip? string +---@field port? number +---@field broadcast_port? number +---@field enable_encryption? boolean + +---@class BandSetNetworkingOpts +---@field method BandNetworkingMethod +---@field options BandNetworkingOptions +local BandSetNetworkingOpts = {} +BandSetNetworkingOpts.__index = BandSetNetworkingOpts + +---@param method BandNetworkingMethod +---@param options BandNetworkingOptions +---@return BandSetNetworkingOpts? +---@return string error +function new.BandSetNetworkingOpts(method, options) + if not is_in(method, BAND_VALID_NET_METHODS) then + return nil, 'method must be one of: ' .. table.concat(BAND_VALID_NET_METHODS, ', ') + end + if type(options) ~= 'table' then + return nil, 'options must be a table' + end + for key, value in pairs(options) do + if not is_in(key, BAND_VALID_NET_OPTS) then + return nil, 'unknown networking option: ' .. tostring(key) + end + if key == 'ip' and type(value) ~= 'string' then + return nil, 'networking.ip must be a string' + end + if (key == 'port' or key == 'broadcast_port') and type(value) ~= 'number' then + return nil, 'networking.' .. key .. ' must be a number' + end + if key == 'enable_encryption' and type(value) ~= 'boolean' then + return nil, 'networking.enable_encryption must be a boolean' + end + end + return setmetatable({ method = method, options = options }, BandSetNetworkingOpts), "" +end + +---@class BandCapabilityReply +---@field ok boolean +---@field reason? string -- error message on failure + +return { + ModemGetOpts = ModemGetOpts, + ModemConnectOpts = ModemConnectOpts, + ModemSignalUpdateOpts = ModemSignalUpdateOpts, + FilesystemReadOpts = FilesystemReadOpts, + FilesystemWriteOpts = FilesystemWriteOpts, + UARTOpenOpts = UARTOpenOpts, + UARTWriteOpts = UARTWriteOpts, + MemoryGetOpts = MemoryGetOpts, + CpuGetOpts = CpuGetOpts, + ThermalGetOpts = ThermalGetOpts, + PlatformGetOpts = PlatformGetOpts, + PowerActionOpts = PowerActionOpts, + ControlStoreGetOpts = ControlStoreGetOpts, + ControlStorePutOpts = ControlStorePutOpts, + ControlStoreDeleteOpts = ControlStoreDeleteOpts, + ControlStoreListOpts = ControlStoreListOpts, + SignatureVerifyEd25519Opts = SignatureVerifyEd25519Opts, + ArtifactStoreCreateSinkOpts = ArtifactStoreCreateSinkOpts, + ArtifactStoreImportPathOpts = ArtifactStoreImportPathOpts, + ArtifactStoreImportSourceOpts = ArtifactStoreImportSourceOpts, + ArtifactStoreOpenOpts = ArtifactStoreOpenOpts, + ArtifactStoreDeleteOpts = ArtifactStoreDeleteOpts, + ArtifactStoreStatusOpts = ArtifactStoreStatusOpts, + RadioSetChannelsOpts = RadioSetChannelsOpts, + RadioSetTxpowerOpts = RadioSetTxpowerOpts, + RadioSetCountryOpts = RadioSetCountryOpts, + RadioSetEnabledOpts = RadioSetEnabledOpts, + RadioAddInterfaceOpts = RadioAddInterfaceOpts, + RadioDeleteInterfaceOpts = RadioDeleteInterfaceOpts, + RadioSetReportPeriodOpts = RadioSetReportPeriodOpts, + BandSetLogLevelOpts = BandSetLogLevelOpts, + BandSetKickingOpts = BandSetKickingOpts, + BandSetStationCountingOpts = BandSetStationCountingOpts, + BandSetRrmModeOpts = BandSetRrmModeOpts, + BandSetNeighbourReportsOpts = BandSetNeighbourReportsOpts, + BandSetLegacyOptionsOpts = BandSetLegacyOptionsOpts, + BandSetBandPriorityOpts = BandSetBandPriorityOpts, + BandSetBandKickingOpts = BandSetBandKickingOpts, + BandSetSupportBonusOpts = BandSetSupportBonusOpts, + BandSetUpdateFreqOpts = BandSetUpdateFreqOpts, + BandSetClientInactiveKickoffOpts = BandSetClientInactiveKickoffOpts, + BandSetCleanupOpts = BandSetCleanupOpts, + BandSetNetworkingOpts = BandSetNetworkingOpts, + new = new, +} diff --git a/src/services/hal/types/core.lua b/src/services/hal/types/core.lua new file mode 100644 index 00000000..f5296040 --- /dev/null +++ b/src/services/hal/types/core.lua @@ -0,0 +1,302 @@ +---@alias DeviceClass string +---@alias DeviceId string|integer + +---@alias Meta table + +---@alias CapList Capability[] + +local cap_types = require "services.hal.types.capabilities" +local channel = require "fibers.channel" +local ChannelMT = getmetatable(channel.new()) + +---@class TypeConstructors +local new = {} + + +---@class ControlRequest +---@field verb string +---@field opts table +---@field reply_ch Channel +---@field cancel_op Op? +local ControlRequest = {} +ControlRequest.__index = ControlRequest + +---Create a new ControlRequest. +---@param verb string +---@param opts table +---@param reply_ch Channel +---@param cancel_op Op? +---@return ControlRequest? +---@return string error +function new.ControlRequest(verb, opts, reply_ch, cancel_op) + if type(verb) ~= 'string' or verb == '' then + return nil, "invalid verb" + end + + if type(opts) ~= 'table' then + return nil, "opts must be a table" + end + + if getmetatable(reply_ch) ~= ChannelMT then + return nil, "invalid reply_ch" + end + + return setmetatable({ + verb = verb, + opts = opts, + reply_ch = reply_ch, + cancel_op = cancel_op, + }, ControlRequest), "" +end + +---@class Reply +---@field ok boolean +---@field reason any +---@field code integer? +local Reply = {} +Reply.__index = Reply + +---Create a new Reply. +---@param ok boolean +---@param reason string? +---@param code integer? +---@return Reply? +---@return string error +function new.Reply(ok, reason, code) + if type(ok) ~= 'boolean' then + return nil, "invalid ok" + end + + return setmetatable({ + ok = ok, + reason = reason, + code = code, + }, Reply), "" +end + +---@alias EmitMode 'event'|'state'|'meta'|'log' + +---@class Emit +---@field class CapabilityClass +---@field id CapabilityId +---@field mode EmitMode +---@field key string +---@field data any +local Emit = {} +Emit.__index = Emit + +---Create a new Emit. +---@param class CapabilityClass +---@param id CapabilityId +---@param mode EmitMode +---@param key string +---@param data any +---@return Emit? +---@return string error +function new.Emit(class, id, mode, key, data) + if type(class) ~= 'string' or class == '' then + return nil, "invalid class" + end + + if type(id) ~= 'string' and type(id) ~= 'number' then + return nil, "invalid id" + end + + if mode ~= 'event' and mode ~= 'state' and mode ~= 'meta' and mode ~= 'log' then + return nil, "invalid mode" + end + + if type(key) ~= 'string' or key == '' then + return nil, "invalid key" + end + + if type(data) == 'nil' then + return nil, "data cannot be nil" + end + + return setmetatable({ + class = class, + id = id, + mode = mode, + key = key, + data = data, + }, Emit), "" +end + +---@alias EventType 'added'|'removed' + +---@class DeviceEvent +---@field event_type EventType +---@field class DeviceClass +---@field id DeviceId +---@field meta Meta +---@field capabilities CapList +---@field ready_cond Cond? Optional one-shot condition; signalled by HAL after the device is fully registered. +local DeviceEvent = {} +DeviceEvent.__index = DeviceEvent + +---Create a new DeviceEvent. +---@param event_type EventType +---@param class DeviceClass +---@param id DeviceId +---@param meta Meta? +---@param capabilities CapList? +---@param ready_cond Cond? Optional one-shot condition; signalled by HAL after the device is fully registered. +---@return DeviceEvent? +---@return string error +function new.DeviceEvent(event_type, class, id, meta, capabilities, ready_cond) + meta = meta or {} + capabilities = capabilities or {} + + if event_type ~= 'added' and event_type ~= 'removed' then + return nil, "invalid event_type" + end + + if type(class) ~= 'string' or class == '' then + return nil, "invalid class" + end + + if type(id) ~= 'string' and type(id) ~= 'number' then + return nil, "invalid id" + end + + if type(meta) ~= 'table' then + return nil, "invalid meta" + end + + if type(capabilities) ~= 'table' then + return nil, "invalid capabilities" + end + for _, cap in ipairs(capabilities) do + if getmetatable(cap) ~= cap_types.Capability then + return nil, "invalid capability in capabilities" + end + end + + if ready_cond ~= nil and type(ready_cond.signal) ~= 'function' then + return nil, "invalid ready_cond" + end + + local ev = setmetatable({ + event_type = event_type, + class = class, + id = id, + meta = meta, + capabilities = capabilities, + ready_cond = ready_cond, + }, DeviceEvent) + + return ev, "" +end + +---@class Device +---@field class DeviceClass +---@field id DeviceId +---@field meta Meta +---@field capabilities Capability[] +local Device = {} +Device.__index = Device + +---Create a new Device. +---@param class DeviceClass +---@param id DeviceId +---@param meta Meta +---@param capabilities CapList +---@return Device? +---@return string error +function new.Device(class, id, meta, capabilities) + meta = meta or {} + capabilities = capabilities or {} + + if type(class) ~= 'string' or class == '' then + return nil, "invalid class" + end + + if type(id) ~= 'string' and type(id) ~= 'number' then + return nil, "invalid id" + end + + if type(meta) ~= 'table' then + return nil, "invalid meta" + end + + if type(capabilities) ~= 'table' then + return nil, "invalid capabilities" + end + for _, cap in ipairs(capabilities) do + if getmetatable(cap) ~= cap_types.Capability then + return nil, "invalid capability in capabilities" + end + end + + local dev = setmetatable({ + class = class, + id = id, + meta = meta, + capabilities = capabilities, + }, Device) + return dev, "" +end + +-- Todo types: +---@class Manager +---@field scope Scope +---@field start fun(logger: table|nil, dev_ev_ch: Channel, cap_emit_ch: Channel): string error +---@field stop fun(): string error +---@field apply_config fun(config: table): boolean ok, string error + +---------------------------------------------------------------------- +-- UART open reply +---------------------------------------------------------------------- + +---@class UARTOpenReply +---@field lease_id string +---@field session any +---@field path string +---@field baud integer|nil +---@field mode string|nil +local UARTOpenReply = {} +UARTOpenReply.__index = UARTOpenReply + +---@param lease_id string +---@param session any +---@param path string +---@param baud integer|nil +---@param mode string|nil +---@return UARTOpenReply?|nil +---@return string +function new.UARTOpenReply(lease_id, session, path, baud, mode) + if type(lease_id) ~= 'string' or lease_id == '' then + return nil, 'invalid lease_id' + end + if session == nil then + return nil, 'missing session' + end + if type(path) ~= 'string' or path == '' then + return nil, 'invalid path' + end + if baud ~= nil and (type(baud) ~= 'number' or baud <= 0 or baud % 1 ~= 0) then + return nil, 'invalid baud' + end + if mode ~= nil and type(mode) ~= 'string' then + return nil, 'invalid mode' + end + + return setmetatable({ + lease_id = lease_id, + session = session, + path = path, + baud = baud, + mode = mode, + }, UARTOpenReply), '' +end + +return { + ControlRequest = ControlRequest, + Reply = Reply, + Emit = Emit, + DeviceEvent = DeviceEvent, + Device = Device, + UARTOpenReply = UARTOpenReply, + new = new, +} diff --git a/src/services/hal/types/modem.lua b/src/services/hal/types/modem.lua new file mode 100644 index 00000000..e504b0d0 --- /dev/null +++ b/src/services/hal/types/modem.lua @@ -0,0 +1,545 @@ +---@alias ModemAddress string + +---@class ModemDevice +---@field driver Modem +---@field device string +local ModemDevice = {} +ModemDevice.__index = ModemDevice + +---@class ModemTypeConstructors +local new = {} + +---Create a new ModemDevice. +---@param driver Modem +---@param device string +---@return ModemDevice? +---@return string error +function new.ModemDevice(driver, device) + if driver == nil then + return nil, "invalid driver" + end + + if type(device) ~= 'string' or device == '' then + return nil, "invalid device" + end + + return setmetatable({ + driver = driver, + device = device, + }, ModemDevice), "" +end + +---@class ModemIdentity +---@field imei string +---@field address ModemAddress +---@field mode_port string +---@field at_port string +---@field net_port string +---@field device string +local ModemIdentity = {} +ModemIdentity.__index = ModemIdentity + +---Create a new ModemIdentity. +---@param imei string +---@param address ModemAddress +---@param mode_port string +---@param at_port string +---@param net_port string +---@param device string +---@return ModemIdentity? +---@return string error +function new.ModemIdentity(imei, address, mode_port, at_port, net_port, device) + if type(imei) ~= 'string' or imei == '' then + return nil, "invalid imei" + end + + if type(address) ~= 'string' or address == '' then + return nil, "invalid address" + end + + if type(mode_port) ~= 'string' or mode_port == '' then + return nil, "invalid mode_port" + end + + if type(at_port) ~= 'string' or at_port == '' then + return nil, "invalid at_port" + end + + if type(net_port) ~= 'string' or net_port == '' then + return nil, "invalid net_port" + end + + if type(device) ~= 'string' or device == '' then + return nil, "invalid device" + end + + local identity = setmetatable({ + imei = imei, + address = address, + mode_port = mode_port, + at_port = at_port, + net_port = net_port, + device = device, + }, ModemIdentity) + return identity, "" +end + +---@class ModemIdentityInfo +---@field imei string +---@field model string? +---@field revision string? +---@field firmware string? +---@field plugin string? +---@field drivers string[] +---@field mode string? +---@field model_variant string? +local ModemIdentityInfo = {} +ModemIdentityInfo.__index = ModemIdentityInfo + +---@class ModemPortsInfo +---@field device string +---@field primary_port string? +---@field at_ports string[] +---@field qmi_ports string[] +---@field gps_ports string[] +---@field net_ports string[] +local ModemPortsInfo = {} +ModemPortsInfo.__index = ModemPortsInfo + +---@class ModemSimInfo +---@field sim string? +---@field iccid string? +---@field imsi string? +---@field gid1 string? +---@field sim_lock string? +---@field sim_lock_retries table? +---@field modem_state string? +local ModemSimInfo = {} +ModemSimInfo.__index = ModemSimInfo + +---@class ModemNetworkInfo +---@field operator string? +---@field access_techs string[] +---@field mcc string? +---@field mnc string? +---@field active_band_class string? +local ModemNetworkInfo = {} +ModemNetworkInfo.__index = ModemNetworkInfo + +---@class ModemSignalInfo +---@field values table +local ModemSignalInfo = {} +ModemSignalInfo.__index = ModemSignalInfo + +---@class ModemTrafficInfo +---@field rx_bytes integer +---@field tx_bytes integer +local ModemTrafficInfo = {} +ModemTrafficInfo.__index = ModemTrafficInfo + +---@param values any +---@return boolean +local function is_string_array(values) + if type(values) ~= 'table' then + return false + end + for index, value in ipairs(values) do + if type(index) ~= 'number' or type(value) ~= 'string' then + return false + end + end + return true +end + +---@param value any +---@param field string +---@return boolean +---@return string +local function validate_optional_string(value, field) + if value == nil then + return true, "" + end + if type(value) ~= 'string' or value == '' then + return false, "invalid " .. field + end + return true, "" +end + +---@param value any +---@return boolean +local function is_signal_value_table(value) + if type(value) ~= 'table' then + return false + end + for key, entry in pairs(value) do + if type(key) ~= 'string' then + return false + end + local entry_type = type(entry) + if entry_type ~= 'string' and entry_type ~= 'number' then + return false + end + end + return true +end + +---Create a new ModemIdentityInfo. +---@param imei string +---@param drivers string[] +---@param model string? +---@param revision string? +---@param firmware string? +---@param plugin string? +---@param mode string? +---@param model_variant string? +---@return ModemIdentityInfo? +---@return string error +function new.ModemIdentityInfo(imei, drivers, model, revision, firmware, plugin, mode, model_variant) + if type(imei) ~= 'string' or imei == '' then + return nil, "invalid imei" + end + if not is_string_array(drivers) then + return nil, "invalid drivers" + end + local ok, err = validate_optional_string(model, 'model') + if not ok then + return nil, err + end + ok, err = validate_optional_string(revision, 'revision') + if not ok then + return nil, err + end + ok, err = validate_optional_string(firmware, 'firmware') + if not ok then + return nil, err + end + ok, err = validate_optional_string(plugin, 'plugin') + if not ok then + return nil, err + end + ok, err = validate_optional_string(mode, 'mode') + if not ok then + return nil, err + end + ok, err = validate_optional_string(model_variant, 'model_variant') + if not ok then + return nil, err + end + + return setmetatable({ + imei = imei, + drivers = drivers, + model = model, + revision = revision, + firmware = firmware, + plugin = plugin, + mode = mode, + model_variant = model_variant, + }, ModemIdentityInfo), "" +end + +---Create a new ModemPortsInfo. +---@param device string +---@param primary_port string? +---@param at_ports string[]? +---@param qmi_ports string[]? +---@param gps_ports string[]? +---@param net_ports string[]? +---@return ModemPortsInfo? +---@return string error +function new.ModemPortsInfo(device, primary_port, at_ports, qmi_ports, gps_ports, net_ports) + if type(device) ~= 'string' or device == '' then + return nil, "invalid device" + end + local ok, err = validate_optional_string(primary_port, 'primary_port') + if not ok then + return nil, err + end + at_ports = at_ports or {} + qmi_ports = qmi_ports or {} + gps_ports = gps_ports or {} + net_ports = net_ports or {} + if not is_string_array(at_ports) then + return nil, "invalid at_ports" + end + if not is_string_array(qmi_ports) then + return nil, "invalid qmi_ports" + end + if not is_string_array(gps_ports) then + return nil, "invalid gps_ports" + end + if not is_string_array(net_ports) then + return nil, "invalid net_ports" + end + + return setmetatable({ + device = device, + primary_port = primary_port, + at_ports = at_ports, + qmi_ports = qmi_ports, + gps_ports = gps_ports, + net_ports = net_ports, + }, ModemPortsInfo), "" +end + +---Create a new ModemSimInfo. +---@param sim string? +---@param iccid string? +---@param imsi string? +---@param gid1 string? +---@param sim_lock string? +---@param sim_lock_retries table? +---@param modem_state string? +---@return ModemSimInfo? +---@return string error +function new.ModemSimInfo(sim, iccid, imsi, gid1, sim_lock, sim_lock_retries, modem_state) + local ok, err = validate_optional_string(sim, 'sim') + if not ok then + return nil, err + end + ok, err = validate_optional_string(iccid, 'iccid') + if not ok then + return nil, err + end + ok, err = validate_optional_string(imsi, 'imsi') + if not ok then + return nil, err + end + ok, err = validate_optional_string(gid1, 'gid1') + if not ok then + return nil, err + end + ok, err = validate_optional_string(sim_lock, 'sim_lock') + if not ok then + return nil, err + end + ok, err = validate_optional_string(modem_state, 'modem_state') + if not ok then + return nil, err + end + if sim_lock_retries ~= nil then + if type(sim_lock_retries) ~= 'table' then + return nil, 'invalid sim_lock_retries' + end + for k, v in pairs(sim_lock_retries) do + if type(k) ~= 'string' or k == '' or type(v) ~= 'number' or v < 0 or v % 1 ~= 0 then + return nil, 'invalid sim_lock_retries' + end + end + end + + return setmetatable({ + sim = sim, + iccid = iccid, + imsi = imsi, + gid1 = gid1, + sim_lock = sim_lock, + sim_lock_retries = sim_lock_retries, + modem_state = modem_state, + }, ModemSimInfo), "" +end + +---Create a new ModemNetworkInfo. +---@param operator string? +---@param access_techs string[]? +---@param mcc string? +---@param mnc string? +---@param active_band_class string? +---@return ModemNetworkInfo? +---@return string error +function new.ModemNetworkInfo(operator, access_techs, mcc, mnc, active_band_class) + local ok, err = validate_optional_string(operator, 'operator') + if not ok then + return nil, err + end + access_techs = access_techs or {} + if not is_string_array(access_techs) then + return nil, "invalid access_techs" + end + ok, err = validate_optional_string(mcc, 'mcc') + if not ok then + return nil, err + end + ok, err = validate_optional_string(mnc, 'mnc') + if not ok then + return nil, err + end + ok, err = validate_optional_string(active_band_class, 'active_band_class') + if not ok then + return nil, err + end + + return setmetatable({ + operator = operator, + access_techs = access_techs, + mcc = mcc, + mnc = mnc, + active_band_class = active_band_class, + }, ModemNetworkInfo), "" +end + +---Create a new ModemSignalInfo. +---@param values table +---@return ModemSignalInfo? +---@return string error +function new.ModemSignalInfo(values) + if not is_signal_value_table(values) then + return nil, "invalid signal values" + end + return setmetatable({ values = values }, ModemSignalInfo), "" +end + +---Create a new ModemTrafficInfo. +---@param rx_bytes integer +---@param tx_bytes integer +---@return ModemTrafficInfo? +---@return string error +function new.ModemTrafficInfo(rx_bytes, tx_bytes) + if type(rx_bytes) ~= 'number' or rx_bytes < 0 then + return nil, "invalid rx_bytes" + end + if type(tx_bytes) ~= 'number' or tx_bytes < 0 then + return nil, "invalid tx_bytes" + end + return setmetatable({ + rx_bytes = rx_bytes, + tx_bytes = tx_bytes, + }, ModemTrafficInfo), "" +end + +---@alias ModemStateEventType "initial"|"changed"|"removed" +---@alias ModemState string + +---@class ModemStateEvent +---@field ev_type ModemStateEventType +---@field from ModemState +---@field to ModemState +---@field reason string? +local ModemStateEvent = {} +ModemStateEvent.__index = ModemStateEvent + +--- Creates a new ModemStateEvent. +---@param ev_type ModemStateEventType +---@param from ModemState +---@param to ModemState +---@param reason string? +---@return ModemStateEvent? +---@return string error +function new.ModemStateEvent(ev_type, from, to, reason) + if ev_type ~= "initial" and ev_type ~= "changed" and ev_type ~= "removed" then + return nil, "invalid event type" + end + + if type(from) ~= 'string' or from == '' then + return nil, "invalid from state" + end + + if type(to) ~= 'string' or to == '' then + return nil, "invalid to state" + end + + reason = reason or "unknown" + if (type(reason) ~= 'string' or reason == '') then + return nil, "invalid reason" + end + + local event = setmetatable({ + ev_type = ev_type, + from = from, + to = to, + reason = reason, + }, { __index = ModemStateEvent }) + return event, "" +end + +function new.ModemStateInitialEvent(state, reason) + return new.ModemStateEvent("initial", state, state, reason) +end + +function new.ModemStateChangeEvent(from, to, reason) + return new.ModemStateEvent("changed", from, to, reason) +end + +function new.ModemStateRemovedEvent(reason) + return new.ModemStateEvent("removed", "removed", "removed", reason) +end + +---@class ModemMonitorEvent +---@field is_added boolean +---@field address ModemAddress +local ModemMonitorEvent = {} +ModemMonitorEvent.__index = ModemMonitorEvent + +---Create a new ModemMonitorEvent. +---@param is_added boolean +---@param address ModemAddress +---@return ModemMonitorEvent? +---@return string error +function new.ModemMonitorEvent(is_added, address) + if type(is_added) ~= 'boolean' then + return nil, "invalid is_added: expected boolean" + end + if type(address) ~= 'string' or address == '' then + return nil, "invalid address" + end + return setmetatable({ + is_added = is_added, + address = address, + }, ModemMonitorEvent), "" +end + +---@class ModemMonitor +---@field next_event_op fun(self: ModemMonitor): Op +---@field shutdown_op fun(self: ModemMonitor, timeout: number?): Op +---@field terminate fun(self: ModemMonitor, reason: string?) + +---@class ModemBackend +---@field identity ModemIdentity +---@field base string +---@field last_state_event ModemStateEvent? +---@field inhibit_cmd Command? +---@field state_monitor table? +---@field sim_present table? +---@field read_identity fun(self: ModemBackend): ModemIdentityInfo?, string +---@field read_ports fun(self: ModemBackend): ModemPortsInfo?, string +---@field read_sim_info fun(self: ModemBackend): ModemSimInfo?, string +---@field read_network_info fun(self: ModemBackend): ModemNetworkInfo?, string +---@field read_signal fun(self: ModemBackend): ModemSignalInfo?, string +---@field read_traffic fun(self: ModemBackend): ModemTrafficInfo?, string +---@field start_state_monitor fun(self: ModemBackend): boolean, string +---@field stop_state_monitor_op fun(self: ModemBackend, timeout: number?): Op +---@field stop_state_monitor fun(self: ModemBackend, timeout: number?): boolean, string +---@field monitor_state_op fun(self: ModemBackend): Op +---@field start_sim_presence_monitor fun(self: ModemBackend): boolean, string +---@field stop_sim_presence_monitor_op fun(self: ModemBackend, timeout: number?): Op +---@field stop_sim_presence_monitor fun(self: ModemBackend, timeout: number?): boolean, string +---@field wait_for_sim_present_op fun(self: ModemBackend): Op +---@field wait_for_sim_present fun(self: ModemBackend): boolean, string +---@field is_sim_present fun(self: ModemBackend): boolean, string +---@field trigger_sim_presence_check fun(self: ModemBackend, cooldown: number?): boolean, string +---@field enable fun(self: ModemBackend): boolean, string +---@field disable fun(self: ModemBackend): boolean, string +---@field reset fun(self: ModemBackend): boolean, string +---@field connect fun(self: ModemBackend, conn_string: string): boolean, string +---@field disconnect fun(self: ModemBackend): boolean, string +---@field inhibit fun(self: ModemBackend): boolean, string +---@field uninhibit_op fun(self: ModemBackend, timeout: number?): Op +---@field uninhibit fun(self: ModemBackend, timeout: number?): boolean, string +---@field shutdown_op fun(self: ModemBackend, timeout: number?): Op +---@field shutdown fun(self: ModemBackend, timeout: number?): boolean, string +---@field terminate fun(self: ModemBackend, reason: string?) +---@field set_signal_update_interval fun(self: ModemBackend, interval: number): boolean, string + +return { + ModemDevice = ModemDevice, + ModemIdentity = ModemIdentity, + ModemIdentityInfo = ModemIdentityInfo, + ModemPortsInfo = ModemPortsInfo, + ModemSimInfo = ModemSimInfo, + ModemNetworkInfo = ModemNetworkInfo, + ModemSignalInfo = ModemSignalInfo, + ModemTrafficInfo = ModemTrafficInfo, + ModemStateEvent = ModemStateEvent, + ModemMonitorEvent = ModemMonitorEvent, + new = new, +} diff --git a/src/services/hal/types/time.lua b/src/services/hal/types/time.lua new file mode 100644 index 00000000..715c7f4f --- /dev/null +++ b/src/services/hal/types/time.lua @@ -0,0 +1,60 @@ +---@alias NTPAction string + +---@class NTPEvent +---@field stratum number NTP stratum level (0-15: synced, 16: unsynced) +---@field action NTPAction Event action (e.g., "step", "clock", "leap") +---@field offset number Clock offset in seconds +---@field freq_drift_ppm number Frequency drift in parts per million +---@field [string] any Additional fields from backend +local NTPEvent = {} +NTPEvent.__index = NTPEvent + +---@class TimeTypeConstructors +local new = {} + +---Create a new NTPEvent. +--- +---@param stratum number NTP stratum (0-15 synced, 16 unsynced) +---@param action string Event action +---@param offset number Clock offset in seconds +---@param freq_drift_ppm number Frequency drift in ppm +---@return NTPEvent? +---@return string error +function new.NTPEvent(stratum, action, offset, freq_drift_ppm) + if type(stratum) ~= 'number' then + return nil, "invalid stratum" + end + + if type(action) ~= 'string' or action == '' then + return nil, "invalid action" + end + + if type(offset) ~= 'number' then + return nil, "invalid offset" + end + + if type(freq_drift_ppm) ~= 'number' then + return nil, "invalid freq_drift_ppm" + end + + local event = setmetatable({ + stratum = stratum, + action = action, + offset = offset, + freq_drift_ppm = freq_drift_ppm, + }, NTPEvent) + return event, "" +end + +---@class TimeBackend +---@field start_ntp_monitor fun(self: TimeBackend): boolean, string +---@field ntp_event_op fun(self: TimeBackend): Op +---@field stop fun(self: TimeBackend): boolean, string +local TimeBackend = {} +TimeBackend.__index = TimeBackend + +return { + NTPEvent = NTPEvent, + TimeBackend = TimeBackend, + new = new, +} diff --git a/src/services/hal/usb3.lua b/src/services/hal/usb3.lua deleted file mode 100644 index 1d60148e..00000000 --- a/src/services/hal/usb3.lua +++ /dev/null @@ -1,188 +0,0 @@ -local exec = require 'fibers.exec' -local sc = require 'fibers.utils.syscall' -local sleep = require 'fibers.sleep' -local log = require 'services.log' - - -local usb_hub_address_prefix = "/sys/devices/platform/scb/fd500000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0/usb" -local usb_hub_port_subpaths = { - [1] = { - [1] = "1-1/1-1.1", - [2] = "1-1/1-1.2", - [3] = "1-1/1-1.3", - [4] = "1-1/1-1.4", - }, - [2] = { - [1] = "2-1", - [2] = "2-2", - } -} - -local function is_device_on_hub_port(hub, port) - if not usb_hub_port_subpaths[hub] then - return false, "invalid hub specified" - elseif not usb_hub_port_subpaths[hub][port] then - return false, "invalid port specified for hub" - end - - local isdir, err = sc.access(usb_hub_address_prefix .. hub .. "/" .. usb_hub_port_subpaths[hub][port], "rw") - if err ~= nil and not string.match(err, "No such file or directory") then - log.warn("DEFAULT_ERROR", "unexpected error checking for device on hub port: " .. err) - end - return isdir == 0 -end - -local function exec_write_to_file(path, value) - local cmd = exec.command("/bin/sh", "-c", string.format("echo %d > %s", value, path)) - local _, err = cmd:combined_output() - return err -end - -local function set_usb_hub_auth_default(enabled, hub) - if type(enabled) ~= "boolean" then - return "default authorization of USB hub must be set with a boolean value" - end - local path = string.format("%s%s/authorized_default", usb_hub_address_prefix, hub) - local err = exec_write_to_file(path, enabled and 1 or 0) - if err then return "setting default authorization of USB hub failed: " .. err end - return nil -end - -local function set_usb_port_auth(enabled, hub, port) - if type(enabled) ~= "boolean" then - return "authorization of USB hub port must be set with a boolean value" - end - local path = string.format("%s%s/%s/authorized", usb_hub_address_prefix, hub, usb_hub_port_subpaths[hub][port]) - local err = exec_write_to_file(path, enabled and 1 or 0) - if err then return "setting authorization of USB hub port failed: " .. err end - return nil -end - -local function clear_usb_3_0_hub() - local is_hub_used = false - for i = 1, #usb_hub_port_subpaths[2] do - local is_port_used, err = is_device_on_hub_port(2, i) - if err ~= nil then return false, "error checking current usb3.0 connections: " .. err end - is_hub_used = is_hub_used or is_port_used - if is_port_used then - err = set_usb_port_auth(false, 2, i) - if err ~= nil then return true, "error clearing current usb3.0 connections: " .. err end - end - end - return is_hub_used, nil -end - -local function repopulate_usb_3_0_hub() - local is_hub_used = false - for i = 1, #usb_hub_port_subpaths[2] do - local is_port_used, err = is_device_on_hub_port(2, i) - if err ~= nil then return false, "error checking current usb3.0 connections: " .. err end - is_hub_used = is_hub_used or is_port_used - if is_port_used then - err = set_usb_port_auth(true, 2, i) - if err ~= nil then return true, "error adding usb3.0 connections: " .. err end - end - end - return is_hub_used, nil -end - -local function set_usb_hub_power(enabled, hub) - if type(enabled) ~= "boolean" then - return "power of USB hub must be set with a boolean value" - end - local cmd = exec.command("uhubctl", "-e", "-l", tostring(hub), "-a", tostring(enabled and 1 or 0)) - local _, err = cmd:combined_output() - if err then return "setting power of USB hub port failed: " .. err end - return nil -end - -local function get_vl805_version_timestamp() - local cmd = exec.command("vcgencmd", "bootloader_version") - local output, err = cmd:combined_output() - if err then - log.error("DEFAULT_OUT_ERR", output, err) - err = err or "" - return nil, output .. err - end - local timestamp = string.match(output, "timestamp%s+(%d+)") - if timestamp == nil then return nil, "timestamp not found in bootloader version info" end - return tonumber(timestamp), nil -end - ----Turn off the USB3 hub and move peripherals to USB2 ----@param ctx Context ----@param model string? -local function disable_usb3(ctx, model) - if model ~= "bigbox-ss" then return end - -- VL805 (usb hub controller) firmware needs to be past a certain version - -- version was added 2019-09-10 so let's check our version is 2019-09-10 or later - local vl805_supported_from = os.time({ year = 2019, month = 09, day = 10, hour = 0, min = 0, sec = 0 }) - local vl805_timestamp, err = get_vl805_version_timestamp() - if err ~= nil or vl805_timestamp < vl805_supported_from then - err = err or "" - log.warn(string.format( - "System: VL805 firmware version is %s, expected version >= %s %s", - vl805_timestamp, - vl805_supported_from, - err - )) - return - end - -- deactivating any current usb 3.0 connections via deauthorisation - local usb_3_0_used, err = clear_usb_3_0_hub() - if err ~= nil then - -- need to reauth devices just in case - log.error(string.format("System: Error clearing usb 3.0 hub, attempting repopulation, %s", err)) - _, err = repopulate_usb_3_0_hub() - if err ~= nil then - log.error(string.format("System: Error reactivating usb 3.0 connections, %s", err)) - end - err = set_usb_hub_auth_default(true, 2) - if err ~= nil then - log.error(string.format("System: Error default-reauthorising usb 3.0 connections, %s", err)) - end - return - elseif not usb_3_0_used then - -- no need to make any changes - log.info("System: NO_USB3") - return - end - -- default deauthorising usb 3.0 hub, prevents future connections - err = set_usb_hub_auth_default(false, 2) - if err ~= nil then - log.warn(string.format("System: Error default-deauthorising usb 3.0 connections, %s", err)) - end - -- powering down usb 3.0 hub to initiate usb 2.0 connections - err = set_usb_hub_power(false, 2) - if err ~= nil then - -- need to try power up the hub just in case, and reauth devices - log.error(string.format("System: Error powering down usb 3.0 hub, attempting power up, %s", err)) - err = set_usb_hub_power(true, 2) - if err ~= nil then - log.error("System: Error powering up usb 3.0 hub, attempting repopulation, %s", err) - end - _, err = repopulate_usb_3_0_hub() - if err ~= nil then - log.error(string.format("System: Error reactivating usb 3.0 connections, %s", err)) - end - err = set_usb_hub_auth_default(true, 2) - if err ~= nil then log.warn(string.format("System: Error default-reauthorising usb 3.0 connections, %s", err)) end - return - end - -- waiting to see any usb 3.0 devices detected on usb 2.0 before moving on - local detection_retries = 10 - local awaiting_port_1, err = is_device_on_hub_port(2, 1) - if err ~= nil then log.warn(string.format("System: Error detecting device on usb 3.0 hub port: %s ", err)) end - local awaiting_port_2, err = is_device_on_hub_port(2, 2) - if err ~= nil then log.warn(string.format("System: Error detecting device on usb 3.0 hub port: %s", err)) end - for _ = 1, detection_retries do - local port_1_ready = not (awaiting_port_1 and not is_device_on_hub_port(1, 1)) - local port_2_ready = not (awaiting_port_2 and not is_device_on_hub_port(1, 2)) - if port_1_ready and port_2_ready then return else sleep.sleep(1) end - end - log.warn(string.format("System: Deactivated usb 3.0 devices not detected on usb 2.0 hub ports, may be unstable")) -end - -return { - disable_usb3 = disable_usb3 -} diff --git a/src/services/hal/utils.lua b/src/services/hal/utils.lua deleted file mode 100644 index c0e7a94e..00000000 --- a/src/services/hal/utils.lua +++ /dev/null @@ -1,188 +0,0 @@ -local file = require "fibers.stream.file" - -local utils = {} - ----@param path string ----@return string? ----@return string? Error -function utils.read_file(path) - local file, err = file.open(path, "r") - if err then return nil, err end - local content = file:read_all_chars() - - file:close() - return content, nil -end - -function utils.parse_monitor(line) - if line == nil then return nil, nil, 'monitor message is nil' end - local status, address = line:match("^(.-)(/org%S+)") - if address then - return not status:match("-"), address, nil - else - return nil, nil, 'line could not be parsed' - end -end -function utils.parse_modem_monitor(line) - if line == nil then return nil, "Modem monitor message is nil" end - local result = {} - - -- Detect type of the line - if line:match("Initial state") then - result.type = "initial" - elseif line:match("State changed") then - result.type = "changed" - elseif line:match("Removed") then - result.type = "removed" - else - return nil, "Unknown modem monitor message: "..line - end - - -- Extract state - if result.type == "initial" or result.type == "changed" then - local states = {} - for state in line:gmatch("'(%w+)'") do - table.insert(states, state) - end - - - result.prev_state = states[1] - -- If there is no next state then we can define the modem as going from state1 to state1 - result.curr_state = states[2] or states[1] - end - - -- Extract reason if present - local reason = line:match("Reason: ([%w%s]+)") - if reason then - result.reason = reason - end - - return result, nil -end - -function utils.parse_slot_monitor(line) - for card_status, slot_status in line:gmatch("Card status:%s*(%S+).-Slot status:%s*(%S+)") do - if slot_status == "active" then - return card_status, nil - end - end - - return line, 'could not parse (no active slot or invalid string format)' -end -function utils.starts_with(main_string, start_string) - if main_string == nil or start_string == nil then return false end - main_string, start_string = main_string:lower(), start_string:lower() - -- Use string.sub to get the prefix of mainString that is equal in length to startString - return string.sub(main_string, 1, string.len(start_string)) == start_string -end - -local function split_topic(topic) - local components = {} - - for token in string.gmatch(topic, "[^/]+") do - table.insert(components, token) - end - - return components -end - -function utils.parse_control_topic(topic) - local components = split_topic(topic) - if #components < 6 then return nil, nil, nil, 'control topic does not contain enough components' end - - local instance = tonumber(components[4]) - if type(instance) ~= 'number' then return nil, nil, nil, 'failed to convert instance to a number' end - - -- capability, instance, endpoint - return components[3], instance, components[6], nil -end - -function utils.parse_device_info_topic(topic) - local components = split_topic(topic) - if #components < 6 then return nil, nil, nil, 'device info topic does not contain enough components' end - - local instance = tonumber(components[4]) - if type(instance) ~= 'number' then return nil, nil, nil, 'failed to convert instance to a number' end - - return components[3], instance, components[6] -end ---- qmicli output is nasty so convert it to a lua table and remap ugly keys ---- it does not handle qmicli lists as of yet ----@param output string the output of a qmicli command ----@param key_map { string: string }? a mapping of qmicli key names against custom key names ----@return table? qmicli output as a lua table ----@return string? error -function utils.parse_qmicli_output(output, key_map) - if output == nil then return nil, "Output is nil" end - key_map = key_map or {} - - local function clean_key(key) - return key_map[key] or key - end - - local result = {} - local current_table = result - local table_stack = {} - local indent_stack = { 0 } - local line_num = 0 - - for line in output:gmatch("[^\r\n]+") do - local indent = line:match("^%s*"):len() - local cleaned_line = line:match("^%s*(.-)%s*$") - if line_num == 0 then goto continue end - if line:match("^%s*$") then goto continue end - - -- Find the appropriate parent table based on indentation - while #indent_stack > 1 and indent <= indent_stack[#indent_stack] do - table.remove(indent_stack) - table.remove(table_stack) - current_table = #table_stack > 0 and table_stack[#table_stack] or result - end - - if cleaned_line:match(":$") then - local section = cleaned_line:match("^(.+):$") - section = clean_key(section) - current_table[section] = {} - table.insert(table_stack, current_table[section]) - table.insert(indent_stack, indent) - current_table = current_table[section] - else - local key, value = cleaned_line:match("^([^:]+):%s*(.+)$") - if key and value then - key = clean_key(key:match("^%s*(.-)%s*$")) - value = value:match("^%s*(.-)%s*$") - - local num_value = tonumber(value) - if num_value then - value = num_value - elseif value == "yes" then - value = true - elseif value == "no" then - value = false - else - -- remove any quotation marks - value = value:match("^'?(.-)'?$") - end - - current_table[key] = value - end - end - - ::continue:: - line_num = line_num + 1 - end - - return result, nil -end - -function utils.is_in(item, list, key) - if key == nil then key = function(x) return x end end - for _, v in ipairs(list) do - if key(v) == item then - return true - end - end - return false -end - -return utils diff --git a/src/services/http.lua b/src/services/http.lua new file mode 100644 index 00000000..971a289f --- /dev/null +++ b/src/services/http.lua @@ -0,0 +1,26 @@ +-- services/http.lua +-- Public entry point for the HTTP capability service. + +local M = { + service = require 'services.http.service', + sdk = require 'services.http.sdk', + headers = require 'services.http.headers', + config = require 'services.http.config', + policy = require 'services.http.policy', + client = require 'services.http.client', + listener = require 'services.http.listener', + context = require 'services.http.context', + websocket = require 'services.http.websocket', + body = require 'services.http.body', + topics = require 'services.http.topics', +} + +function M.start(conn, opts) + M.service.start(conn, opts) +end + +function M.run(scope, params) + return M.service.run(scope, params) +end + +return M diff --git a/src/services/http/backend.lua b/src/services/http/backend.lua new file mode 100644 index 00000000..1a89b131 --- /dev/null +++ b/src/services/http/backend.lua @@ -0,0 +1,119 @@ +-- services/http/backend.lua +-- Named backend component owner for the HTTP service. +-- +-- The real backend pump is long-running service work with its own identity and +-- completion. It is represented through devicecode.support.scoped_work so that +-- backend failure/termination is reported as ordinary data to the HTTP +-- coordinator. The small start()-only path is retained only for simple unit +-- fakes that have no pump to reap; production drivers should expose run() and +-- terminate(). + +local scoped_work = require 'devicecode.support.scoped_work' + +local M = {} + +local function copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +function M.start(spec) + if type(spec) ~= 'table' then return nil, 'invalid_args' end + local lifetime_scope = spec.lifetime_scope + local driver = spec.driver + local events_port = spec.events_port + if events_port ~= nil and (type(events_port) ~= 'table' or type(events_port.emit_required) ~= 'function') then + return nil, 'events_port_required' + end + local function emit(ev, label) + if not events_port then return true, nil end + return events_port:emit_required(ev, label or 'http_backend_event_report_failed') + end + if not lifetime_scope or type(lifetime_scope.spawn) ~= 'function' then return nil, 'lifetime_scope_required' end + if not driver then return nil, 'driver_required' end + + local identity = copy(spec.identity or {}) + identity.kind = identity.kind or 'backend_done' + identity.component = identity.component or 'backend' + identity.component_id = identity.component_id or 'http_backend' + identity.generation = identity.generation or spec.generation or 1 + + local component = { + driver = driver, + scope = nil, + closed = false, + _work = nil, + _identity = identity, + } + + function component:terminate(reason) + local why = reason or 'backend_terminated' + if self.closed then return true end + self.closed = true + if self._work and type(self._work.cancel) == 'function' then self._work:cancel(why) end + if self.scope and type(self.scope.cancel) == 'function' then self.scope:cancel(why) end + if driver and type(driver.terminate) == 'function' then driver:terminate(why) end + return true + end + + function component:identity() + return copy(self._identity) + end + + function component:outcome_op() + if self._work and type(self._work.outcome_op) == 'function' then return self._work:outcome_op() end + return nil, 'backend_has_no_outcome_op' + end + + function component:outcome() + if self._work and type(self._work.outcome) == 'function' then return self._work:outcome() end + return nil + end + + -- Preferred path: the backend pump is named scoped work. The worker owns the + -- driver pump and installs immediate termination as its finaliser. The + -- coordinator observes the stored completion through a reporter event. + if type(driver.run) == 'function' then + local handle, err = scoped_work.start({ + lifetime_scope = lifetime_scope, + reaper_scope = lifetime_scope, + report_scope = lifetime_scope, + identity = identity, + run = function (scope) + component.scope = scope + scope:finally(function (_, status, primary) + if type(driver.terminate) == 'function' then + driver:terminate(primary or status or 'backend_scope_finalised') + end + end) + + local ok, run_err = driver:run() + if ok == nil or ok == false then error(run_err or 'backend_failed', 0) end + return { reason = 'backend_run_returned' } + end, + report = function (ev) + return emit(ev, 'http_backend_done_report_failed') + end, + }) + if not handle then return nil, err or 'backend_start_failed' end + component._work = handle + emit({ kind = 'backend_ready', component = 'backend', component_id = identity.component_id, generation = identity.generation }, 'http_backend_ready_report_failed') + return component, nil + end + + -- Compatibility path for simple unit fakes. There is no pump, and therefore + -- no meaningful backend completion to reap. This is deliberately not the + -- production backend shape; it exists so isolated service tests can supply a + -- small object with start()/terminate(). + if type(driver.start) == 'function' then + local ok, err = driver:start(lifetime_scope) + if ok ~= true then return nil, err or 'backend_start_failed' end + emit({ kind = 'backend_ready', component = 'backend', component_id = identity.component_id, generation = identity.generation }, 'http_backend_ready_report_failed') + return component, nil + end + + return nil, 'backend_driver_has_no_run_or_start' +end + +return M diff --git a/src/services/http/body.lua b/src/services/http/body.lua new file mode 100644 index 00000000..901ac4fb --- /dev/null +++ b/src/services/http/body.lua @@ -0,0 +1,142 @@ +-- services/http/body.lua +-- Shared HTTP body object-capability helpers. The bus may carry local Lua +-- capabilities with the right shape. It must not carry inline bulk bytes in +-- control-plane fields. + +local fibers = require 'fibers' + +local M = {} + +local function has_method(obj, name) + return type(obj) == 'table' and type(obj[name]) == 'function' +end + +local function reject_inline_bytes(args) + args = args or {} + if args.body ~= nil or args.body_string ~= nil or args.body_chunks ~= nil or args.response_body ~= nil then return nil, 'invalid_args' end + if args.data ~= nil or args.chunk ~= nil or args.chunks ~= nil or args.bytes ~= nil then return nil, 'invalid_args' end + return true, nil +end + +function M.validate_source(source) + if source == nil then return nil, nil end + if not has_method(source, 'read_chunk_op') then return nil, 'invalid_args' end + if not has_method(source, 'terminate') then return nil, 'invalid_args' end + return source, nil +end + +function M.validate_sink(sink) + if sink == nil then return nil, nil end + if not has_method(sink, 'write_chunk_op') then return nil, 'invalid_args' end + if not has_method(sink, 'terminate') then return nil, 'invalid_args' end + if sink.finish_op ~= nil and type(sink.finish_op) ~= 'function' then return nil, 'invalid_args' end + if sink.commit_op ~= nil and type(sink.commit_op) ~= 'function' then return nil, 'invalid_args' end + return sink, nil +end + +function M.validate_exchange_bodies(args) + args = args or {} + local ok, berr = reject_inline_bytes(args) + if not ok then return nil, berr end + local source, serr = M.validate_source(args.body_source) + if serr then return nil, serr end + local sink, skerr = M.validate_sink(args.response_sink) + if skerr then return nil, skerr end + return { source = source, sink = sink }, nil +end + +local function terminate(obj, reason) + if obj and type(obj.terminate) == 'function' then return obj:terminate(reason) end + return true +end + +function M.terminate(obj, reason) + return terminate(obj, reason) +end + +local function unwrap_scope_result(scope_op) + return scope_op:wrap(function (st, _rep, result_or_primary, err) + if st == 'ok' then + if result_or_primary == nil and err ~= nil then return nil, err end + return result_or_primary, nil + end + return nil, result_or_primary or st + end) +end + +function M.copy_response_to_sink_op(response, sink, opts) + opts = opts or {} + return unwrap_scope_result(fibers.run_scope_op(function () + local max_chunk = opts.max_chunk or 65536 + local limit = opts.max_bytes or math.huge + local copied = 0 + while true do + local chunk, err = fibers.perform(response:read_chunk_op(max_chunk)) + if chunk == nil then + if err then return nil, err end + break + end + copied = copied + #chunk + if copied > limit then return nil, opts.too_large_error or 'response_body_too_large' end + local ok, werr = fibers.perform(sink:write_chunk_op(chunk)) + if not ok then return nil, werr end + end + if sink.finish_op then + local ok, ferr = fibers.perform(sink:finish_op()) + if not ok then return nil, ferr end + elseif sink.commit_op then + local ok, ferr = fibers.perform(sink:commit_op()) + if not ok then return nil, ferr end + end + return { bytes = copied } + end)) +end + +function M.drain_response_op(response, opts) + opts = opts or {} + return unwrap_scope_result(fibers.run_scope_op(function () + local max_chunk = opts.max_chunk or 65536 + local limit = opts.max_bytes or math.huge + local copied = 0 + while true do + local chunk, err = fibers.perform(response:read_chunk_op(max_chunk)) + if chunk == nil then + if err then return nil, err end + break + end + copied = copied + #chunk + if copied > limit then return nil, opts.too_large_error or 'response_body_too_large' end + end + return { bytes = copied } + end)) +end + +function M.copy_source_to_sink_op(source, sink, opts) + opts = opts or {} + return unwrap_scope_result(fibers.run_scope_op(function () + local max_chunk = opts.max_chunk or 65536 + local limit = opts.max_bytes or math.huge + local copied = 0 + while true do + local chunk, err = fibers.perform(source:read_chunk_op(max_chunk)) + if chunk == nil then + if err then return nil, err end + break + end + copied = copied + #chunk + if copied > limit then return nil, opts.too_large_error or 'request_body_too_large' end + local ok, werr = fibers.perform(sink:write_chunk_op(chunk)) + if not ok then return nil, werr end + end + if sink.finish_op then + local ok, ferr = fibers.perform(sink:finish_op()) + if not ok then return nil, ferr end + elseif sink.commit_op then + local ok, ferr = fibers.perform(sink:commit_op()) + if not ok then return nil, ferr end + end + return { bytes = copied } + end)) +end + +return M diff --git a/src/services/http/cap_surface.lua b/src/services/http/cap_surface.lua new file mode 100644 index 00000000..048beb14 --- /dev/null +++ b/src/services/http/cap_surface.lua @@ -0,0 +1,71 @@ +-- services/http/cap_surface.lua +-- Bus-facing HTTP capability surface. The surface owns endpoint bindings and +-- retained capability metadata/status; service.lua supplies the handlers. + +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local topics = require 'services.http.topics' + +local M = {} + +local OFFERINGS = { + 'status', + 'listen', + 'open-exchange', + 'exchange', + 'connect-ws', +} + +local function offerings_map() + local out = {} + for _, name in ipairs(OFFERINGS) do out[name] = true end + return out +end + +function M.retain_static(conn, id, status, stats) + conn:retain(topics.meta(id), { + kind = 'cap.http', + class = 'http', + id = id or 'main', + owner = 'http', + methods = offerings_map(), + offerings = offerings_map(), + local_handles = { listen = true, ['open-exchange'] = true, ['connect-ws'] = true }, + control_plane_only = true, + compat = { response_parsers = { strict = true, ['legacy-http1-close'] = true } }, + state = { stats = topics.state(id, 'stats') }, + observability = { status = topics.obs_metric(id, 'status') }, + }) + conn:retain(topics.status(id), status or { state = 'starting', available = false }) + conn:retain(topics.state(id, 'stats'), stats or {}) + conn:retain(topics.obs_metric(id, 'status'), stats or {}) +end + +function M.unretain_static(conn, id) + conn:unretain(topics.meta(id)) + conn:unretain(topics.status(id)) + conn:unretain(topics.state(id, 'stats')) + conn:unretain(topics.obs_metric(id, 'status')) + conn:unretain(topics.obs_metric(id, 'stats')) +end + +function M.bind(conn, id, handlers, opts) + opts = opts or {} + local endpoints = {} + for _, verb in ipairs(OFFERINGS) do + local ep, err = bus_cleanup.bind(conn, topics.rpc(id, verb), opts.endpoint_opts) + if not ep then + for _, prev in pairs(endpoints) do bus_cleanup.unbind(conn, prev) end + return nil, err + end + endpoints[verb] = ep + end + return endpoints +end + +function M.unbind(conn, endpoints) + for _, ep in pairs(endpoints or {}) do bus_cleanup.unbind(conn, ep) end + return true +end + +M.OFFERINGS = OFFERINGS +return M diff --git a/src/services/http/client.lua b/src/services/http/client.lua new file mode 100644 index 00000000..ff13d4b8 --- /dev/null +++ b/src/services/http/client.lua @@ -0,0 +1,209 @@ +-- services/http/client.lua +-- Policy-facing outgoing HTTP client operations. lua-http request machinery is +-- confined to services.http.transport.client; request/response body object +-- capabilities are owned here by operation scopes. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local policy = require 'services.http.policy' +local transport_client = require 'services.http.transport.client' +local request_body = require 'services.http.transport.request_body' +local exchange = require 'services.http.exchange' +local body = require 'services.http.body' + +local M = {} + +local function copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +local function checked_args(args, opts) + if not opts.driver or type(opts.driver.run_op) ~= 'function' then return nil, 'driver_required' end + -- Service operations may pass the already-normalised table returned by + -- policy.validate_exchange_args. Do not send that internal shape back + -- through the public-payload allow-list. + if type(args) == 'table' and args._uri ~= nil then return args end + return policy.validate_exchange_args(args, opts.policy or opts) +end + +local function unwrap(scope_op) + return scope_op:wrap(function (st, _rep, result_or_primary, err) + if st == 'ok' then + if result_or_primary == nil and err ~= nil then return nil, err end + return result_or_primary, nil + end + return nil, result_or_primary or st + end) +end + + +local function install_request_source(scope, checked, opts) + if not checked.body_source then return checked, nil end + + local source = checked.body_source + local source_owned = true + scope:finally(function (_, status, primary) + if source_owned then body.terminate(source, primary or status or 'exchange_source_finalised') end + end) + + if type(source.read_chunk_op) ~= 'function' then + return nil, 'request_body_source_invalid' + end + + local pipe, perr = request_body.new_pipe({ + max_buffered_chunks = opts.max_request_body_buffer_chunks or 8, + condition_factory = opts.request_body_condition_factory, + }) + if not pipe then return nil, perr or 'request_body_pipe_failed' end + + local pipe_owned = true + scope:finally(function (_, status, primary) + if pipe_owned then pipe:terminate(primary or status or 'exchange_request_body_finalised') end + end) + + local spawned, spawn_err = scope:spawn(function () + local copied, cerr = fibers.perform(body.copy_source_to_sink_op(source, pipe, { + max_bytes = opts.max_request_body_bytes or opts.max_request_body or math.huge, + max_chunk = opts.max_request_body_chunk or 65536, + too_large_error = 'request_body_too_large', + })) + if not copied then + pipe:fail(cerr or 'request_body_source_failed') + error(cerr or 'request_body_source_failed', 0) + end + return true + end) + if spawned ~= true then + pipe:terminate(spawn_err or 'request_body_spawn_failed') + return nil, spawn_err or 'request_body_spawn_failed' + end + + local prepared = copy(checked) + prepared._request_body = pipe:body_iterator() + return prepared, { + source = source, + pipe = pipe, + abort_now = function (reason) + local why = reason or 'request_body_aborted' + body.terminate(source, why) + pipe:terminate(why) + return true + end, + mark_done = function () + -- req:go() has returned response headers; lua-http has consumed the + -- request body iterator. The source is still operation-owned and is + -- released by the operation finaliser; the pipe is no longer needed. + pipe_owned = false + pipe:terminate('request_body_consumed') + return true + end, + } +end + +local function open_exchange_result_op(driver, checked, opts) + return unwrap(fibers.run_scope_op(function (scope) + -- lua-http consumes request bodies from a cqueues iterator. The legacy + -- close-delimited transport runs in Fibers and must read the source through + -- its Fibers body capability directly, otherwise the cqueues condition used + -- by request_body.new_pipe can deadlock. + if checked.response_parser == 'legacy-http1-close' then + local prepared = copy(checked) + local source = checked.body_source + local source_owned = source ~= nil + if source_owned then + scope:finally(function (_, status, primary) + if source_owned then body.terminate(source, primary or status or 'legacy_exchange_source_finalised') end + end) + prepared._request_source = source + end + local response_headers, stream_or_err = fibers.perform(transport_client.open_exchange_op(driver, prepared, opts)) + if not response_headers then return nil, stream_or_err end + if source_owned then + source_owned = false + body.terminate(source, 'legacy_exchange_source_consumed') + end + return { response_headers = response_headers, stream = stream_or_err } + end + + local prepared, req_body = install_request_source(scope, checked, opts) + if not prepared then return nil, req_body end + + local response_headers, stream_or_err = fibers.perform(transport_client.open_exchange_op(driver, prepared, opts)) + if not response_headers then + if req_body and req_body.abort_now then req_body.abort_now(stream_or_err or 'open_exchange_failed') end + return nil, stream_or_err + end + if req_body then req_body.mark_done() end + return { response_headers = response_headers, stream = stream_or_err } + end)) +end + +function M.open_exchange_op(driver, args, opts) + opts = opts or {} + return op.guard(function () + local checked, perr = checked_args(args, { driver = driver, policy = opts.policy or opts }) + if not checked then return op.always(nil, perr) end + return open_exchange_result_op(driver, checked, opts):wrap(function (opened, err) + if not opened then return nil, err end + return exchange.make(driver, opened.response_headers, opened.stream, opts), nil + end) + end) +end + +function M.exchange_op(driver, args, opts) + opts = opts or {} + return op.guard(function () + local checked, perr = checked_args(args, { driver = driver, policy = opts.policy or opts }) + if not checked then return op.always(nil, perr) end + return unwrap(fibers.run_scope_op(function (scope) + local sink, ex + local sink_owned, ex_owned = false, false + local sink_final_reason, ex_final_reason = 'exchange_sink_finalised', 'exchange_finalised' + + if checked.response_sink then + sink = checked.response_sink + sink_owned = true + scope:finally(function (_, status, primary) + if sink_owned then body.terminate(sink, sink_final_reason or primary or status or 'exchange_sink_finalised') end + end) + end + + local err + ex, err = fibers.perform(M.open_exchange_op(driver, checked, opts)) + if not ex then sink_final_reason = 'failed'; return nil, err end + ex_owned = true + scope:finally(function (_, status, primary) + if ex_owned and not ex:is_closed() then ex:terminate(ex_final_reason or primary or status or 'exchange_finalised') end + end) + + local sink_result + if sink then + local written, werr = fibers.perform(body.copy_response_to_sink_op(ex, sink, { + max_bytes = checked.max_response_bytes or opts.max_response_body_bytes or opts.max_response_body or math.huge, + max_chunk = opts.max_response_body_chunk or 65536, + })) + if not written then sink_final_reason = 'failed'; ex_final_reason = 'failed'; return nil, werr end + sink_owned = false + sink_result = { bytes = written.bytes } + else + local drained, derr = fibers.perform(body.drain_response_op(ex, { + max_bytes = checked.max_response_bytes or opts.max_response_body_bytes or opts.max_response_body or math.huge, + max_chunk = opts.max_response_body_chunk or 65536, + })) + if not drained then ex_final_reason = 'failed'; return nil, derr end + sink_result = nil + end + + return { + status = ex:status(), + headers = ex:response_headers_table(), + response_sink = sink_result, + } + end)) + end) +end + +M.HttpExchange = exchange.HttpExchange +return M diff --git a/src/services/http/config.lua b/src/services/http/config.lua new file mode 100644 index 00000000..fcd681a5 --- /dev/null +++ b/src/services/http/config.lua @@ -0,0 +1,212 @@ +-- services/http/config.lua +-- +-- Pure HTTP capability-service configuration normaliser. +-- This module performs no I/O and starts no Fibres. + +local M = {} + +M.SCHEMA = 'devicecode.config/http/1' + +local DEFAULTS = { + enabled = true, + id = 'main', + policy = { + allowed_schemes = { http = true, https = true, ws = true, wss = true }, + allowed_response_parsers = { strict = true }, + allow_loopback = true, + max_request_body = 16 * 1024 * 1024, + max_response_body = 16 * 1024 * 1024, + legacy_http1_close_max_response_bytes = 1024 * 1024, + }, + observability = { + status_interval_s = 30, + stats_interval_s = 30, + request_trace = false, + success_events = false, + failure_rate_limit_s = 60, + }, +} + +local ROOT_KEYS = { + schema = true, + enabled = true, + id = true, + policy = true, + observability = true, +} + +local OBSERVABILITY_KEYS = { + status_interval_s = true, + stats_interval_s = true, + request_trace = true, + success_events = true, + failure_rate_limit_s = true, +} + +local POLICY_KEYS = { + allowed_schemes = true, + allowed_hosts = true, + denied_hosts = true, + allowed_response_parsers = true, + allow_loopback = true, + max_request_body = true, + max_response_body = true, + legacy_http1_close_max_response_bytes = true, +} + +local function fail(msg) return nil, msg end + +local function copy_plain(v) + if type(v) ~= 'table' then return v end + local out = {} + for k, subv in pairs(v) do out[k] = copy_plain(subv) end + return out +end + +local function allowed(t, keys, path) + for k in pairs(t or {}) do + if not keys[k] then return nil, path .. ' has unknown field: ' .. tostring(k) end + end + return true, nil +end + +local function bool_or_nil(v, path) + if v == nil then return nil, nil end + if type(v) ~= 'boolean' then return nil, path .. ' must be boolean' end + return v, nil +end + +local function non_empty_string_or_nil(v, path) + if v == nil then return nil, nil end + if type(v) ~= 'string' or v == '' then return nil, path .. ' must be a non-empty string' end + return v, nil +end + +local function positive_number_or_nil(v, path) + if v == nil then return nil, nil end + if type(v) ~= 'number' or v < 0 then return nil, path .. ' must be a non-negative number' end + return v, nil +end + +local function string_bool_map_or_nil(v, path) + if v == nil then return nil, nil end + if type(v) ~= 'table' then return nil, path .. ' must be a table' end + local out = {} + for k, allowed_value in pairs(v) do + if type(k) ~= 'string' or k == '' then return nil, path .. ' keys must be non-empty strings' end + if type(allowed_value) ~= 'boolean' then return nil, path .. ' values must be boolean' end + out[k] = allowed_value + end + return out, nil +end + +local function normalise_policy(raw) + if raw == nil then raw = {} end + if type(raw) ~= 'table' then return fail('policy must be a table') end + local ok, err = allowed(raw, POLICY_KEYS, 'policy') + if not ok then return nil, err end + + local out = copy_plain(DEFAULTS.policy) + local v + + v, err = string_bool_map_or_nil(raw.allowed_schemes, 'policy.allowed_schemes') + if err then return nil, err end + if v ~= nil then out.allowed_schemes = v end + + v, err = string_bool_map_or_nil(raw.allowed_hosts, 'policy.allowed_hosts') + if err then return nil, err end + if v ~= nil then out.allowed_hosts = v end + + v, err = string_bool_map_or_nil(raw.allowed_response_parsers, 'policy.allowed_response_parsers') + if err then return nil, err end + if v ~= nil then out.allowed_response_parsers = v end + + v, err = string_bool_map_or_nil(raw.denied_hosts, 'policy.denied_hosts') + if err then return nil, err end + if v ~= nil then out.denied_hosts = v end + + v, err = bool_or_nil(raw.allow_loopback, 'policy.allow_loopback') + if err then return nil, err end + if v ~= nil then out.allow_loopback = v end + + v, err = positive_number_or_nil(raw.max_request_body, 'policy.max_request_body') + if err then return nil, err end + if v ~= nil then out.max_request_body = v end + + v, err = positive_number_or_nil(raw.max_response_body, 'policy.max_response_body') + if err then return nil, err end + if v ~= nil then out.max_response_body = v end + + v, err = positive_number_or_nil(raw.legacy_http1_close_max_response_bytes, 'policy.legacy_http1_close_max_response_bytes') + if err then return nil, err end + if v ~= nil then out.legacy_http1_close_max_response_bytes = v end + + return out, nil +end + + +local function normalise_observability(raw) + if raw == nil then raw = {} end + if type(raw) ~= 'table' then return fail('observability must be a table') end + local ok, err = allowed(raw, OBSERVABILITY_KEYS, 'observability') + if not ok then return nil, err end + + local out = copy_plain(DEFAULTS.observability) + local v + + v, err = positive_number_or_nil(raw.status_interval_s, 'observability.status_interval_s') + if err then return nil, err end + if v ~= nil then out.status_interval_s = v end + + v, err = positive_number_or_nil(raw.stats_interval_s, 'observability.stats_interval_s') + if err then return nil, err end + if v ~= nil then out.stats_interval_s = v end + + v, err = bool_or_nil(raw.request_trace, 'observability.request_trace') + if err then return nil, err end + if v ~= nil then out.request_trace = v end + + v, err = bool_or_nil(raw.success_events, 'observability.success_events') + if err then return nil, err end + if v ~= nil then out.success_events = v end + + v, err = positive_number_or_nil(raw.failure_rate_limit_s, 'observability.failure_rate_limit_s') + if err then return nil, err end + if v ~= nil then out.failure_rate_limit_s = v end + + return out, nil +end + +function M.normalise(raw) + if raw == nil then raw = {} end + if type(raw) ~= 'table' then return fail('http config must be a table') end + local ok, err = allowed(raw, ROOT_KEYS, 'http config') + if not ok then return nil, err end + + if raw.schema ~= nil and raw.schema ~= M.SCHEMA then + return nil, 'http config schema must be ' .. M.SCHEMA + end + + local enabled, eerr = bool_or_nil(raw.enabled, 'enabled') + if eerr then return nil, eerr end + + local id, ierr = non_empty_string_or_nil(raw.id, 'id') + if ierr then return nil, ierr end + + local policy, perr = normalise_policy(raw.policy) + if not policy then return nil, perr end + + local observability, oerr = normalise_observability(raw.observability) + if not observability then return nil, oerr end + + return { + schema = M.SCHEMA, + enabled = enabled ~= false, + id = id or DEFAULTS.id, + policy = policy, + observability = observability, + }, nil +end + +M.DEFAULTS = DEFAULTS +return M diff --git a/src/services/http/context.lua b/src/services/http/context.lua new file mode 100644 index 00000000..9232067f --- /dev/null +++ b/src/services/http/context.lua @@ -0,0 +1,152 @@ +-- services/http/context.lua +-- Public server-side HTTP context handle. +-- +-- The transport context is deliberately kept behind this wrapper. Consumers get +-- Fibers-native Ops and an immediate termination path; lua-http/cqueues details +-- stay below services/http/transport. + +local M = {} +local HttpContext = {} +HttpContext.__index = HttpContext + +local function registry_id_for(raw) + local id = (raw and type(raw.id) == 'function') and raw:id() or tostring(raw) + return 'ctx' .. tostring(id) +end + +function M.wrap(raw, opts) + if raw == nil then return nil, 'context_required' end + if type(raw) == 'table' and raw._http_public_context then return raw._http_public_context end + + opts = opts or {} + local self = setmetatable({ + _raw = raw, + _listener = opts.listener, + _registry_id = opts.registry_id or registry_id_for(raw), + _on_terminate = opts.on_terminate, + _on_server_websocket = opts.on_server_websocket, + _closed = false, + }, HttpContext) + + if type(raw) == 'table' then raw._http_public_context = self end + return self +end + +function HttpContext:_raw_context() + return self._raw +end + +function HttpContext:_raw_stream_for_test() + return self._raw and self._raw._raw_stream_for_test and self._raw:_raw_stream_for_test() +end + +function HttpContext:_raw_server_for_test() + return self._raw and self._raw._raw_server_for_test and self._raw:_raw_server_for_test() +end + +function HttpContext:id() + return self._raw:id() +end + +function HttpContext:registry_id() + return self._registry_id +end + +function HttpContext:is_closed() + return self._closed or self._raw:is_closed() +end + +function HttpContext:why() + return self._raw:why() +end + +function HttpContext:run_stream_op(label, fn, opts) + return self._raw:run_stream_op(label, function (stream) + return fn(stream, self) + end, opts) +end + +function HttpContext:get_headers_op() + return self._raw:get_headers_op() +end + +function HttpContext:read_chunk_op(max) + return self._raw:read_chunk_op(max) +end + +function HttpContext:read_chars_op(n) + return self._raw:read_chars_op(n) +end + +function HttpContext:read_body_as_string_op() + return self._raw:read_body_as_string_op() +end + +function HttpContext:write_headers_op(headers, end_stream) + return self._raw:write_headers_op(headers, end_stream) +end + +function HttpContext:write_chunk_op(chunk, end_stream) + return self._raw:write_chunk_op(chunk, end_stream) +end + +function HttpContext:write_body_from_string_op(str) + return self._raw:write_body_from_string_op(str) +end + +function HttpContext:peername_op() + return self._raw:peername_op() +end + +function HttpContext:localname_op() + return self._raw:localname_op() +end + +function HttpContext:checktls_op() + return self._raw:checktls_op() +end + +function HttpContext:connection_version_op() + return self._raw:connection_version_op() +end + +function HttpContext:write_continue_op() + return self._raw:write_continue_op() +end + +function HttpContext:unget_op(str) + return self._raw:unget_op(str) +end + +function HttpContext:shutdown_op() + return self._raw:shutdown_op() +end + +function HttpContext:_notify_terminated(reason) + if self._closed then return true end + self._closed = true + local hook = self._on_terminate + self._on_terminate = nil + if hook then hook(self, reason or (self._raw and self._raw:why()) or 'closed') end + return true +end + +function HttpContext:terminate(reason) + local ok, err = self._raw:terminate(reason) + self:_notify_terminated(reason or (self._raw and self._raw:why()) or 'closed') + return ok, err +end + +function HttpContext:_register_server_websocket(ws) + local hook = self._on_server_websocket + if hook then return hook(self, ws) end + return true +end + +function HttpContext:upgrade_websocket_op(headers, opts) + return require('services.http.websocket').from_context_op(self, headers, opts) +end + +M.HttpContext = HttpContext +M.registry_id_for = registry_id_for +return M diff --git a/src/services/http/errors.lua b/src/services/http/errors.lua new file mode 100644 index 00000000..079de578 --- /dev/null +++ b/src/services/http/errors.lua @@ -0,0 +1,35 @@ +-- services/http/errors.lua + +local M = {} + +local stable = { + invalid_args = true, + forbidden = true, + unsupported_scheme = true, + host_denied = true, + backend_unavailable = true, + connect_failed = true, + tls_failed = true, + timeout = true, + request_body_too_large = true, + response_body_too_large = true, + accept_queue_full = true, + closed = true, + cancelled = true, + aborted = true, + not_local = true, + unsupported_remote_handle = true, +} + +function M.normalise(err) + if type(err) == 'table' and err.code then return err end + local s = tostring(err or 'failed') + if stable[s] then return { code = s, message = s } end + return { code = 'backend_unavailable', message = s } +end + +function M.code(err) + return M.normalise(err).code +end + +return M diff --git a/src/services/http/exchange.lua b/src/services/http/exchange.lua new file mode 100644 index 00000000..b29ac514 --- /dev/null +++ b/src/services/http/exchange.lua @@ -0,0 +1,111 @@ +-- services/http/exchange.lua +-- Client-side HTTP exchange handle. Stream operations run inside the cqueues +-- driver so callers compose Fibers Ops rather than lua-http timeouts. + +local op = require 'fibers.op' +local headers_mod = require 'services.http.headers' +local terminate = require 'services.http.transport.terminate' + +local M = {} +local HttpExchange = {} +HttpExchange.__index = HttpExchange + +local function make(driver, response_headers, stream, opts) + return setmetatable({ + _driver = driver, + _headers = response_headers, + _stream = stream, + _opts = opts or {}, + _closed = false, + _close_reason = nil, + _id = opts and opts.id, + _on_terminate = opts and opts.on_terminate, + }, HttpExchange) +end + +function HttpExchange:response_headers() + return self._headers +end + +function HttpExchange:response_headers_table() + return headers_mod.to_table(self._headers) +end + +function HttpExchange:status() + return headers_mod.get_one(self._headers, ':status') +end + +function HttpExchange:is_closed() + return self._closed +end + +function HttpExchange:why() + return self._close_reason +end + +local function stream_timeout(self) + return self._opts and self._opts.intra_stream_timeout or nil +end + +function HttpExchange:_mark_closed(reason) + if self._closed then return false end + self._closed = true + self._close_reason = reason or 'closed' + local hook = self._on_terminate + self._on_terminate = nil + if hook then pcall(hook, self, self._close_reason) end + return true +end + +function HttpExchange:_stream_op(label, fn) + if self._closed then return op.always(nil, self._close_reason or 'closed') end + return self._driver:run_op(label, fn, { + detach_on_abort = true, + on_detach = function (reason) + self:_mark_closed(reason or 'exchange_op_aborted') + end, + on_detached_complete = function (reason) + terminate.terminate_stream(self._stream, reason or 'exchange_op_aborted') + end, + }) +end + +function HttpExchange:read_chunk_op(_max) + return self:_stream_op('http.exchange.read_chunk', function () + return self._stream:get_next_chunk(stream_timeout(self)) + end) +end + +function HttpExchange:read_chars_op(n) + return self:_stream_op('http.exchange.read_chars', function () + return self._stream:get_body_chars(n, stream_timeout(self)) + end) +end + +function HttpExchange:read_body_as_string_op() + return self:_stream_op('http.exchange.read_body_as_string', function () + return self._stream:get_body_as_string(stream_timeout(self)) + end) +end + +function HttpExchange:shutdown_op() + return self:_stream_op('http.exchange.shutdown', function () + if type(self._stream.shutdown) == 'function' then return self._stream:shutdown() end + return true + end):wrap(function (ok, err) + self:_mark_closed(err or 'closed') + return ok, err + end) +end + +function HttpExchange:terminate(reason) + if self._closed then return true end + self:_mark_closed(reason or 'closed') + terminate.terminate_stream(self._stream, self._close_reason) + return true +end + +M.make = make +M.HttpExchange = HttpExchange + +return M diff --git a/src/services/http/headers.lua b/src/services/http/headers.lua new file mode 100644 index 00000000..0e686abb --- /dev/null +++ b/src/services/http/headers.lua @@ -0,0 +1,3 @@ +-- services/http/headers.lua +-- Public header boundary. Backend-specific construction lives in transport. +return require 'services.http.transport.headers' diff --git a/src/services/http/listener.lua b/src/services/http/listener.lua new file mode 100644 index 00000000..52fce475 --- /dev/null +++ b/src/services/http/listener.lua @@ -0,0 +1,163 @@ +-- services/http/listener.lua +-- Public listener handle boundary above the lua-http transport listener. + +local lua_http = require 'services.http.transport.lua_http' +local context_mod = require 'services.http.context' + +local M = {} +local HttpListener = {} +HttpListener.__index = HttpListener + +local function copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +local function make(raw, opts) + opts = opts or {} + local self = setmetatable({ + _raw = raw, + _contexts = setmetatable({}, { __mode = 'k' }), + _admitted_raw = setmetatable({}, { __mode = 'k' }), + _events_port = opts.events_port, + }, HttpListener) + return self +end + +function HttpListener:_wrap_context(raw) + if raw == nil then return nil end + local ctx = self._contexts[raw] + if ctx == nil then + ctx = assert(context_mod.wrap(raw, { + listener = self, + on_server_websocket = function (public_ctx, ws) + return self:_emit_context_event('server_websocket_registered', public_ctx, { websocket = ws }) + end, + on_terminate = function (public_ctx, reason) + return self:_emit_context_event('context_terminated', public_ctx, { reason = reason }) + end, + })) + self._contexts[raw] = ctx + end + return ctx +end + +function M.listen(opts) + opts = opts or {} + local transport_opts = copy(opts) + transport_opts.on_context = nil + transport_opts.on_context_admitted = nil + transport_opts.on_context_transferred = nil + transport_opts.on_context_terminated = nil + transport_opts.on_server_websocket = nil + + local raw, err = lua_http.listen(transport_opts) + if not raw then return nil, err end + return make(raw, opts) +end + + +local function context_id_of(ctx) + if ctx and type(ctx.id) == 'function' then return ctx:id() end + if ctx and ctx.id ~= nil then return ctx.id end + return nil +end + +function HttpListener:_emit_context_event(kind, ctx, extra) + local ev = { + kind = kind, + ctx = ctx, + context_id = context_id_of(ctx), + } + for k, v in pairs(extra or {}) do ev[k] = v end + local port = self._events_port + if port and type(port.emit_required) == 'function' then + return port:emit_required(ev, 'http_listener_context_event_report_failed') + end + + return true +end + +function HttpListener:_raw_listener() + return self._raw +end + +function HttpListener:_raw_server_for_test() + return self._raw and self._raw:_raw_server_for_test() +end + +function HttpListener:localname() + return self._raw:localname() +end + +function HttpListener:is_closed() + return self._raw:is_closed() +end + +function HttpListener:why() + return self._raw:why() +end + +function HttpListener:listen_op() + return self._raw:listen_op() +end + +function HttpListener:pump_once_op() + return self._raw:pump_once_op() +end + +function HttpListener:run() + return self._raw:run() +end + +function HttpListener:start(scope) + return self._raw:start(scope) +end + +function HttpListener:_admit_raw_context(raw_ctx) + if raw_ctx == nil then return nil, 'context_required' end + local ctx = self:_wrap_context(raw_ctx) + if not self._admitted_raw[raw_ctx] then + local ok, err = self:_emit_context_event('context_admitted', ctx) + if ok == nil or ok == false then return nil, err or 'context_admission_report_failed' end + self._admitted_raw[raw_ctx] = true + end + if ctx:is_closed() then ctx:_notify_terminated(ctx:why() or 'closed') end + return ctx, nil +end + +function HttpListener:context_admission_op() + return self._raw:admission_op():wrap(function (raw_ctx, err) + if raw_ctx == nil then return nil, err end + return self:_admit_raw_context(raw_ctx) + end) +end + +function HttpListener:accept_op() + return self._raw:accept_op():wrap(function (raw_ctx, err) + if raw_ctx == nil then return nil, err end + local ctx, aerr = self:_admit_raw_context(raw_ctx) + if ctx == nil then return nil, aerr end + local ok, terr = self:_emit_context_event('context_transferred', ctx) + if ok == nil or ok == false then return nil, terr or 'context_transfer_report_failed' end + return ctx, nil + end) +end + +function HttpListener:pause() + return self._raw:pause() +end + +function HttpListener:resume() + return self._raw:resume() +end + +function HttpListener:terminate(reason) + return self._raw:terminate(reason) +end + +M.listen = M.listen +M.HttpListener = HttpListener +M.Listener = HttpListener +return M diff --git a/src/services/http/listener_owner.lua b/src/services/http/listener_owner.lua new file mode 100644 index 00000000..a0e13c7d --- /dev/null +++ b/src/services/http/listener_owner.lua @@ -0,0 +1,157 @@ +-- services/http/listener_owner.lua +-- Named listener runtime owner. +-- +-- A listen RPC performs setup only until the listener is bound. The listener +-- runtime then lives in a child owner scope under the HTTP service. Accepted +-- contexts transfer to caller/request scopes via accept_op(). Runtime +-- completion is reported as an identity-bearing scoped-work completion. + +local fibers = require 'fibers' +local cond = require 'fibers.cond' + +local listener_mod = require 'services.http.listener' +local scoped_work = require 'devicecode.support.scoped_work' + +local M = {} + +local function copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +local function make_listener(opts) + local made, err = listener_mod.listen(opts) + if not made then return nil, err or 'listen_create_failed' end + return made, nil +end + +function M.start(spec) + if type(spec) ~= 'table' then return nil, 'invalid_args' end + local lifetime_scope = spec.lifetime_scope + if not lifetime_scope or type(lifetime_scope.child) ~= 'function' then return nil, 'lifetime_scope_required' end + + local opts = copy(spec.listen_opts or {}) + local events_port = spec.events_port + local identity = { + kind = 'listener_done', + handle_id = spec.handle_id, + generation = spec.generation or 1, + } + if events_port ~= nil and (type(events_port) ~= 'table' or type(events_port.emit_required) ~= 'function') then + return nil, 'events_port_required' + end + local function emit(ev, label) + if not events_port then return true, nil end + return events_port:emit_required(ev, label or 'http_listener_owner_event_report_failed') + end + + local ready = cond.new() + local ready_done = false + local ready_listener, ready_err + + local function signal_ready(listener, err) + if ready_done then return end + ready_done = true + ready_listener = listener + ready_err = err + ready:signal() + end + + local handle, err, setup = scoped_work.start({ + lifetime_scope = lifetime_scope, + reaper_scope = lifetime_scope, + report_scope = lifetime_scope, + identity = identity, + + setup = function () + local listener, lerr = make_listener(opts) + if not listener then return nil, lerr or 'listen_create_failed' end + return { + listener = listener, + cancel_owned_now = function (reason) + listener:terminate(reason or 'listener_start_cancelled') + return true + end, + } + end, + + run = function (scope, run_setup) + local listener = assert(run_setup.listener) + scope:finally(function (_, status, primary) + listener:terminate(primary or status or 'listener_scope_finalised') + end) + + local ok, lerr = fibers.perform(listener:listen_op()) + if not ok then + listener:terminate(lerr or 'listener_listen_failed') + signal_ready(nil, lerr or 'listener_listen_failed') + error(lerr or 'listener_listen_failed', 0) + end + + local spawned, spawn_err = scope:spawn(function () + while true do + local ctx, cerr = fibers.perform(listener:context_admission_op()) + if ctx == nil then + if cerr == nil or tostring(cerr):match('closed') or tostring(cerr):match('listener') then + return true + end + error(cerr, 0) + end + end + end) + if spawned ~= true then + listener:terminate(spawn_err or 'listener_context_manager_spawn_failed') + signal_ready(nil, spawn_err or 'listener_context_manager_spawn_failed') + error(spawn_err or 'listener_context_manager_spawn_failed', 0) + end + + emit({ + kind = 'listener_owner_started', + handle_id = identity.handle_id, + generation = identity.generation, + }) + signal_ready(listener, nil) + + local rok, rerr = listener:run() + if not rok then error(rerr or 'listener_runtime_failed', 0) end + return { + handle_id = identity.handle_id, + reason = listener:why() or 'listener_runtime_ended', + } + end, + + report = function (ev) + return emit(ev, 'http_listener_owner_done_report_failed') + end, + }) + + if not handle then + return nil, err or 'listener_start_failed' + end + + local listener, wait_err = fibers.perform(ready:wait_op():wrap(function () + return ready_listener, ready_err + end):on_abort(function () + handle:cancel('listener_start_cancelled') + end)) + if not listener then + handle:cancel(wait_err or 'listener_start_failed') + return nil, wait_err or 'listener_start_failed' + end + + return { + listener = listener, + scope = setup and setup.scope, + work = handle, + cancel = function (_, reason) + handle:cancel(reason or 'listener_cancelled') + listener:terminate(reason or 'listener_cancelled') + return true + end, + outcome_op = function () return handle:outcome_op() end, + outcome = function () return handle:outcome() end, + }, nil +end + +return M diff --git a/src/services/http/model.lua b/src/services/http/model.lua new file mode 100644 index 00000000..bb44cc37 --- /dev/null +++ b/src/services/http/model.lua @@ -0,0 +1,40 @@ +-- services/http/model.lua +-- Pulse-backed HTTP service model. + +local base_model = require 'devicecode.support.model' +local tablex = require 'shared.table' + +local M = {} + +local function equals(a, b) + for k, v in pairs(a or {}) do if (b or {})[k] ~= v then return false end end + for k in pairs(b or {}) do if (a or {})[k] == nil then return false end end + return true +end + +function M.initial() + return { + state = 'starting', + backend = 'starting', + ready = false, + active_listeners = 0, + active_contexts = 0, + active_exchanges = 0, + active_websockets = 0, + completed_exchanges = 0, + failed_exchanges = 0, + rejected_requests = 0, + last_error = nil, + policy_generation = 1, + } +end + +function M.new(initial) + return base_model.new(initial or M.initial(), { + copy = tablex.shallow_copy, + equals = equals, + label = 'http.model', + }) +end + +return M diff --git a/src/services/http/operation_owner.lua b/src/services/http/operation_owner.lua new file mode 100644 index 00000000..4ef510d2 --- /dev/null +++ b/src/services/http/operation_owner.lua @@ -0,0 +1,115 @@ +-- services/http/operation_owner.lua +-- Request-owner and scoped_work glue for HTTP capability operations. + +local request_owner = require 'devicecode.support.request_owner' +local scoped_work = require 'devicecode.support.scoped_work' +local service_events = require 'devicecode.support.service_events' + +local M = {} + +function M.next_request(service, verb, req) + service._next_request_id = service._next_request_id + 1 + local request_id = 'req' .. tostring(service._next_request_id) + local owner = request_owner.new(req) + service._owned_requests[request_id] = owner + return request_id, owner +end + +function M.operation_identity(service, verb, request_id) + service._next_operation_id = service._next_operation_id + 1 + return { + kind = 'http_operation_done', + operation = verb, + operation_id = 'op' .. tostring(service._next_operation_id), + request_id = request_id, + generation = service._generation, + } +end + +function M.operation_setup(service, owner) + return function (scope) + scope:finally(function (_, status, primary) + if status ~= 'ok' then + owner:finalise_unresolved(primary or status or 'http_request_finalised') + end + end) + return { + owner = owner, + reserved_handles = {}, + cancel_owned_now = function (reason) + if not owner:done() then + owner:finalise_unresolved(reason or 'scoped_work_start_failed') + end + return true + end, + } + end +end + +function M.start(service, verb, req, request_id, owner, run) + local identity = M.operation_identity(service, verb, request_id) + service._state.operations[identity.operation_id] = { + operation_id = identity.operation_id, + generation = identity.generation, + operation = verb, + request_id = request_id, + state = 'admitted', + } + local reqrec = service._state.requests[request_id] + if reqrec then reqrec.state = 'running'; reqrec.operation_id = identity.operation_id end + service:_log_event { kind = 'operation_started', operation = verb, operation_id = identity.operation_id, request_id = request_id, generation = identity.generation } + + local setup_fn = function (scope) + local setup = M.operation_setup(service, owner)(scope) + local old_cancel = setup.cancel_owned_now + setup.cancel_owned_now = function (reason) + old_cancel(reason) + for _, handle_id in ipairs(setup.reserved_handles or {}) do service:_terminate_handle(handle_id, reason or 'operation_start_failed') end + return true + end + return setup + end + + local events_port = service._event_port and service:_event_port({ + source = 'http_operation', + source_id = identity.operation_id, + operation_id = identity.operation_id, + operation = verb, + request_id = request_id, + generation = identity.generation, + }, { label = 'http_operation_done_report_failed' }) + local report_event = events_port and service_events.reporter(events_port, 'http_operation_done_report_failed') + or function (ev) return service:_submit_event(ev, 'http_operation_done_report_failed', { fatal = true }) end + + local cancel_op = owner.caller_cancel_op and owner:caller_cancel_op() or nil + + local handle, err = scoped_work.start({ + lifetime_scope = service._scope, + reaper_scope = service._scope, + report_scope = service._scope, + identity = identity, + setup = setup_fn, + run = run, + report = report_event, + cancel_op = cancel_op, + }) + if not handle then + local rec = service._state.operations[identity.operation_id] + if rec then + rec.state = 'completed' + rec.status = 'failed' + rec.primary = err or 'operation_start_failed' + if service._account_completed_operation then service:_account_completed_operation(rec, 'failed') end + service._state.operations[identity.operation_id] = nil + end + service:_finish_request(request_id, 'failed', err or 'operation_start_failed') + owner:finalise_unresolved(err or 'operation_start_failed') + service._state.last_error = err or 'operation_start_failed' + return true + end + service._state.operations[identity.operation_id].state = 'running' + service._state.operations[identity.operation_id].handle = handle + return true +end + +return M diff --git a/src/services/http/operations.lua b/src/services/http/operations.lua new file mode 100644 index 00000000..f5548283 --- /dev/null +++ b/src/services/http/operations.lua @@ -0,0 +1,203 @@ +-- services/http/operations.lua +-- Capability operation validation and worker bodies. Coordinator code calls this +-- module to admit named scoped work; workers may perform Ops inside their own +-- scopes. + +local fibers = require 'fibers' + +local client = require 'services.http.client' +local websocket = require 'services.http.websocket' +local policy = require 'services.http.policy' +local listener_owner = require 'services.http.listener_owner' +local operation_owner = require 'services.http.operation_owner' +local service_events = require 'devicecode.support.service_events' + +local M = {} +local perform = fibers.perform + +local function copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + + +local function event_port_from_submit(tx, submit, identity, default_label) + local events_port = service_events.port(tx, identity, { + label = default_label or 'http_event_report_failed', + }) + return { + emit_required = function (_, ev, label) + if type(ev) ~= 'table' then return nil, 'invalid_event' end + return submit(events_port:event(ev), label or default_label or 'http_event_report_failed') + end, + event = function (_, ev, attrs) + return events_port:event(ev, attrs) + end, + identity = function () + return events_port:identity() + end, + } +end + +local function terminate_handle(h, reason) + if h and type(h.terminate) == 'function' then return h:terminate(reason) end + return true +end + +local function assert_local(service, req) + return policy.require_local_origin(req and req.origin) +end + +local function policy_opts(service) + return service._opts.policy or service._opts +end + +local function normalise_verb(verb) + return tostring(verb or ''):gsub('-', '_') +end + +function M.validate_cap_request(service, verb, req) + verb = normalise_verb(verb) + if verb == 'status' then return true end + if verb == 'listen' or verb == 'open_exchange' or verb == 'connect_ws' then + local ok, lerr = assert_local(service, req) + if not ok then return nil, lerr end + end + if verb == 'listen' then + local args, err = policy.validate_listen_args(req.payload or {}) + if not args then return nil, err end + req.payload = args + elseif verb == 'open_exchange' or verb == 'exchange' then + local checked, err = policy.validate_exchange_args(req.payload or {}, policy_opts(service)) + if not checked then return nil, err end + req.payload = checked + elseif verb == 'connect_ws' then + local checked, err = policy.validate_connect_ws_args(req.payload or {}, policy_opts(service)) + if not checked then return nil, err end + req.payload = checked + else + return nil, 'invalid_args' + end + return true +end + +local function run_listen(service, scope, req, setup) + local owner = setup.owner + local args = req.payload or {} + local opts = copy(args) + opts.driver = service._driver + opts.http_server = service._opts.http_server + opts.backend_timeout = service._opts.backend_timeout + opts.connection_setup_timeout = service._opts.connection_setup_timeout + opts.intra_stream_timeout = service._opts.intra_stream_timeout + opts.context_terminator = service._opts.context_terminator + opts.max_accept_queue = args.max_accept_queue or service._opts.max_accept_queue or 64 + + local handle_id = assert(service:_reserve_handle('listener', { owner = 'http_service' })) + table.insert(setup.reserved_handles, handle_id) + local generation = service._generation + opts.events_port = event_port_from_submit(service._event_tx, function (ev, label) + return service:_submit_registry_event(ev, label or 'http_listener_context_event_report_failed') + end, { + service_id = service._id, + source = 'http_listener', + source_id = handle_id, + listener_id = handle_id, + handle_id = handle_id, + generation = generation, + }, 'http_listener_context_event_report_failed') + local component, cerr = listener_owner.start({ + lifetime_scope = service._scope, + listen_opts = opts, + handle_id = handle_id, + generation = generation, + events_port = service:_event_port({ + source = 'http_listener_owner', + source_id = handle_id, + listener_id = handle_id, + handle_id = handle_id, + generation = generation, + }, { + label = 'http_listener_owner_event_report_failed', + }), + }) + if not component then + owner:fail_once(cerr) + service:_remove_handle(handle_id, cerr or 'listener_start_failed') + error(cerr or 'listener_start_failed', 0) + end + + scope:finally(function (_, status, primary) + if status ~= 'ok' then component:cancel(primary or status or 'listen_setup_finalised') end + end) + + service:_register_handle('listener', component.listener, { id = handle_id, owner = 'http_service' }) + return { listener = component.listener, handle_id = handle_id } +end + +local function run_open_exchange(service, scope, req, setup) + local owner = setup.owner + local handle_id = assert(service:_reserve_handle('exchange', { owner = 'http_service' })) + table.insert(setup.reserved_handles, handle_id) + local opts = copy(service._opts) + for k, v in pairs(opts.policy or {}) do opts[k] = v end + opts.origin = req.origin + opts.principal = req.origin and req.origin.principal + opts.on_terminate = function (_, reason) + service:_remove_handle(handle_id, reason or 'exchange_terminated') + end + local ex, err = perform(client.open_exchange_op(service._driver, req.payload or {}, opts)) + if not ex then owner:fail_once(err); service:_remove_handle(handle_id, err or 'open_exchange_failed'); error(err or 'open_exchange_failed', 0) end + + scope:finally(function (_, status, primary) + if status ~= 'ok' then terminate_handle(ex, primary or status or 'open_exchange_finalised') end + end) + + service:_register_handle('exchange', ex, { id = handle_id, owner = 'http_service' }) + return { exchange = ex, handle_id = handle_id } +end + +local function run_exchange(service, _scope, req, _setup) + local opts = copy(service._opts) + for k, v in pairs(opts.policy or {}) do opts[k] = v end + opts.origin = req.origin + opts.principal = req.origin and req.origin.principal + local result, err = perform(client.exchange_op(service._driver, req.payload or {}, opts)) + if not result then error(err or 'exchange_failed', 0) end + return { result = result } +end + +local function run_connect_ws(service, scope, req, setup) + local owner = setup.owner + local handle_id = assert(service:_reserve_handle('websocket', { owner = 'http_service' })) + table.insert(setup.reserved_handles, handle_id) + local opts = copy(service._opts) + opts.origin = req.origin + opts.principal = req.origin and req.origin.principal + opts.on_terminate = function (_, reason) + service:_remove_handle(handle_id, reason or 'websocket_terminated') + end + local ws, err = perform(websocket.connect_op(service._driver, req.payload or {}, opts)) + if not ws then owner:fail_once(err); service:_remove_handle(handle_id, err or 'connect_ws_failed'); error(err or 'connect_ws_failed', 0) end + + scope:finally(function (_, status, primary) + if status ~= 'ok' then terminate_handle(ws, primary or status or 'connect_ws_finalised') end + end) + + service:_register_handle('websocket', ws, { id = handle_id, owner = 'http_service' }) + return { websocket = ws, handle_id = handle_id } +end + +function M.start_operation(service, verb, req, request_id, owner) + verb = normalise_verb(verb) + local run + if verb == 'listen' then run = function (scope, setup) return run_listen(service, scope, req, setup) end + elseif verb == 'open_exchange' then run = function (scope, setup) return run_open_exchange(service, scope, req, setup) end + elseif verb == 'exchange' then run = function (scope, setup) return run_exchange(service, scope, req, setup) end + elseif verb == 'connect_ws' then run = function (scope, setup) return run_connect_ws(service, scope, req, setup) end + else return service:_reject_request(request_id, owner, 'invalid_args') end + return operation_owner.start(service, verb, req, request_id, owner, run) +end + +return M diff --git a/src/services/http/policy.lua b/src/services/http/policy.lua new file mode 100644 index 00000000..5a9db6da --- /dev/null +++ b/src/services/http/policy.lua @@ -0,0 +1,237 @@ +-- services/http/policy.lua +-- Validation, locality and basic egress policy. This module is deliberately +-- conservative: it rejects before backend admission where it can. + +local body = require 'services.http.body' +local uri_util = require 'services.http.transport.uri' + +local M = {} + +local SAFE_METHODS = { GET = true, HEAD = true, OPTIONS = true } +local METHODS = { + GET = true, HEAD = true, POST = true, PUT = true, PATCH = true, + DELETE = true, OPTIONS = true, +} + +local function copy_headers(h) + if h == nil then return nil end + if type(h) ~= 'table' then return nil, 'invalid_args' end + local out = {} + for k, v in pairs(h) do + if type(k) ~= 'string' or k == '' or k:find('[\r\n:]') then return nil, 'invalid_args' end + if type(v) ~= 'string' and type(v) ~= 'number' then return nil, 'invalid_args' end + v = tostring(v) + if v:find('[\r\n]') then return nil, 'invalid_args' end + out[k] = v + end + return out, nil +end + +local function bool_or_nil(v) + if v == nil then return nil, nil end + if type(v) ~= 'boolean' then return nil, 'invalid_args' end + return v, nil +end + +local function non_negative_number_or_nil(v) + if v == nil then return nil, nil end + if type(v) ~= 'number' or v < 0 then return nil, 'invalid_args' end + return v, nil +end + +local function reject_backend_payload_fields(args) + local forbidden = { + server = true, server_options = true, http_server = true, request_module = true, + websocket_module = true, cq = true, driver = true, socket = true, onstream = true, + onerror = true, condition_factory = true, context_terminator = true, + on_context = true, on_context_transferred = true, on_context_terminated = true, + on_terminate = true, stream = true, ws = true, exchange = true, listener = true, + } + for k in pairs(args or {}) do + if forbidden[k] then return nil, 'invalid_args' end + end + return true +end + +local function require_only_fields(args, allowed) + local ok, ferr = reject_backend_payload_fields(args) + if not ok then return nil, ferr end + for k in pairs(args or {}) do + if not allowed[k] then return nil, 'invalid_args' end + end + return true +end + +function M.is_local_origin(origin) + return origin + and origin.kind == 'local' + and origin.link_id == nil + and origin.peer_node == nil + and origin.peer_sid == nil +end + +function M.require_local_origin(origin) + if M.is_local_origin(origin) then return true, nil end + return nil, 'not_local' +end + + +function M.validate_method(method) + method = tostring(method or 'GET'):upper() + if not METHODS[method] then return nil, 'invalid_args' end + return method +end + +function M.is_safe_method(method) + return SAFE_METHODS[tostring(method or ''):upper()] or false +end + +function M.validate_uri(uri, opts) + opts = opts or {} + local parsed, perr = uri_util.parse(uri) + if not parsed then return nil, perr or 'invalid_args' end + + local allowed = opts.allowed_schemes or { http = true, https = true, ws = true, wss = true } + if not allowed[parsed.scheme] then return nil, 'unsupported_scheme' end + + return { + uri = parsed.uri, + scheme = parsed.scheme, + authority = parsed.authority, + host = parsed.host, + port = parsed.port, + path = parsed.path, + }, nil +end + +function M.validate_listen_args(args) + args = args or {} + if type(args) ~= 'table' then return nil, 'invalid_args' end + local ok, ferr = require_only_fields(args, { host = true, port = true, path = true, tls = true, max_accept_queue = true }) + if not ok then return nil, ferr end + local out = {} + if args.host ~= nil and type(args.host) ~= 'string' then return nil, 'invalid_args' end + if args.port ~= nil and (type(args.port) ~= 'number' or args.port < 0 or args.port > 65535) then + return nil, 'invalid_args' + end + if args.path ~= nil and type(args.path) ~= 'string' then return nil, 'invalid_args' end + if args.tls ~= nil and type(args.tls) ~= 'boolean' then return nil, 'invalid_args' end + if args.max_accept_queue ~= nil and (type(args.max_accept_queue) ~= 'number' or args.max_accept_queue < 0) then return nil, 'invalid_args' end + out.host = args.host + out.port = args.port + out.path = args.path + out.tls = args.tls + out.max_accept_queue = args.max_accept_queue + if out.host == nil and out.path == nil then out.host = '127.0.0.1' end + if out.port == nil and out.path == nil then out.port = 0 end + if out.tls == nil then out.tls = false end + return out, nil +end + +local function host_denied(parsed_uri, opts) + local host = parsed_uri and parsed_uri.host or nil + if host == nil then return true end + if opts.denied_hosts and opts.denied_hosts[host] then return true end + if opts.allowed_hosts and not opts.allowed_hosts[host] then return true end + if opts.allow_loopback == false and (host == '127.0.0.1' or host == 'localhost' or host == '::1') then return true end + return false +end + +local function response_parser_allowed(parser, opts) + parser = parser or 'strict' + local allowed = opts.allowed_response_parsers or { strict = true } + return allowed[parser] == true +end + +local function validate_response_parser(v, opts) + if v == nil then v = 'strict' end + if v ~= 'strict' and v ~= 'legacy-http1-close' then return nil, 'invalid_args' end + if not response_parser_allowed(v, opts or {}) then return nil, 'response_parser_denied' end + return v, nil +end + +local function positive_number_or_nil(v) + if v == nil then return nil, nil end + if type(v) ~= 'number' or v <= 0 then return nil, 'invalid_args' end + return v, nil +end + +function M.validate_exchange_args(args, opts) + opts = opts or {} + if type(args) ~= 'table' then return nil, 'invalid_args' end + local ok, ferr = require_only_fields(args, { + uri = true, method = true, headers = true, body_source = true, response_sink = true, + expect_100_continue = true, expect_100_timeout = true, + response_parser = true, timeout_s = true, max_response_bytes = true, + }) + if not ok then return nil, ferr end + local uri, uerr = M.validate_uri(args.uri, opts) + if not uri then return nil, uerr end + if uri.scheme == 'ws' or uri.scheme == 'wss' then return nil, 'unsupported_scheme' end + if host_denied(uri, opts) then return nil, 'host_denied' end + local method, merr = M.validate_method(args.method or 'GET') + if not method then return nil, merr end + local headers, herr = copy_headers(args.headers) + if herr then return nil, herr end + + local expect_100_continue, eerr = bool_or_nil(args.expect_100_continue) + if eerr then return nil, eerr end + + local expect_100_timeout, terr = non_negative_number_or_nil(args.expect_100_timeout) + if terr then return nil, terr end + + local response_parser, rperr = validate_response_parser(args.response_parser, opts) + if rperr then return nil, rperr end + + local timeout_s, toerr = positive_number_or_nil(args.timeout_s) + if toerr then return nil, toerr end + + local max_response_bytes, mrerr = positive_number_or_nil(args.max_response_bytes) + if mrerr then return nil, mrerr end + + if response_parser == 'legacy-http1-close' then + if uri.scheme ~= 'http' then return nil, 'unsupported_scheme' end + if method ~= 'GET' and method ~= 'POST' and method ~= 'HEAD' then return nil, 'unsupported_method' end + if timeout_s == nil then return nil, 'timeout_required' end + local policy_max = opts.legacy_http1_close_max_response_bytes or opts.max_response_body or (1024 * 1024) + if type(policy_max) ~= 'number' or policy_max <= 0 then return nil, 'invalid_args' end + if max_response_bytes == nil then max_response_bytes = policy_max end + if max_response_bytes > policy_max then return nil, 'response_too_large' end + end + + local bodies, derr = body.validate_exchange_bodies(args) + if not bodies then return nil, derr end + + return { + uri = uri.uri, + method = method, + headers = headers, + _uri = uri, + body_source = bodies.source, + response_sink = bodies.sink, + expect_100_continue = expect_100_continue, + expect_100_timeout = expect_100_timeout, + response_parser = response_parser, + timeout_s = timeout_s, + max_response_bytes = max_response_bytes, + }, nil +end + +function M.validate_connect_ws_args(args, opts) + opts = opts or {} + if type(args) ~= 'table' then return nil, 'invalid_args' end + local ok, ferr = require_only_fields(args, { uri = true, headers = true }) + if not ok then return nil, ferr end + local uri, uerr = M.validate_uri(args.uri, opts) + if not uri then return nil, uerr end + if uri.scheme ~= 'ws' and uri.scheme ~= 'wss' then return nil, 'unsupported_scheme' end + if host_denied(uri, opts) then return nil, 'host_denied' end + local headers, herr = copy_headers(args.headers) + if herr then return nil, herr end + return { uri = uri.uri, headers = headers, _uri = uri }, nil +end + +M._reject_backend_payload_fields = reject_backend_payload_fields +M._require_only_fields = require_only_fields + +return M diff --git a/src/services/http/registry.lua b/src/services/http/registry.lua new file mode 100644 index 00000000..1343b7c3 --- /dev/null +++ b/src/services/http/registry.lua @@ -0,0 +1,202 @@ +-- services/http/registry.lua +-- HTTP handle registry. The registry is an accounting and shutdown backstop; +-- callers own normal protocol use after handoff. +-- +-- The registry is deliberately not the service reducer. It emits identity- +-- bearing events and provides immediate, idempotent termination/removal paths. + +local M = {} +local Registry = {} +Registry.__index = Registry + +local function copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +local function copy_record(rec) + return { + kind = rec.kind, + handle = rec.handle, + handle_id = rec.handle_id, + generation = rec.generation, + owner = rec.owner, + state = rec.state, + listener_id = rec.listener_id, + context_id = rec.context_id, + } +end + +local function event_for(rec, suffix, extra) + local kind = rec.kind .. '_' .. suffix + if rec.kind == 'context' and suffix == 'registered' then kind = 'context_admitted' end + local ev = { + kind = kind, + handle_id = rec.handle_id, + generation = rec.generation, + listener_id = rec.listener_id, + context_id = rec.context_id, + } + for k, v in pairs(extra or {}) do ev[k] = v end + return ev +end + +function M.new(opts) + opts = opts or {} + return setmetatable({ + _next_id = 0, + _records = {}, + _events_port = opts.events_port, + }, Registry) +end + +function Registry:_new_id() + while true do + self._next_id = self._next_id + 1 + local id = 'h' .. tostring(self._next_id) + if self._records[id] == nil then return id end + end +end + +function Registry:_emit(ev) + local port = self._events_port + if port and type(port.emit_required) == 'function' then + return port:emit_required(copy(ev), 'http_registry_event_report_failed') + end + + return true +end + +local function apply_opts(rec, opts) + rec.generation = opts.generation or rec.generation or 1 + rec.owner = opts.owner or rec.owner or 'http_service' + if opts.listener_id ~= nil then rec.listener_id = opts.listener_id end + if opts.context_id ~= nil then rec.context_id = opts.context_id end + return rec +end + +function Registry:reserve(kind, opts) + opts = opts or {} + local id = opts.id or self:_new_id() + if self._records[id] ~= nil then return nil, 'handle_exists' end + local rec = apply_opts({ + kind = kind, + handle = nil, + handle_id = id, + state = 'reserved', + }, opts) + self._records[id] = rec + return id, copy_record(rec) +end + +function Registry:register(kind, handle, opts) + opts = opts or {} + local id = opts.id + local rec + if id ~= nil then + rec = self._records[id] + if rec ~= nil then + if rec.kind ~= kind then return nil, 'handle_kind_mismatch' end + if rec.state ~= 'reserved' then return nil, 'handle_not_reserved' end + else + rec = { + kind = kind, + handle_id = id, + state = 'reserved', + } + self._records[id] = rec + end + else + id = self:_new_id() + rec = { + kind = kind, + handle_id = id, + state = 'reserved', + } + self._records[id] = rec + end + + rec.handle = handle + apply_opts(rec, opts) + rec.state = 'registered' + self:_emit(event_for(rec, 'registered')) + return id, copy_record(rec) +end + +function Registry:get(id) + return self._records[id] +end + +function Registry:mark_transferred(id, generation) + local rec = self._records[id] + if not rec then return nil, 'stale_handle' end + if generation ~= nil and rec.generation ~= generation then return nil, 'stale_handle' end + if rec.state == 'terminated' then return nil, 'stale_handle' end + if rec.owner == 'caller_after_handoff' and rec.state == 'transferred' then return true end + rec.owner = 'caller_after_handoff' + rec.state = 'transferred' + self:_emit(event_for(rec, 'transferred')) + return true +end + +function Registry:remove(id, reason, generation) + local rec = self._records[id] + if not rec then return false, 'stale_handle' end + if generation ~= nil and rec.generation ~= generation then return false, 'stale_handle' end + self._records[id] = nil + local prev_state = rec.state + rec.state = 'terminated' + rec.reason = reason or 'closed' + -- Reserved-only records were never externally live. Removing them should be + -- idempotent cleanup, not an active-handle termination event. + if prev_state ~= 'reserved' then + self:_emit(event_for(rec, 'terminated', { reason = rec.reason })) + end + return true +end + +function Registry:terminate(id, reason, generation) + local rec = self._records[id] + if not rec then return false, 'stale_handle' end + if generation ~= nil and rec.generation ~= generation then return false, 'stale_handle' end + local h = rec.handle + if h and type(h.terminate) == 'function' then h:terminate(reason or 'http_service_shutdown') end + return self:remove(id, reason, generation) +end + +function Registry:terminate_all(reason) + local ids = {} + for id in pairs(self._records) do ids[#ids + 1] = id end + for _, id in ipairs(ids) do self:terminate(id, reason or 'http_service_shutdown') end + return true +end + +function Registry:count(kind) + local n = 0 + for _, rec in pairs(self._records) do + if rec.state ~= 'reserved' and rec.state ~= 'terminated' and (kind == nil or rec.kind == kind) then + n = n + 1 + end + end + return n +end + +function Registry:snapshot() + local out = {} + for id, rec in pairs(self._records) do + out[id] = { + handle_id = rec.handle_id, + kind = rec.kind, + generation = rec.generation, + owner = rec.owner, + state = rec.state, + listener_id = rec.listener_id, + context_id = rec.context_id, + } + end + return out +end + +M.Registry = Registry +return M diff --git a/src/services/http/sdk.lua b/src/services/http/sdk.lua new file mode 100644 index 00000000..b3cd260b --- /dev/null +++ b/src/services/http/sdk.lua @@ -0,0 +1,34 @@ +-- services/http/sdk.lua +-- Op-native bus client for cap/http/. + +local topics = require 'services.http.topics' + +local M = {} +local Ref = {} +Ref.__index = Ref + +function M.new_ref(conn, id) + return setmetatable({ conn = conn, id = id or 'main' }, Ref) +end + +local function call_opts(opts) + local out = {} + for k, v in pairs(opts or {}) do out[k] = v end + if out.timeout == nil and out.deadline == nil then + out.timeout = false + end + return out +end + +function Ref:call_op(verb, args, opts) + return self.conn:call_op(topics.rpc(self.id, verb), args or {}, call_opts(opts)) +end + +function Ref:status_op(opts) return self:call_op('status', {}, opts) end +function Ref:listen_op(args, opts) return self:call_op('listen', args or {}, opts) end +function Ref:open_exchange_op(args, opts) return self:call_op('open-exchange', args or {}, opts) end +function Ref:exchange_op(args, opts) return self:call_op('exchange', args or {}, opts) end +function Ref:connect_ws_op(args, opts) return self:call_op('connect-ws', args or {}, opts) end + +M.Ref = Ref +return M diff --git a/src/services/http/service.lua b/src/services/http/service.lua new file mode 100644 index 00000000..c455d9f7 --- /dev/null +++ b/src/services/http/service.lua @@ -0,0 +1,1110 @@ +-- services/http/service.lua +-- System HTTP capability service. It owns backend lifetime, handle registry, +-- retained status and bus endpoints. Protocol I/O lives in handles/workers. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' + +local driver_mod = require 'services.http.transport.cqueues_driver' +local backend_mod = require 'services.http.backend' +local model_mod = require 'services.http.model' +local topics = require 'services.http.topics' +local cap_surface = require 'services.http.cap_surface' +local registry_mod = require 'services.http.registry' +local operations = require 'services.http.operations' +local operation_owner = require 'services.http.operation_owner' +local queue = require 'devicecode.support.queue' +local config_watch = require 'devicecode.support.config_watch' +local config_mod = require 'services.http.config' +local service_events = require 'devicecode.support.service_events' +local service_base = require 'devicecode.service_base' +local retained_publish = require 'devicecode.support.retained_publish' + +local M = {} +local perform = fibers.perform + +local DEFAULT_MAX_EVENT_HISTORY = 0 + +local function normalise_initial_config(opts) + local raw = opts.config + if raw == nil then raw = { id = opts.id, policy = opts.policy } end + local cfg, err = config_mod.normalise(raw) + if not cfg then return nil, err end + return cfg, nil +end + +local HttpService = {} +HttpService.__index = HttpService + +local function copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +local function copy_event(ev) + local out = {} + for k, v in pairs(ev or {}) do + if k ~= 'req' + and k ~= 'owner' + and k ~= 'ctx' + and k ~= 'websocket' + and k ~= 'result' + and k ~= 'report' + and k ~= 'raw' then + out[k] = v + end + end + return out +end + + +-- Event ingress is a documented immediate Fibers-side attempt. All code in +-- this system runs inside Fibers, so use the public Op interface rather than +-- probing primitive internals. +local function try_admit_event_now(tx, ev, label) + local prefix = label or 'http_event_report_failed' + if not tx or type(tx.send_op) ~= 'function' then + return nil, prefix .. ': closed' + end + return queue.try_admit_required(tx, ev, prefix) +end + +local function fail_req(req, reason) + if req and type(req.fail) == 'function' then return req:fail(reason) end + return false, 'request has no fail' +end + +local function terminate_handle(h, reason) + if h and type(h.terminate) == 'function' then return h:terminate(reason) end + return true +end + +local function is_live_handle(rec) + return rec and rec.state ~= 'reserved' and rec.state ~= 'terminated' +end + +local function count_where(t, pred) + local n = 0 + for _, rec in pairs(t or {}) do if pred(rec) then n = n + 1 end end + return n +end + +local function now() + return fibers.now and fibers.now() or os.clock() +end + +local function obs_config(self) + local cfg = self._state and self._state.config and self._state.config.observability + return cfg or config_mod.DEFAULTS.observability +end + +local function obs_status_key(snap) + return table.concat({ + tostring(snap.state), + tostring(snap.backend), + tostring(snap.ready), + tostring(snap.active_listeners), + tostring(snap.active_websockets), + tostring(snap.last_error), + tostring(snap.policy_generation), + }, '|') +end + +local function retained_stats_semantic_key(snap) + -- Retained stats need to remain responsive for public handle/dependency + -- state, but completed/failed/rejected counters are diagnostic and can + -- be emitted on the lower-frequency stats interval. + return table.concat({ + tostring(snap.state), + tostring(snap.backend), + tostring(snap.ready), + tostring(snap.active_listeners), + tostring(snap.active_contexts), + tostring(snap.active_exchanges), + tostring(snap.active_websockets), + tostring(snap.last_error), + tostring(snap.policy_generation), + }, '|') +end + +local function cap_status_payload(snap) + return { + state = snap.state, + available = snap.ready, + backend = snap.backend, + reason = snap.ready and nil or (snap.last_error or snap.backend), + last_error = snap.last_error, + } +end + +local function stats_publish_interval(self) + local cfg = obs_config(self) + return tonumber(cfg.stats_interval_s or cfg.status_interval_s) or 30 +end + +local function uri_summary(uri) + if type(uri) ~= 'string' then return {} end + local scheme, rest = uri:match('^([%w%+%-%.]+)://(.+)$') + if not scheme then return { uri = uri } end + local authority, path_query = rest:match('^([^/%?#]+)(.*)$') + path_query = path_query or '' + local host = authority or '' + host = host:gsub('^.-@', '') + local path, query = path_query:match('^([^%?]*)(%?.*)$') + if path == nil then path = path_query end + if path == '' then path = '/' end + local cmd = query and query:match('[%?&]cmd=([^&]+)') or nil + return { scheme = scheme, host = host, path = path, query = query, cmd = cmd } +end + +local function request_summary(req) + local p = req and req.payload or {} + local u = uri_summary(p.uri) + u.method = p.method or 'GET' + u.response_parser = p.response_parser + return u +end + +function HttpService:_derive_snapshot() + local st = self._state + return { + state = st.service_state, + backend = st.backend, + ready = st.ready, + active_listeners = count_where(st.listeners, is_live_handle), + active_contexts = count_where(st.contexts, function (rec) return rec.state ~= 'terminated' end), + active_exchanges = count_where(st.exchanges, is_live_handle), + active_websockets = count_where(st.websockets, is_live_handle), + completed_exchanges = st.completed_exchanges or 0, + failed_exchanges = st.failed_exchanges or 0, + rejected_requests = st.rejected_requests or 0, + last_error = st.last_error, + policy_generation = st.policy_generation, + } +end + + + +function HttpService:_log_http_summary(reason) + local snap = self:_derive_snapshot() + local completed = tonumber(snap.completed_exchanges) or 0 + local failed = tonumber(snap.failed_exchanges) or 0 + local rejected = tonumber(snap.rejected_requests) or 0 + local active = tonumber(snap.active_exchanges) or 0 + local summary = string.format('http summary backend=%s active=%d completed=%d failed=%d rejected=%d', tostring(snap.backend), active, completed, failed, rejected) + local key = summary + local tnow = now() + if self._last_operator_summary_key == key and (tnow - (self._last_operator_summary_at or 0)) < 600 then return end + self._last_operator_summary_key = key + self._last_operator_summary_at = tnow + self:_publish_obs_log('info', { what = 'http_summary', summary = summary, reason = reason }) +end + +function HttpService:_publish_obs_log(level, payload) + payload = payload or {} + payload.service = 'http' + payload.http_id = self._id + payload.ts = now() + self._conn:publish(topics.obs_log(self._id, level), payload) + return true +end + +function HttpService:_should_publish_obs_status(snap, ev) + local cfg = obs_config(self) + local key = obs_status_key(snap) + local obs = self._obs or {} + if key ~= obs.last_status_key then return true, key end + local interval = cfg.status_interval_s or 30 + if interval > 0 and ((now() - (obs.last_status_emit_at or 0)) >= interval) then return true, key end + if ev and ev.kind == 'http_operation_done' and ev.status == 'ok' and cfg.success_events == true then return true, key end + return false, key +end + +function HttpService:_publish_obs_status(snap, ev) + local publish, key = self:_should_publish_obs_status(snap, ev) + if not publish then return true end + self._conn:retain(topics.obs_metric(self._id, 'status'), snap) + self._obs.last_status_key = key + self._obs.last_status_emit_at = now() + return true +end + +function HttpService:_record_http_failure(rec, ev) + local cfg = obs_config(self) + local limit_s = cfg.failure_rate_limit_s or 60 + local req = self._state.requests[rec.request_id] or {} + local target = req.target or {} + local reason = ev.primary or ev.status or 'operation_failed' + self._obs.had_failure = true + local key = table.concat({ + tostring(rec.operation), + tostring(reason), + tostring(target.method), + tostring(target.host), + tostring(target.path), + tostring(target.cmd or target.query), + }, '|') + local windows = self._obs.failure_windows + local win = windows[key] + local t = now() + if not win then + windows[key] = { first_at = t, last_at = t, suppressed = 0 } + return self:_publish_obs_log('warn', { + what = 'request_failed', + reason = reason, + operation = rec.operation, + method = target.method, + host = target.host, + path = target.path, + cmd = target.cmd, + response_parser = target.response_parser, + }) + end + win.last_at = t + win.suppressed = (win.suppressed or 0) + 1 + if limit_s <= 0 or (t - (win.first_at or t)) >= limit_s then + local suppressed = win.suppressed or 0 + win.first_at = t + win.suppressed = 0 + return self:_publish_obs_log('warn', { + what = 'request_failed_suppressed', + reason = reason, + operation = rec.operation, + method = target.method, + host = target.host, + path = target.path, + cmd = target.cmd, + response_parser = target.response_parser, + suppressed = suppressed, + window_s = limit_s, + }) + end + return true +end + +function HttpService:_record_http_recovery(rec) + if not self._obs.had_failure then return true end + self._obs.had_failure = false + self._obs.failure_windows = {} + return self:_publish_obs_log('info', { + what = 'request_recovered', + operation = rec and rec.operation, + }) +end + +function HttpService:_should_publish_retained_stats(snap, ev, status_changed) + local key = retained_stats_semantic_key(snap) + if self._retained_stats_cache == nil or self._retained_stats_cache.stats == nil then return true, key end + if status_changed == true then return true, key end + if key ~= self._retained_stats_semantic_key then return true, key end + local interval = stats_publish_interval(self) + if interval <= 0 then return false, key end + return (now() - (self._retained_stats_emit_at or 0)) >= interval, key +end + +function HttpService:_publish_model(ev) + if self._closed and not self._publishing_after_close then return end + local snap = self._model:snapshot() + self._retained_status_cache = self._retained_status_cache or {} + self._retained_stats_cache = self._retained_stats_cache or {} + local ok, err, status_changed = retained_publish.retain_if_changed( + self._conn, + self._retained_status_cache, + 'status', + topics.status(self._id), + cap_status_payload(snap) + ) + if ok ~= true then return nil, err end + local publish_stats, stats_key = self:_should_publish_retained_stats(snap, ev, status_changed) + if publish_stats then + ok, err = retained_publish.retain_if_changed( + self._conn, + self._retained_stats_cache, + 'stats', + topics.state(self._id, 'stats'), + snap + ) + if ok ~= true then return nil, err end + self._retained_stats_emit_at = now() + self._retained_stats_semantic_key = stats_key + end + self:_publish_obs_status(snap, ev) + return true +end + +function HttpService:_refresh_model(ev) + local changed = self._model:set_snapshot(self:_derive_snapshot()) + if changed == true then return self:_publish_model(ev) end + return true +end + +function HttpService:_publish_lifecycle() + local lifecycle = self._opts and self._opts.lifecycle or nil + if lifecycle and type(lifecycle.running) == 'function' then + lifecycle:running({ + ready = self._state.ready == true, + http_id = self._id, + backend = self._state.backend, + state = self._state.service_state, + }) + end + return true +end + +function HttpService:_log_event(ev) + if not ev or not ev.kind then return true end + local max_events = tonumber(self._opts and self._opts.max_event_history) or DEFAULT_MAX_EVENT_HISTORY + if max_events <= 0 then return true end + self._event_seq = self._event_seq + 1 + local saved = copy_event(ev) + saved.seq = self._event_seq + self._events[#self._events + 1] = saved + while #self._events > max_events do table.remove(self._events, 1) end + return true +end + +function HttpService:_submit_event(ev, label, opts) + opts = opts or {} + if not ev or not ev.kind then return nil, 'invalid_event' end + local ok, err = try_admit_event_now(self._event_tx, ev, label or 'http_event_report_failed') + if ok == true then return true end + + self._event_admission_failure = err or 'http_event_report_failed' + -- Once service shutdown has begun, the observing scope is no longer healthy. + -- Late stored completions may try to report after the event queue has been + -- closed; that must not turn orderly shutdown into reporter failure. + if self._closed and tostring(self._event_admission_failure):match('closed') then + return true, nil + end + if opts.fail_request and ev.owner then ev.owner:fail_once(self._event_admission_failure) end + if opts.fatal and self._scope then + local reason = self._event_admission_failure + if type(self._scope.spawn) == 'function' then + local spawned = self._scope:spawn(function () error(reason, 0) end) + if spawned ~= true and type(self._scope.cancel) == 'function' then self._scope:cancel(reason) end + elseif type(self._scope.cancel) == 'function' then + self._scope:cancel(reason) + end + end + return nil, self._event_admission_failure +end + + +-- Registry callbacks may run from the caller's scope, including while that +-- scope is being actively cancelled because a public handle Op lost interest. +-- A normal scope-aware immediate send is then rejected before it can even probe +-- the service event queue. Preserve the event-driven model by falling back to +-- a tiny reporter fibre in the HTTP service scope; the coordinator still sees a +-- semantic event and remains the only reducer. + +function HttpService:_event_port(identity, opts) + opts = opts or {} + local attrs = { + service_id = self._id, + source = identity and identity.source or 'http_service', + source_id = identity and identity.source_id or self._id, + generation = identity and identity.generation or self._generation, + } + for k, v in pairs(identity or {}) do attrs[k] = v end + + -- Build event envelopes with the shared helper, but route admission through + -- _submit_event. That preserves the HTTP service's shutdown policy: late + -- component completions after service termination are benign, while admission + -- failure during a healthy service remains explicit/fatal where requested. + local base = service_events.port(self._event_tx, attrs, opts) + local svc = self + local default_label = opts.label or 'http_service_event_report_failed' + return { + emit_required = function (_, ev, label) + return svc:_submit_event(base:event(ev), label or default_label, { fatal = opts.fatal ~= false }) + end, + event = function (_, ev, extra) + return base:event(ev, extra) + end, + identity = function () return base:identity() end, + } +end + +function HttpService:_submit_registry_event(ev, label) + local ok, r1, r2 = pcall(function () + return self:_submit_event(ev, label or 'http_registry_event_report_failed', { fatal = true }) + end) + if ok then return r1, r2 end + + if self._closed or not self._scope or type(self._scope.spawn) ~= 'function' then + return nil, r1 + end + + local spawned, serr = self._scope:spawn(function () + local admitted, aerr = self:_submit_event(ev, label or 'http_registry_event_report_failed', { fatal = true }) + if admitted ~= true then error(aerr or 'http_registry_event_report_failed', 0) end + end) + if spawned == true then return true, nil end + return nil, serr or r1 +end + +function HttpService:_next_request_identity(verb, req) + return operation_owner.next_request(self, verb, req) +end + +function HttpService:_account_completed_operation(rec, status) + if not rec or rec.operation ~= 'exchange' then return true end + if status == 'ok' then + self._state.completed_exchanges = (self._state.completed_exchanges or 0) + 1 + else + self._state.failed_exchanges = (self._state.failed_exchanges or 0) + 1 + end + return true +end + +function HttpService:_prune_request(request_id) + local rec = self._state.requests[request_id] + if rec then rec.owner = nil end + self._state.requests[request_id] = nil + self._owned_requests[request_id] = nil + return true +end + +function HttpService:_finish_request(request_id, _state, _reason) + return self:_prune_request(request_id) +end + +function HttpService:_reject_request(request_id, owner, reason) + if owner then owner:fail_once(reason or 'invalid_args') end + self._state.rejected_requests = (self._state.rejected_requests or 0) + 1 + self._state.last_error = reason or 'invalid_args' + self:_log_event { kind = 'cap_request_rejected', request_id = request_id, reason = reason or 'invalid_args' } + self:_prune_request(request_id) + return true +end + +function HttpService:_reserve_handle(kind, opts) + opts = opts or {} + return self._registry:reserve(kind, { + generation = opts.generation or self._generation, + owner = opts.owner or 'http_service', + }) +end + +function HttpService:_register_handle(kind, handle, opts) + opts = opts or {} + return self._registry:register(kind, handle, { + id = opts.id, + generation = opts.generation or self._generation, + owner = opts.owner or 'http_service', + }) +end + +function HttpService:_remove_handle(id, reason) + return self._registry:remove(id, reason or 'closed') +end + +function HttpService:_terminate_handle(id, reason) + return self._registry:terminate(id, reason or 'http_service_shutdown') +end + +function HttpService:_context_registry_id(ctx) + if ctx and type(ctx.registry_id) == 'function' then return ctx:registry_id() end + if ctx and type(ctx.id) == 'function' then return 'ctx' .. tostring(ctx:id()) end + return nil +end + +function HttpService:_register_context(listener_id, ctx) + local id = self:_context_registry_id(ctx) + if not id then return nil, 'context_missing_identity' end + return self._registry:register('context', ctx, { + id = id, + generation = self._generation, + owner = 'listener', + listener_id = listener_id, + context_id = (ctx and type(ctx.id) == 'function') and ctx:id() or id, + }) +end + +function HttpService:_mark_context_transferred(ctx) + local id = self:_context_registry_id(ctx) + if not id then return nil, 'context_missing_identity' end + return self._registry:mark_transferred(id, self._generation) +end + +function HttpService:_remove_context(ctx, reason) + local id = self:_context_registry_id(ctx) + if not id then return nil, 'context_missing_identity' end + return self._registry:remove(id, reason or 'context_terminated', self._generation) +end + +function HttpService:_register_server_websocket(ctx, ws) + if self._closed then + terminate_handle(ws, 'service_closed') + return nil, 'service_closed' + end + + local handle_id, err = self:_register_handle('websocket', ws, { + generation = self._generation, + owner = 'caller_after_handoff', + context_id = ctx and type(ctx.id) == 'function' and ctx:id() or nil, + }) + if not handle_id then return nil, err or 'websocket_register_failed' end + + if ws and type(ws.add_terminate_hook) == 'function' then + ws:add_terminate_hook(function (_, reason) + self:_remove_handle(handle_id, reason or 'websocket_terminated') + end) + end + + self._registry:mark_transferred(handle_id, self._generation) + return true, handle_id +end + +function HttpService:_record_handle_event(ev) + local kind, id = ev.kind, ev.handle_id + local generation = ev.generation or self._generation + if not id then return true end + + local function table_for(k) + if k:match('^listener_') then return self._state.listeners, 'listener' end + if k:match('^exchange_') then return self._state.exchanges, 'exchange' end + if k:match('^websocket_') then return self._state.websockets, 'websocket' end + return nil, nil + end + + local tbl, hkind = table_for(kind) + if not tbl then return true end + local rec = tbl[id] + if rec and rec.generation ~= generation then return true end + + if kind == hkind .. '_registered' then + tbl[id] = rec or { handle_id = id, generation = generation, kind = hkind } + tbl[id].state = 'registered' + tbl[id].owner = 'http_service' + elseif kind == hkind .. '_transferred' then + if not rec then + tbl[id] = { handle_id = id, generation = generation, kind = hkind, state = 'transferred', owner = 'caller_after_handoff' } + else + rec.state = 'transferred' + rec.owner = 'caller_after_handoff' + end + elseif kind == hkind .. '_terminated' then + if not rec then + tbl[id] = { handle_id = id, generation = generation, kind = hkind, state = 'terminated', reason = ev.reason } + elseif rec.state ~= 'terminated' then + rec.state = 'terminated' + rec.reason = ev.reason + end + end + return true +end + +function HttpService:_record_context_event(ev) + local context_id = ev.context_id + if context_id == nil then + self._state.last_error = 'context_event_missing_identity' + return true + end + local id = tostring(context_id) + local rec = self._state.contexts[id] + local generation = ev.generation or self._generation + if rec and rec.generation ~= generation then return true end + + -- Context handles may report from caller/request scopes through an event port. + -- The coordinator remains the owner of the public model; registry operations + -- here are immediate accounting/termination backstop updates. + if ev.ctx ~= nil then + if ev.kind == 'context_admitted' or ev.kind == 'context_registered' then + self:_register_context(ev.listener_id, ev.ctx) + elseif ev.kind == 'context_transferred' then + self:_mark_context_transferred(ev.ctx) + elseif ev.kind == 'context_terminated' then + self:_remove_context(ev.ctx, ev.reason or 'context_terminated') + end + end + if ev.kind == 'context_admitted' or ev.kind == 'context_registered' then + if not rec then + self._state.contexts[id] = { + context_id = context_id, + listener_id = ev.listener_id, + generation = generation, + state = 'admitted', + owner = 'listener', + } + end + elseif ev.kind == 'context_transferred' then + if not rec then + self._state.contexts[id] = { + context_id = context_id, + listener_id = ev.listener_id, + generation = generation, + state = 'transferred', + owner = 'caller_after_handoff', + } + else + rec.state = 'transferred' + rec.owner = 'caller_after_handoff' + end + elseif ev.kind == 'context_terminated' then + if not rec then + self._state.contexts[id] = { + context_id = context_id, + listener_id = ev.listener_id, + generation = generation, + state = 'terminated', + reason = ev.reason, + } + elseif rec.state ~= 'terminated' then + rec.state = 'terminated' + rec.reason = ev.reason + end + end + return true +end + +function HttpService:_handle_cap_request(ev) + local verb, req, request_id, owner = ev.verb, ev.req, ev.request_id, ev.owner + if not request_id or not owner then + return nil, 'cap_request_missing_owner' + end + + if not self._state.requests[request_id] then + self._state.requests[request_id] = { + request_id = request_id, + generation = ev.generation or self._generation, + verb = verb, + owner = owner, + target = request_summary(req), + state = 'received', + } + end + + if type(owner.done) == 'function' and owner:done() then + self:_finish_request(request_id, 'cancelled', 'request_already_resolved') + return true + end + + if verb ~= 'status' and self._state.config and self._state.config.enabled == false then + return self:_reject_request(request_id, owner, 'disabled') + end + + if verb == 'status' then + local ok, err = owner:reply_once({ status = self._model:snapshot() }) + if ok ~= true then return nil, err or 'request_reply_failed' end + self:_finish_request(request_id, 'resolved') + return true + end + + if self._state.ready ~= true then + return self:_reject_request(request_id, owner, self._state.last_error or 'http_backend_not_ready') + end + + local ok, err = operations.validate_cap_request(self, verb, req) + if not ok then return self:_reject_request(request_id, owner, err or 'invalid_args') end + return operations.start_operation(self, verb, req, request_id, owner) +end + +function HttpService:_handle_operation_done(ev) + local rec = self._state.operations[ev.operation_id] + if not rec or rec.generation ~= ev.generation then + return true + end + if rec.state == 'completed' then return true end + + rec.state = 'completed' + rec.status = ev.status + rec.report = ev.report + rec.result = ev.result + rec.primary = ev.primary + if ev.status ~= 'ok' then self._state.last_error = ev.primary or ev.status; self._obs.had_failure = true end + + local owner = self._owned_requests[rec.request_id] + local reqrec = self._state.requests[rec.request_id] + local operation_success = false + if ev.status == 'ok' then + local reply = ev.result or {} + local ok, rerr = true, nil + if owner and not owner:done() then + ok, rerr = owner:reply_once(reply) + end + if ok == true then + operation_success = true + if reply.handle_id then self._registry:mark_transferred(reply.handle_id) end + if reqrec then reqrec.state = 'resolved' end + else + if reply.handle_id then self:_terminate_handle(reply.handle_id, rerr or 'reply_failed') end + if reqrec then reqrec.state = 'failed'; reqrec.reason = rerr or 'reply_failed' end + self._state.last_error = rerr or 'reply_failed' + self._obs.had_failure = true + self:_record_http_failure(rec, { status = 'failed', primary = rerr or 'reply_failed' }) + end + else + if owner and not owner:done() then owner:fail_once(ev.primary or ev.status or 'operation_failed') end + if reqrec then reqrec.state = ev.status or 'failed'; reqrec.reason = ev.primary end + self:_record_http_failure(rec, ev) + end + if obs_config(self).request_trace == true then + local target = reqrec and reqrec.target or {} + self:_publish_obs_log('debug', { + what = 'request_completed', + operation = rec.operation, + status = ev.status, + reason = ev.primary, + method = target.method, + host = target.host, + path = target.path, + cmd = target.cmd, + response_parser = target.response_parser, + }) + end + if operation_success == true then self:_record_http_recovery(rec) end + self:_account_completed_operation(rec, ev.status) + self:_prune_request(rec.request_id) + self._state.operations[ev.operation_id] = nil + self:_log_event(ev) + return true +end + +function HttpService:_handle_config_changed(ev) + local cfg, err = config_mod.normalise(ev.raw or {}) + if not cfg then + self._state.last_error = tostring(err or 'invalid_config') + self._state.service_state = 'degraded' + return true + end + + if cfg.id ~= self._id then + self._state.last_error = 'config_id_mismatch' + self._state.service_state = 'degraded' + return true + end + + self._state.config = cfg + self._state.config_generation = ev.generation or (self._state.config_generation + 1) + self._state.policy_generation = self._state.config_generation + self._opts.policy = cfg.policy + if cfg.enabled == false then + self._state.service_state = 'disabled' + self._state.ready = false + else + self._state.ready = self._state.backend == 'ready' + if self._state.backend == 'ready' then self._state.service_state = 'ready' end + end + self._state.last_error = nil + self:_log_event({ kind = 'policy_changed', generation = self._state.policy_generation }) + return true +end + +function HttpService:_reduce_event(ev) + local k = ev.kind + if k == 'backend_ready' then + if self._state.backend ~= 'ready' then self:_publish_obs_log('info', { what = 'http_ready', summary = 'http backend ready' }) end + self._state.backend = 'ready' + if self._state.config and self._state.config.enabled == false then + self._state.service_state = 'disabled'; self._state.ready = false + else + self._state.service_state = 'ready'; self._state.ready = true + end + elseif k == 'backend_failed' then + self:_publish_obs_log('error', { what = 'http_failed', reason = ev.reason }) + self._state.backend = 'failed'; self._state.service_state = 'failed'; self._state.ready = false; self._state.last_error = ev.reason + if self._registry then self._registry:terminate_all(ev.reason or 'backend_failed') end + elseif k == 'backend_terminated' then + self._state.backend = 'terminated'; self._state.service_state = 'closed'; self._state.ready = false; self._state.last_error = ev.reason + if self._registry then self._registry:terminate_all(ev.reason or 'backend_terminated') end + elseif k == 'backend_done' then + if ev.generation and ev.generation ~= self._generation then return true end + if ev.status == 'ok' then + self._state.backend = 'terminated'; self._state.service_state = 'closed'; self._state.ready = false + self._state.last_error = ev.result and ev.result.reason or 'backend_done' + if self._registry then self._registry:terminate_all(self._state.last_error or 'backend_done') end + elseif ev.status == 'cancelled' then + self._state.backend = 'terminated'; self._state.service_state = 'closed'; self._state.ready = false + self._state.last_error = ev.primary or 'backend_cancelled' + if self._registry then self._registry:terminate_all(ev.primary or 'backend_cancelled') end + else + self._state.backend = 'failed'; self._state.service_state = 'failed'; self._state.ready = false + self._state.last_error = ev.primary or ev.status or 'backend_failed' + if self._registry then self._registry:terminate_all(self._state.last_error or 'backend_failed') end + end + return true + elseif k == 'config_changed' then + return self:_handle_config_changed(ev) + elseif k == 'cap_request_received' then + return self:_handle_cap_request(ev) + elseif k == 'http_operation_done' then + return self:_handle_operation_done(ev) + elseif k == 'listener_registered' or k == 'listener_transferred' or k == 'listener_terminated' + or k == 'exchange_registered' or k == 'exchange_transferred' or k == 'exchange_terminated' + or k == 'websocket_registered' or k == 'websocket_transferred' or k == 'websocket_terminated' then + return self:_record_handle_event(ev) + elseif k == 'context_admitted' or k == 'context_registered' or k == 'context_transferred' or k == 'context_terminated' then + return self:_record_context_event(ev) + elseif k == 'server_websocket_registered' then + if ev.ctx ~= nil and ev.websocket ~= nil then + local ok, err = self:_register_server_websocket(ev.ctx, ev.websocket) + if ok ~= true then self._state.last_error = err or 'server_websocket_register_failed' end + end + return true + elseif k == 'listener_done' then + if ev.status ~= 'ok' then self._state.last_error = ev.primary or ev.status end + if ev.handle_id then + local reason = ev.primary or (ev.result and ev.result.reason) or ev.status or 'listener_done' + self:_remove_handle(ev.handle_id, reason) + end + return true + elseif k == 'policy_changed' then + self._state.policy_generation = ev.generation or (self._state.policy_generation + 1) + end + return true +end + +function HttpService:_handle_event(ev) + if not ev or not ev.kind then return nil, 'invalid_event' end + local log_now = ev.kind ~= 'http_operation_done' + if log_now then self:_log_event(ev) end + local ok, err = self:_reduce_event(ev) + self:_refresh_model(ev) + return ok, err +end + +function HttpService:_coordinator_loop() + while true do + local ev, err = perform(self._event_rx:recv_op()) + if not ev then return nil, err end + local ok, herr = self:_handle_event(ev) + if ok == nil or ok == false then error(herr or 'http_coordinator_event_failed', 0) end + end +end + +function HttpService:_config_loop() + while not self._closed do + local ev = perform(self._cfg_watch:recv_op()) + if ev == nil or ev.kind == 'config_closed' then + return nil, ev and ev.err or 'config_closed' + end + local ok, err = self:_submit_event({ + kind = 'config_changed', + generation = ev.generation, + rev = ev.rev, + raw = ev.raw, + }, 'http_config_event_report_failed', { fatal = true }) + if ok ~= true then error(err or 'http_config_event_report_failed', 0) end + end + return true +end + +function HttpService:_endpoint_loop(verb, ep) + while not self._closed do + local req, err = perform(ep:recv_op()) + if not req then return nil, err end + local request_id, owner = self:_next_request_identity(verb, req) + local ok, qerr = self:_submit_event({ + kind = 'cap_request_received', verb = verb, req = req, request_id = request_id, owner = owner, generation = self._generation, + }, 'http_cap_request_admission_failed') + if ok ~= true then + self._owned_requests[request_id] = nil + owner:fail_once(qerr or 'service_busy') + end + end + return true +end + +function HttpService:terminate(reason) + if self._closed then return true end + local why = reason or 'closed' + self._closed = true + self._publishing_after_close = true + for _, rec in pairs(self._state.operations or {}) do + if rec.handle and type(rec.handle.cancel) == 'function' and rec.state ~= 'completed' then rec.handle:cancel(why) end + end + for request_id, owner in pairs(self._owned_requests or {}) do + owner:finalise_unresolved(why) + local rec = self._state.requests[request_id] + if rec and rec.state ~= 'resolved' then rec.state = 'cancelled'; rec.reason = why end + self._owned_requests[request_id] = nil + end + if self._registry then self._registry:terminate_all(why) end + if self._cfg_watch then self._cfg_watch:close(); self._cfg_watch = nil end + if self._backend then self._backend:terminate(why) + elseif self._driver then self._driver:terminate(why) end + if self._event_tx then + self:_submit_event({ kind = 'backend_terminated', reason = why }, 'http_backend_terminated_report_failed') + if type(self._event_tx.close) == 'function' then self._event_tx:close(why) end + end + if self._endpoints then cap_surface.unbind(self._conn, self._endpoints) end + cap_surface.unretain_static(self._conn, self._id) + self._model:terminate(why) + return true +end + +function HttpService:stats() + local snap = self._model:snapshot() + snap.handles = self._registry:snapshot() + return snap +end + +function HttpService:events() + local out = {} + for i, ev in ipairs(self._events) do out[i] = copy(ev) end + return out +end + +function M.open_handle(conn, opts) + opts = opts or {} + local scope = fibers.current_scope() + if not scope then return nil, 'http.open_handle must be called inside a scope' end + + local initial_config, config_err = normalise_initial_config(opts) + if not initial_config then return nil, config_err or 'invalid_http_config' end + local driver = opts.driver or assert(driver_mod.new(opts.driver_options or { label = 'http-service' })) + opts.policy = initial_config.policy + local model = model_mod.new() + local event_tx, event_rx = mailbox.new(opts.event_queue_len or 64, { full = 'reject_newest' }) + local self = setmetatable({ + _conn = conn, + _scope = scope, + _id = opts.id or 'main', + _opts = opts, + _driver = driver, + _backend = nil, + _model = model, + _event_tx = event_tx, + _event_rx = event_rx, + _next_operation_id = 0, + _next_request_id = 0, + _generation = 1, + _config = initial_config, + _closed = false, + _events = {}, + _event_seq = 0, + _owned_requests = {}, + _obs = { + last_status_key = nil, + last_status_emit_at = nil, + failure_windows = {}, + had_failure = false, + }, + _state = { + service_state = 'starting', + backend = 'starting', + ready = false, + last_error = nil, + policy_generation = 1, + config_generation = 1, + config = initial_config, + completed_exchanges = 0, + failed_exchanges = 0, + rejected_requests = 0, + requests = {}, + operations = {}, + listeners = {}, + contexts = {}, + exchanges = {}, + websockets = {}, + }, + }, HttpService) + self._registry = registry_mod.new({ + events_port = { + emit_required = function (_, ev, label) + return self:_submit_registry_event(ev, label or 'http_registry_event_report_failed') + end, + }, + }) + + local cfg_watch, cfg_err = config_watch.open(conn, 'http', { + topic = opts.config_topic or { 'cfg', 'http' }, + queue_len = opts.config_queue_len or 4, + full = 'reject_newest', + }) + if not cfg_watch then self:terminate(cfg_err); return nil, cfg_err or 'http config subscribe failed' end + self._cfg_watch = cfg_watch + + scope:finally(function (_, status, primary) + self:terminate(primary or status or 'scope_finalised') + end) + + cap_surface.retain_static(conn, self._id, { state = 'starting', available = false, backend = 'starting', reason = 'backend_starting' }, model:snapshot()) + self._endpoints = assert(cap_surface.bind(conn, self._id, {}, { endpoint_opts = { queue_len = opts.endpoint_queue_len or 10 } })) + + local ok_coord, cerr = scope:spawn(function () return self:_coordinator_loop() end) + if not ok_coord then self:terminate(cerr); return nil, cerr end + + local ok_cfg, cfg_loop_err = scope:spawn(function () return self:_config_loop() end) + if not ok_cfg then self:terminate(cfg_loop_err); return nil, cfg_loop_err end + + local backend, derr = backend_mod.start({ + lifetime_scope = scope, + driver = driver, + generation = self._generation, + identity = { + kind = 'backend_done', + component = 'backend', + component_id = 'http_backend', + generation = self._generation, + }, + events_port = self:_event_port({ + source = 'http_backend', + source_id = 'http_backend', + component = 'backend', + component_id = 'http_backend', + }, { label = 'http_backend_event_report_failed' }), + }) + if not backend then + self:_submit_event({ kind = 'backend_failed', reason = derr }, 'http_backend_failed_report_failed') + self:terminate(derr or 'backend_failed') + return nil, derr + end + self._backend = backend + + for verb, ep in pairs(self._endpoints) do + local spawned, serr = scope:spawn(function () return self:_endpoint_loop(verb, ep) end) + if not spawned then self:terminate(serr); return nil, serr end + end + + return self +end + +function M.run(scope, params) + params = params or {} + local conn = params.conn + if conn == nil then error('http.run: conn required', 2) end + if scope == nil then error('http.run: scope required', 2) end + + local svc, err = M.open_handle(conn, params) + if not svc then error(err or 'http service start failed', 0) end + + local lifecycle = params.lifecycle + if lifecycle and type(lifecycle.running) == 'function' then + lifecycle:running({ ready = false, http_id = params.id or 'main' }) + end + + fibers.perform(fibers.never()) + svc:terminate('returned') + return nil, 'http service returned unexpectedly' +end + +function M.start(conn, opts) + opts = opts or {} + if conn == nil then error('http.start: conn required', 2) end + local scope = fibers.current_scope() + if not scope then error('http.start must be called inside a fiber', 2) end + + local svc = service_base.new(conn, { + name = opts.name or 'http', + env = opts.env, + meta = opts.meta, + announce = opts.announce, + }) + svc:starting({ ready = false }) + + local params = copy(opts) + params.conn = conn + params.name = opts.name or 'http' + params.lifecycle = svc + + M.run(scope, params) + + svc:stopped({ reason = 'returned' }) + error('http service returned unexpectedly', 0) +end + +M.HttpService = HttpService +return M diff --git a/src/services/http/tls.lua b/src/services/http/tls.lua new file mode 100644 index 00000000..b8975ab2 --- /dev/null +++ b/src/services/http/tls.lua @@ -0,0 +1,3 @@ +-- services/http/tls.lua +-- Public TLS policy boundary; lua-http details live in transport. +return require 'services.http.transport.tls' diff --git a/src/services/http/topics.lua b/src/services/http/topics.lua new file mode 100644 index 00000000..2b5d87b1 --- /dev/null +++ b/src/services/http/topics.lua @@ -0,0 +1,21 @@ +-- services/http/topics.lua +-- Pure topic helpers for cap/http/. + +local topic = require 'shared.topic' + +local M = {} + +local function cap(id) + return { 'cap', 'http', id or 'main' } +end + +function M.base(id) return cap(id) end +function M.meta(id) return topic.append(cap(id), 'meta') end +function M.status(id) return topic.append(cap(id), 'status') end +function M.state(id, key) return topic.append(cap(id), 'state', key) end +function M.event(id, name) return topic.append(cap(id), 'event', name) end +function M.obs_metric(id, name) return { 'obs', 'v1', 'http', 'metric', id or 'main', name } end +function M.obs_log(id, level) return { 'obs', 'v1', 'http', 'log', id or 'main', level or 'info' } end +function M.rpc(id, verb) return topic.append(cap(id), 'rpc', verb) end + +return M diff --git a/src/services/http/transport/client.lua b/src/services/http/transport/client.lua new file mode 100644 index 00000000..9b91ac61 --- /dev/null +++ b/src/services/http/transport/client.lua @@ -0,0 +1,135 @@ +-- services/http/transport/client.lua +-- lua-http request integration. This layer constructs requests and returns +-- raw lua-http response headers and streams; public handles are built above this layer. + +local op = require 'fibers.op' +local headers_mod = require 'services.http.headers' +local terminate = require 'services.http.transport.terminate' +local legacy_http1_close = require 'services.http.transport.legacy_http1_close' + +local M = {} + +local function require_request(opts) + if opts and opts.request_module then return opts.request_module end + local ok, mod = pcall(require, 'http.request') + if not ok then return nil, mod end + return mod +end + +local function apply_headers(req, fields) + if not fields then return true end + for _, pair in ipairs(headers_mod.to_pairs(fields)) do + req.headers:append(tostring(pair[1]), tostring(pair[2] or '')) + end + return true +end + +local function has_explicit_header(fields, name) + name = tostring(name or ''):lower() + for k in pairs(fields or {}) do + if tostring(k):lower() == name then return true end + end + return false +end + +local function request_expect_option(args, opts, key) + if args and args[key] ~= nil then return args[key] end + return opts and opts[key] +end + +local function delete_header(headers, name) + if not headers then return false end + if type(headers.delete) == 'function' then return headers:delete(name) end + if type(headers.remove) == 'function' then return headers:remove(name) end + return false +end + +local function configure_expect_100(req, args, opts) + local expect_100_continue = request_expect_option(args, opts, 'expect_100_continue') + local expect_100_timeout = request_expect_option(args, opts, 'expect_100_timeout') + + if expect_100_continue ~= nil and type(expect_100_continue) ~= 'boolean' then + return nil, 'invalid_args' + end + + if expect_100_timeout ~= nil then + if type(expect_100_timeout) ~= 'number' or expect_100_timeout < 0 then + return nil, 'invalid_args' + end + req.expect_100_timeout = expect_100_timeout + end + + -- lua-http adds Expect: 100-continue for iterator/function request bodies + -- because their length is not known to set_body(). That also introduces + -- lua-http's default one-second expect_100_timeout unless the caller/backend + -- has deliberately opted in. The HTTP service keeps timeout policy explicit, + -- so implicit Expect is suppressed by default. + if expect_100_continue == true then + if req.headers and type(req.headers.upsert) == 'function' then + req.headers:upsert('expect', '100-continue') + end + return true + end + + if has_explicit_header(args.headers, 'expect') then + return true + end + + delete_header(req.headers, 'expect') + return true +end + +local function build_request(request_module, args, opts) + opts = opts or {} + local req, err = request_module.new_from_uri(args.uri) + if not req then return nil, err end + if args.method then req.headers:upsert(':method', args.method) end + apply_headers(req, args.headers) + if args._request_body ~= nil then + if type(req.set_body) == 'function' then req:set_body(args._request_body) + else return nil, 'request_body_not_supported' end + local ok, cerr = configure_expect_100(req, args, opts) + if not ok then return nil, cerr end + end + return req, nil +end + +function M.open_exchange_op(driver, checked_args, opts) + opts = opts or {} + if checked_args and checked_args.response_parser == 'legacy-http1-close' then + return legacy_http1_close.open_exchange_op(driver, checked_args, opts) + end + return op.guard(function () + local request_module, rerr = require_request(opts) + if not request_module then return op.always(nil, rerr) end + local active = { request = nil, stream = nil } + return driver:run_op('http.client.open_exchange', function () + local req, berr = build_request(request_module, checked_args, opts) + if not req then return nil, berr end + active.request = req + local response_headers, stream = req:go(opts.backend_timeout) + if not response_headers then return nil, stream or 'connect_failed' end + active.stream = stream + return response_headers, stream + end, { + detach_on_abort = true, + on_detached_complete = function (reason) + local why = reason or 'open_exchange_aborted' + if active.stream then + terminate.terminate_stream(active.stream, why) + elseif active.request then + terminate.terminate_request(active.request, why) + end + end, + }) + end) +end + +-- One-shot exchange setup only. Body copy/drain/commit is owned by +-- services.http.client so leases and finalisers are installed in that operation +-- scope, not inside the transport job. +function M.exchange_open_op(driver, checked_args, opts) + return M.open_exchange_op(driver, checked_args, opts) +end + +return M diff --git a/src/services/http/transport/cqueues_driver.lua b/src/services/http/transport/cqueues_driver.lua new file mode 100644 index 00000000..f23fc9b5 --- /dev/null +++ b/src/services/http/transport/cqueues_driver.lua @@ -0,0 +1,560 @@ +-- services/http/transport/cqueues_driver.lua +-- +-- Fibers-facing bridge for cqueues-style pollable objects. +-- +-- This module is the only place that knows how to wait for a cqueues/lua-http +-- pollable from Fibers. The seam is deliberately small: +-- * wait on pollfd()/events()/timeout() with Fibers waitable2; +-- * treat readiness as a hint to call step(0); +-- * expose cqueues coroutine work as Fibers Ops; +-- * provide immediate termination for finalisers and abort paths. + +local op = require 'fibers.op' +local wait = require 'fibers.wait' +local poller = require 'fibers.io.poller' +local performer = require 'fibers.performer' +local runtime = require 'fibers.runtime' +local safe = require 'coxpcall' + +local perform = performer.perform +local unpack = rawget(table, 'unpack') or _G.unpack +local pack = rawget(table, 'pack') or function (...) + return { n = select('#', ...), ... } +end + +local M = {} + +local Driver = {} +Driver.__index = Driver + +local function identity(...) + return ... +end + +local function tostring_error(e) + if type(e) == 'string' or type(e) == 'number' then + return e + end + return tostring(e) +end + +local function is_timeout_error(err) + if err == nil then return false end + local s = tostring(err):lower() + return s:find('timeout', 1, true) ~= nil + or s:find('timed out', 1, true) ~= nil + or s:find('again', 1, true) ~= nil +end + +--- Map cqueues event strings to Fibers poll directions. +--- cqueues commonly uses r/w/p style event strings; p is treated as read-side +--- progress because it asks the outer loop to step the pollable. +local function event_wants(events) + events = events or 'r' + events = tostring(events) + + local rd = events:find('r', 1, true) ~= nil + or events:find('p', 1, true) ~= nil + local wr = events:find('w', 1, true) ~= nil + + return rd, wr +end + +M.event_wants = event_wants + +local function notify_waitset(waitset, key) + local sched = runtime.current_scheduler + if sched then waitset:notify_all(key, sched) end +end + +local function new_default_cqueue() + local ok, cqueues = pcall(require, 'cqueues') + if not ok or not cqueues then + return nil, 'cqueues module is not available' + end + return cqueues.new() +end + +--- Create a new driver. +--- +--- opts.controller may be supplied by tests or by lua-http server construction. +--- The controller needs the cqueues pollable protocol for run()/start() and +--- :wrap(fn) for run_op(). +function M.new(opts) + opts = opts or {} + + local controller = opts.controller + local err + if controller == nil and opts.create_controller ~= false then + controller, err = new_default_cqueue() + if not controller then return nil, err end + end + + local self = setmetatable({ + _controller = controller, + _label = opts.label or 'cqueues_driver', + _closed = false, + _close_reason = nil, + _state_waiters = wait.new_waitset(), + _poke_pending = false, + _job_waiters = wait.new_waitset(), + _jobs = {}, + _pump_started = false, + _step_fn = opts.step_fn, + _wrap_fn = opts.wrap_fn, + _terminate_fn = opts.terminate_fn, + }, Driver) + + return self +end + +function Driver:label() + return self._label +end + +function Driver:controller() + return self._controller +end + +function Driver:is_closed() + return self._closed +end + +function Driver:why() + return self._close_reason +end + +function Driver:_signal_changed() + self._poke_pending = true + notify_waitset(self._state_waiters, 'state') +end + +--- Wake the pump. Used when new work was queued into cqueues, and by tests. +function Driver:poke() + self:_signal_changed() +end + +function Driver:terminate(reason) + if self._closed then return true end + + self._closed = true + self._close_reason = reason or 'closed' + + local term = self._terminate_fn + if term then safe.pcall(term, self._controller, self._close_reason) end + + for job in pairs(self._jobs) do + job.abandoned = true + job.abort_reason = self._close_reason + job.phase = 'abandoned' + notify_waitset(self._job_waiters, job) + end + + self:_signal_changed() + return true +end + +local function step_return(ok, a, b, ...) + if not ok then + return nil, tostring_error(a) + end + + if a == nil or a == false then + if is_timeout_error(b) then + -- A spurious readiness hint should not kill the bridge. + return true, nil + end + return nil, b or 'cqueues step failed', ... + end + + return true, nil +end + +function Driver:_step_controller_now() + local controller = self._controller + if not controller then return nil, 'driver has no cqueues controller' end + + local step_fn = self._step_fn + return step_return(safe.pcall(function () + if step_fn then return step_fn(controller, 0) end + return controller:step(0) + end)) +end + +function Driver:step_controller_now() + return self:_step_controller_now() +end + +function Driver:_step_pollable_now(pollable, step_fn) + if self._closed then return nil, self._close_reason or 'closed' end + + return step_return(safe.pcall(function () + if step_fn then return step_fn(pollable, 0) end + return pollable:step(0) + end)) +end + +function Driver:step_pollable_now(pollable, step_fn) + return self:_step_pollable_now(pollable, step_fn) +end + +local function token2(tokens) + return { + unlink = function () + for i = 1, #tokens do + local t = tokens[i] + if t and t.unlink then t:unlink() end + tokens[i] = nil + end + return false + end, + } +end + +--- Wait until a cqueues-style pollable should be stepped. +--- +--- Implemented with waitable2 because readiness is only a hint: after any fd, +--- timer, or explicit state wake, the operation reports that the pollable should +--- be stepped by its owner. +--- +--- Returns: reason, err +--- reason: 'fd' | 'timeout' | 'poke' +--- err: non-nil only when the driver is closed before readiness +function Driver:pollable_ready_op(pollable, opts) + opts = opts or {} + + return op.guard(function () + assert(type(pollable) == 'table' or type(pollable) == 'userdata', + 'pollable_ready_op: pollable object required') + + local wake_reason + + local function compute_interest() + if self._closed then + return true, nil, self._close_reason or 'closed' + end + + if self._poke_pending then + self._poke_pending = false + return true, 'poke', nil + end + + local timeout + if type(pollable.timeout) == 'function' then + timeout = pollable:timeout() + end + if type(timeout) == 'number' and timeout <= 0 then + return true, 'timeout', nil + end + + local fd + if type(pollable.pollfd) == 'function' then + fd = pollable:pollfd() + end + + local rd, wr = false, false + if fd ~= nil then + local events = 'r' + if type(pollable.events) == 'function' then + events = pollable:events() + end + rd, wr = event_wants(events) + end + + return false, { + fd = fd, + rd = rd, + wr = wr, + timeout = timeout, + state = true, + } + end + + local function probe_step() + return compute_interest() + end + + local function run_step() + if wake_reason ~= nil then + local r = wake_reason + wake_reason = nil + if self._closed then return true, nil, self._close_reason or 'closed' end + return true, r, nil + end + return compute_interest() + end + + local function register(task, waker, want) + local tokens = {} + local active = true + + local function make_wake(reason) + return { + run = function () + if not active then return end + wake_reason = reason + active = false + task:run() + end, + } + end + + if type(want) == 'table' then + if want.fd ~= nil then + if want.rd then + tokens[#tokens + 1] = poller.get():wait(want.fd, 'rd', make_wake('fd')) + end + if want.wr then + tokens[#tokens + 1] = poller.get():wait(want.fd, 'wr', make_wake('fd')) + end + end + + if type(want.timeout) == 'number' then + waker:after(want.timeout, make_wake('timeout')) + end + end + + tokens[#tokens + 1] = self._state_waiters:add('state', make_wake('poke')) + + return { + unlink = function () + active = false + return token2(tokens):unlink() + end, + } + end + + return wait.waitable2(register, probe_step, run_step) + end) +end + +M.pollable_ready_op = function (driver, pollable, opts) + return driver:pollable_ready_op(pollable, opts) +end + +--- Wait until a pollable is ready, then call step(0). +function Driver:pollable_step_op(pollable, step_fn) + return self:pollable_ready_op(pollable):wrap(function (reason, err) + if reason == nil then return nil, err end + return self:_step_pollable_now(pollable, step_fn) + end) +end + +function Driver:_complete_job(job, ok, results_or_err) + if job.done or job.abandoned then return end + + job.done = true + job.phase = 'done' + job.ok = ok + if ok then job.results = results_or_err or pack() else job.err = results_or_err end + + self._jobs[job] = nil + notify_waitset(self._job_waiters, job) + self:_signal_changed() +end + +function Driver:_finish_detached_job(job, ok, results_or_err) + if job.done then return end + + job.done = true + job.phase = 'detached_done' + job.ok = ok + if ok then job.results = results_or_err or pack() else job.err = results_or_err end + + local cleanup = job.on_detached_complete + if cleanup then safe.pcall(cleanup, job.abort_reason or 'aborted', ok, results_or_err, job) end + + self._jobs[job] = nil + notify_waitset(self._job_waiters, job) + self:_signal_changed() +end + +function Driver:_abandon_job(job, reason) + if job.done or job.abandoned then return end + + local abort_reason = reason or 'aborted' + local phase = job.phase or 'queued' + + -- Ownership invariant: once a cqueues/lua-http job has become active, the + -- descriptor-owning coroutine remains the only place that may clean up + -- cqueues-owned request, stream, connection, or socket state. A Fibers + -- caller losing interest is therefore detached from the result; it must not + -- synchronously close or cancel descriptors that may still be installed in + -- the cqueues controller. on_detached_complete is run after the cqueues + -- coroutine returns, when that ownership boundary is safe again. + + job.abandoned = true + job.abort_reason = abort_reason + + if phase == 'active' and job.detach_on_abort then + job.detached = true + job.phase = 'detached' + local on_detach = job.on_detach + if on_detach then safe.pcall(on_detach, abort_reason, job) end + notify_waitset(self._job_waiters, job) + self:_signal_changed() + return + end + + job.phase = 'abandoned' + self._jobs[job] = nil + + if phase == 'active' then + local on_active_abort = job.on_active_abort + if on_active_abort then + safe.pcall(on_active_abort, abort_reason, job) + else + -- Once cqueues/lua-http work is active, a plain loss of interest is + -- not enough. If no narrower owner supplied an active-abort hook, + -- the driver is the only safe owner boundary left. + self:terminate(abort_reason) + end + else + local on_abort = job.on_abort + if on_abort then safe.pcall(on_abort, abort_reason, job) end + end + + notify_waitset(self._job_waiters, job) + self:_signal_changed() +end + +function Driver:_start_job(label, fn, opts) + opts = opts or {} + assert(type(fn) == 'function', 'run_op expects a function') + + if self._closed then return nil, self._close_reason or 'closed' end + + local controller = self._controller + if not controller then return nil, 'driver has no cqueues controller' end + + local job = { + label = label or 'cqueues job', + phase = 'queued', + done = false, + ok = nil, + results = nil, + err = nil, + abandoned = false, + detached = false, + abort_reason = nil, + detach_on_abort = opts.detach_on_abort, + on_abort = opts.on_abort, + on_active_abort = opts.on_active_abort, + on_detach = opts.on_detach, + on_detached_complete = opts.on_detached_complete, + } + + self._jobs[job] = true + + local function body() + if job.abandoned then return end + job.phase = 'active' + local ok, result = safe.xpcall(function () + return pack(fn()) + end, function (e, tb) + return tb or tostring_error(e) + end) + + if job.abandoned then + self:_finish_detached_job(job, ok, result or 'cqueues job failed') + return + end + if ok then + self:_complete_job(job, true, result) + else + self:_complete_job(job, false, result or 'cqueues job failed') + end + end + + local ok, err = safe.pcall(function () + local wrap_fn = self._wrap_fn + if wrap_fn then return wrap_fn(controller, body) end + return controller:wrap(body) + end) + + if not ok then + self._jobs[job] = nil + return nil, tostring_error(err) + end + + self:_signal_changed() + return job, nil +end + +function Driver:_job_outcome_op(job) + local function step() + if job.abandoned then return true, nil, job.abort_reason or 'aborted' end + + if job.done then + if job.ok then + local r = job.results or pack() + return true, unpack(r, 1, r.n) + end + return true, nil, job.err or 'cqueues job failed' + end + if self._closed then return true, nil, self._close_reason or 'closed' end + + return false + end + + local function register(task) + return self._job_waiters:add(job, task) + end + + return wait.waitable(register, step, identity) +end + +--- Run a cqueues coroutine operation and expose its result as a Fibers Op. +--- The function is executed inside the driver's cqueues controller. +function Driver:run_op(label, fn, opts) + return op.guard(function () + local job, err = self:_start_job(label, fn, opts) + if not job then return op.always(nil, err) end + + return self:_job_outcome_op(job):on_abort(function () + self:_abandon_job(job, 'aborted') + end) + end) +end + +--- Pump the driver's own cqueues controller until terminated. +function Driver:run() + while not self._closed do + local ok, err = perform(self:pollable_step_op(self._controller, function () + return self:_step_controller_now() + end)) + if ok == nil then + self:terminate(err or 'cqueues pump failed') + return nil, err + end + end + return true, nil +end + +function Driver:start(scope) + assert(scope and scope.spawn, 'Driver:start expects a scope') + if self._pump_started then return true, nil end + self._pump_started = true + + local ok, err = scope:spawn(function (owning_scope) + -- A scope that has already started may only have finalisers installed + -- from code running inside that same scope. Driver:start may be called + -- by setup work in a child scope, so the ownership finaliser is + -- installed by the pump fibre itself. + owning_scope:finally(function () + self:terminate('scope_finalised') + end) + + return self:run() + end) + if not ok then + self._pump_started = false + return nil, err + end + + return true, nil +end + +M.Driver = Driver + +return M diff --git a/src/services/http/transport/headers.lua b/src/services/http/transport/headers.lua new file mode 100644 index 00000000..0eefb48e --- /dev/null +++ b/src/services/http/transport/headers.lua @@ -0,0 +1,141 @@ +-- services/http/headers.lua +-- +-- Small boundary around lua-http header objects. The rest of the service should +-- not hand-build backend header objects or depend on their metatable shape. + +local M = {} + +local function require_headers() + local ok, mod = pcall(require, 'http.headers') + if not ok then return nil, mod end + return mod +end + +local function new_raw() + local headers, err = require_headers() + if not headers then return nil, err end + return headers.new() +end + +local function append_all(h, values) + for _, item in ipairs(values) do + if type(item) == 'table' then + h:append(tostring(item[1]), tostring(item[2] or '')) + else + return nil, 'invalid_header_value' + end + end + return true +end + +local function apply_table(h, tbl) + for k, v in pairs(tbl or {}) do + if type(k) == 'number' then + if type(v) ~= 'table' then return nil, 'invalid_header_pair' end + h:append(tostring(v[1]), tostring(v[2] or '')) + elseif type(v) == 'table' then + for _, vv in ipairs(v) do h:append(tostring(k), tostring(vv)) end + else + h:append(tostring(k), tostring(v)) + end + end + return true +end + +function M.is_backend_headers(v) + return type(v) == 'table' and type(v.get) == 'function' and type(v.append) == 'function' +end + +function M.new(fields) + local h, err = new_raw() + if not h then return nil, err end + local ok, aerr + if type(fields) == 'table' and fields[1] ~= nil and type(fields[1]) == 'table' then + ok, aerr = append_all(h, fields) + else + ok, aerr = apply_table(h, fields or {}) + end + if not ok then return nil, aerr end + return h +end + +function M.request(method, path, authority, scheme, fields) + local h, err = M.new(fields or {}) + if not h then return nil, err end + if method ~= nil then h:upsert(':method', tostring(method)) end + if path ~= nil then h:upsert(':path', tostring(path)) end + if authority ~= nil then h:upsert(':authority', tostring(authority)) end + if scheme ~= nil then h:upsert(':scheme', tostring(scheme)) end + return h +end + +function M.status(status, fields) + local h, err = M.new(fields or {}) + if not h then return nil, err end + h:upsert(':status', tostring(status or 200)) + return h +end + +function M.clone(headers) + if headers and type(headers.clone) == 'function' then return headers:clone() end + return M.new(M.to_table(headers)) +end + +function M.get_one(headers, name) + if not headers or type(headers.get) ~= 'function' then return nil end + return headers:get(string.lower(tostring(name))) +end + +function M.get_all(headers, name) + local out = {} + if not headers then return out end + name = string.lower(tostring(name)) + if type(headers.each) == 'function' then + for k, v in headers:each() do + if tostring(k):lower() == name then out[#out + 1] = v end + end + return out + end + local v = M.get_one(headers, name) + if v ~= nil then out[1] = v end + return out +end + +function M.to_table(headers) + local out = {} + if not headers then return out end + if type(headers.each) == 'function' then + for k, v in headers:each() do + k = tostring(k) + if out[k] == nil then + out[k] = v + elseif type(out[k]) == 'table' then + out[k][#out[k] + 1] = v + else + out[k] = { out[k], v } + end + end + return out + end + for k, v in pairs(headers) do out[k] = v end + return out +end + +function M.to_pairs(headers) + local out = {} + if not headers then return out end + if type(headers.each) == 'function' then + for k, v in headers:each() do out[#out + 1] = { k, v } end + return out + end + for k, v in pairs(headers) do + if type(v) == 'table' then + for _, vv in ipairs(v) do out[#out + 1] = { k, vv } end + else + out[#out + 1] = { k, v } + end + end + return out +end + +return M diff --git a/src/services/http/transport/init.lua b/src/services/http/transport/init.lua new file mode 100644 index 00000000..aeb35de1 --- /dev/null +++ b/src/services/http/transport/init.lua @@ -0,0 +1,9 @@ +-- services/http/transport/init.lua + +return { + cqueues_driver = require 'services.http.transport.cqueues_driver', + lua_http = require 'services.http.transport.lua_http', + websocket = require 'services.http.transport.websocket', + request_body = require 'services.http.transport.request_body', + terminate = require 'services.http.transport.terminate', +} diff --git a/src/services/http/transport/legacy_http1_close.lua b/src/services/http/transport/legacy_http1_close.lua new file mode 100644 index 00000000..b6a2be63 --- /dev/null +++ b/src/services/http/transport/legacy_http1_close.lua @@ -0,0 +1,329 @@ +-- services/http/transport/legacy_http1_close.lua +-- +-- Strict, bounded HTTP/1.0 close-delimited client transport for legacy embedded +-- devices whose response headers are rejected by lua-http. This is not a raw +-- socket escape hatch: it is selected explicitly by response_parser = +-- "legacy-http1-close" and is still admitted through services.http policy. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local socket = require 'fibers.io.socket' +local headers_mod = require 'services.http.transport.headers' + +local M = {} + +local Stream = {} +Stream.__index = Stream + +local function new_stream(body) + return setmetatable({ body = body or '', off = 1, closed = false }, Stream) +end + +function Stream:get_next_chunk() + if self.closed then return nil end + if self.off > #self.body then return nil end + local chunk = self.body:sub(self.off) + self.off = #self.body + 1 + return chunk +end + +function Stream:get_body_chars(n) + if self.closed then return nil, 'closed' end + n = tonumber(n) or 0 + if n <= 0 then return '' end + local chunk = self.body:sub(self.off, self.off + n - 1) + self.off = self.off + #chunk + return chunk +end + +function Stream:get_body_as_string() + if self.closed then return nil, 'closed' end + local body = self.body:sub(self.off) + self.off = #self.body + 1 + return body +end + +function Stream:shutdown() + self.closed = true + return true +end + +function Stream:close() + return self:shutdown() +end + +local function target_from_uri(parsed) + -- The HTTP policy parser keeps the original URI as well as the normalised + -- path. Use the original URI to reconstruct the request target so that the + -- query string is never lost before the legacy transport reaches CGI-style + -- endpoints such as /cgi/get.cgi?cmd=panel_info. Some embedded servers do + -- not return promptly for /cgi/get.cgi without cmd=..., which makes this a + -- correctness issue rather than a cosmetic one. + local uri = parsed and parsed.uri + if type(uri) == 'string' then + local target = uri:match('^%a[%w+.-]*://[^/]*(/.*)$') + if target and target ~= '' then return target end + end + local path = parsed and parsed.path or '/' + if path == '' then path = '/' end + return path +end + +local function has_header(headers, name) + name = tostring(name or ''):lower() + for k in pairs(headers or {}) do + if tostring(k):lower() == name then return true end + end + return false +end + +local function collect_request_body(args, max_bytes) + local source = args._request_source + local iter = args._request_body + if source == nil and iter == nil then return '', nil end + max_bytes = tonumber(max_bytes) or (1024 * 1024) + if max_bytes <= 0 then return nil, 'invalid_max_request_bytes' end + local out, total = {}, 0 + local function add(chunk) + chunk = tostring(chunk) + total = total + #chunk + if total > max_bytes then return nil, 'request_body_too_large' end + out[#out + 1] = chunk + return true, nil + end + if source ~= nil then + while true do + local chunk, err = fibers.perform(source:read_chunk_op(65536)) + if err then return nil, err end + if chunk == nil then break end + local ok, aerr = add(chunk) + if not ok then return nil, aerr end + end + else + while true do + local chunk = iter() + if chunk == nil then break end + local ok, aerr = add(chunk) + if not ok then return nil, aerr end + end + end + return table.concat(out), nil +end + +local function render_request(args, body) + body = body or '' + local uri = args._uri or {} + local method = tostring(args.method or 'GET'):upper() + local lines = { + ('%s %s HTTP/1.0'):format(method, target_from_uri(uri)), + } + local headers = args.headers or {} + lines[#lines + 1] = 'Host: ' .. tostring(uri.authority or uri.host or '') + lines[#lines + 1] = 'Connection: close' + if body ~= '' then lines[#lines + 1] = 'Content-Length: ' .. tostring(#body) end + for k, v in pairs(headers) do + local lk = tostring(k):lower() + -- The legacy transport owns hop-by-hop and length framing. Policy has + -- already rejected CR/LF injection; avoid duplicate or contradictory fields. + if lk ~= 'host' and lk ~= 'connection' and lk ~= 'content-length' and lk ~= 'transfer-encoding' then + lines[#lines + 1] = tostring(k) .. ': ' .. tostring(v) + end + end + lines[#lines + 1] = '' + lines[#lines + 1] = body or '' + return table.concat(lines, '\r\n') +end + +local function parse_header_block(header_block) + header_block = tostring(header_block or '') + local status_line = header_block:match('^([^\r\n]+)') or '' + local status = status_line:match('^HTTP/%d+%.%d+%s+(%d%d%d)') + if not status then return nil, nil, 'invalid_status_line: ' .. status_line end + local pairs, content_length = {}, nil + for line in header_block:gmatch('[^\r\n]+') do + if line ~= status_line then + local k, v = line:match('^([^:]+):%s*(.*)$') + if k and k ~= '' then + local lk = tostring(k):lower() + v = v or '' + if lk == 'transfer-encoding' and tostring(v):lower() ~= 'identity' then + return nil, nil, 'unsupported_transfer_encoding' + end + if lk == 'content-length' then + content_length = tonumber(v) + if content_length == nil or content_length < 0 then return nil, nil, 'invalid_content_length' end + end + -- Store normal header names in lower case. The rest of the + -- HTTP service uses lower-case lookups at the boundary, and + -- http.headers:get() is not guaranteed to perform + -- case-insensitive lookup for hand-built headers. + pairs[#pairs + 1] = { lk, v } + end + end + end + local headers, herr = headers_mod.status(status, pairs) + if not headers then return nil, nil, herr end + return headers, content_length, nil +end + +local function parse_response(raw) + raw = tostring(raw or '') + local header_block, response_body = raw:match('^(.-\r\n\r\n)(.*)$') + if not header_block then header_block, response_body = raw:match('^(.-\n\n)(.*)$') end + if not header_block then return nil, nil, 'no_header_separator' end + local headers, _content_length, err = parse_header_block(header_block) + if not headers then return nil, nil, err end + return headers, new_stream(response_body or ''), nil +end + +local function find_header_separator(raw) + local a, b = raw:find('\r\n\r\n', 1, true) + if a then return a, b end + a, b = raw:find('\n\n', 1, true) + return a, b +end + +local function read_some_bounded(stream, want) + local chunk, err = fibers.perform(stream:read_some_op(want)) + if err then return nil, err end + return chunk, nil +end + +local function read_response_bounded(stream, max_bytes, method) + max_bytes = tonumber(max_bytes) or 0 + if max_bytes <= 0 then return nil, nil, 'invalid_max_response_bytes' end + + local chunks, total = {}, 0 + local header_start, header_end + while not header_end do + local remaining = max_bytes - total + if remaining <= 0 then return nil, nil, 'response_too_large' end + local chunk, err = read_some_bounded(stream, math.min(4096, remaining)) + if err then return nil, nil, err end + if chunk == nil then return nil, nil, 'connection_closed_before_headers' end + if chunk ~= '' then + total = total + #chunk + if total > max_bytes then return nil, nil, 'response_too_large' end + chunks[#chunks + 1] = chunk + local raw = table.concat(chunks) + header_start, header_end = find_header_separator(raw) + if header_end then + local header_block = raw:sub(1, header_end) + local body = raw:sub(header_end + 1) + local headers, content_length, perr = parse_header_block(header_block) + if not headers then return nil, nil, perr end + + if tostring(method or 'GET'):upper() == 'HEAD' then + return headers, new_stream(''), nil + end + + if content_length ~= nil then + if content_length < 0 then return nil, nil, 'invalid_content_length' end + while #body < content_length do + remaining = max_bytes - total + if remaining <= 0 then return nil, nil, 'response_too_large' end + local need = math.min(content_length - #body, 4096, remaining) + local more, rerr = read_some_bounded(stream, need) + if rerr then return nil, nil, rerr end + if more == nil then return nil, nil, 'connection_closed_before_body_complete' end + if more ~= '' then + total = total + #more + if total > max_bytes then return nil, nil, 'response_too_large' end + body = body .. more + end + end + return headers, new_stream(body:sub(1, content_length)), nil + end + + while true do + remaining = max_bytes - total + if remaining <= 0 then return nil, nil, 'response_too_large' end + local more, rerr = read_some_bounded(stream, math.min(4096, remaining)) + if rerr then return nil, nil, rerr end + if more == nil then break end + if more ~= '' then + total = total + #more + if total > max_bytes then return nil, nil, 'response_too_large' end + body = body .. more + end + end + return headers, new_stream(body), nil + end + end + end +end + +local function close_stream(stream) + if not stream then return end + if stream.close_op then fibers.perform(stream:close_op()) + elseif stream.close then stream:close() end +end + +local function open_once(args, opts, active) + local uri = args._uri or {} + if uri.scheme ~= 'http' then return nil, nil, 'unsupported_scheme' end + local body, berr = collect_request_body(args, opts.max_request_body or opts.max_request_body_bytes or 1024 * 1024) + if body == nil then return nil, nil, berr end + local stream, serr = socket.connect_inet(uri.host, uri.port or 80) + if not stream then return nil, nil, serr end + if active then active.stream = stream end + local owned = true + local function close() + if owned then + owned = false + if active then active.stream = nil end + close_stream(stream) + end + end + local ok, werr = fibers.perform(stream:write_op(render_request(args, body))) + if not ok then close(); return nil, nil, werr end + if stream.flush_op then fibers.perform(stream:flush_op()) end + local response_headers, body_stream, rerr = read_response_bounded( + stream, + args.max_response_bytes or 1048576, + args.method + ) + close() + if not response_headers then return nil, nil, rerr end + return response_headers, body_stream, nil +end + +function M.open_exchange_op(_driver, args, opts) + opts = opts or {} + return op.guard(function () + local active = { stream = nil } + local result = fibers.run_scope_op(function (scope) + scope:finally(function (_, status, primary) + local stream = active.stream + active.stream = nil + close_stream(stream) + end) + return open_once(args, opts, active) + end):wrap(function (status, _report, a, b, c) + if status == 'ok' then return a, b or c end + return nil, nil, a or status + end) + return op.named_choice({ + result = result, + timeout = sleep.sleep_op(args.timeout_s), + }):wrap(function (which, a, b, c) + if which == 'timeout' then + local stream = active.stream + active.stream = nil + close_stream(stream) + return nil, 'timeout' + end + return a, b or c + end) + end) +end + +M._test = { + render_request = render_request, + parse_response = parse_response, + read_response_bounded = read_response_bounded, + new_stream = new_stream, +} + +return M diff --git a/src/services/http/transport/lua_http.lua b/src/services/http/transport/lua_http.lua new file mode 100644 index 00000000..e28c05a9 --- /dev/null +++ b/src/services/http/transport/lua_http.lua @@ -0,0 +1,620 @@ +-- services/http/transport/lua_http.lua +-- +-- Fibers-facing lua-http backend wrapper. +-- +-- lua-http still owns HTTP parsing, HTTP/2 stream state, WebSocket handshake +-- plumbing, and cqueues coroutine execution. UI service code sees Fibers Ops +-- and transport-shaped objects. The lua-http onstream callback is kept at this +-- edge: it creates an HttpContext, admits it to the listener queue, and then +-- runs the stream command loop. + +local op = require 'fibers.op' +local wait = require 'fibers.wait' +local fifo = require 'fibers.utils.fifo' +local runtime = require 'fibers.runtime' +local safe = require 'coxpcall' + +local cqueues_driver = require 'services.http.transport.cqueues_driver' +local terminate = require 'services.http.transport.terminate' + +local unpack = rawget(table, 'unpack') or _G.unpack +local pack = rawget(table, 'pack') or function (...) + return { n = select('#', ...), ... } +end + +local M = {} + +local Listener = {} +Listener.__index = Listener + +local HttpContext = {} +HttpContext.__index = HttpContext + +local function copy_table(t) + local out = {} + if not t then return out end + for k, v in pairs(t) do out[k] = v end + return out +end + +local function tostring_error(e) + if type(e) == 'string' or type(e) == 'number' then return e end + return tostring(e) +end + +local function notify_waitset(ws, key) + local sched = runtime.current_scheduler + if sched then ws:notify_all(key, sched) end +end + +local function default_condition_factory() + local ok, cc = pcall(require, 'cqueues.condition') + if not ok or not cc then return nil, 'cqueues.condition module is not available' end + if type(cc.new) == 'function' then return cc.new() end + if type(cc) == 'function' then return cc() end + return nil, 'cqueues.condition has no constructor' +end + +local function signal_condition(cv) + if cv and type(cv.signal) == 'function' then return cv:signal() end + return nil, 'condition has no signal method' +end + +local function wait_condition(cv) + if cv and type(cv.wait) == 'function' then return cv:wait() end + return nil, 'condition has no wait method' +end + +local function make_server(opts) + if opts.server then return opts.server end + + local http_server = opts.http_server + if not http_server then + local ok, mod = pcall(require, 'http.server') + if not ok then return nil, mod end + http_server = mod + end + + local listen_opts = copy_table(opts.server_options or opts) + listen_opts.server = nil + listen_opts.http_server = nil + listen_opts.driver = nil + listen_opts.driver_options = nil + listen_opts.max_accept_queue = nil + listen_opts.condition_factory = nil + listen_opts.context_terminator = nil + listen_opts.on_context = nil + listen_opts.on_context_admitted = nil + listen_opts.on_context_transferred = nil + listen_opts.on_context_terminated = nil + listen_opts.on_terminate = nil + listen_opts.backend_timeout = nil + + return http_server.listen(listen_opts) +end + +local next_context_id = 0 + +local function new_context(listener, server, stream) + next_context_id = next_context_id + 1 + + local factory = listener._condition_factory or default_condition_factory + local cv, err = factory() + if not cv then error(err or 'failed to create cqueues condition', 2) end + + return setmetatable({ + _id = next_context_id, + _listener = listener, + _driver = listener._driver, + _server = server, + _stream = stream, + _cmdq = fifo.new(), + _cmd_cv = cv, + _cmd_waiters = wait.new_waitset(), + _closed = false, + _close_reason = nil, + _terminator = listener._context_terminator, + _active_cmd = nil, + _commands_started = 0, + _commands_done = 0, + _stream_timeout = listener._intra_stream_timeout, + }, HttpContext) +end + +function HttpContext:id() return self._id end +function HttpContext:_raw_stream_for_test() return self._stream end +function HttpContext:_raw_server_for_test() return self._server end +function HttpContext:is_closed() return self._closed end +function HttpContext:why() return self._close_reason end + +function HttpContext:_notify_public_terminated(reason) + local ws = rawget(self, '_http_transport_websocket') + if ws and type(ws._notify_terminated) == 'function' then + ws:_notify_terminated(reason or self._close_reason or 'closed') + end + + local public = rawget(self, '_http_public_context') + if public and type(public._notify_terminated) == 'function' then + public:_notify_terminated(reason or self._close_reason or 'closed') + end +end + +function HttpContext:_complete_command(cmd, ok, results_or_err) + if cmd.done or cmd.abandoned then return end + cmd.done = true + cmd.phase = 'done' + cmd.ok = ok + if ok then cmd.results = results_or_err or pack() else cmd.err = results_or_err end + self._commands_done = self._commands_done + 1 + notify_waitset(self._cmd_waiters, cmd) +end + +function HttpContext:_abandon_command(cmd, reason) + if cmd.done or cmd.abandoned then return end + + local abort_reason = reason or 'aborted' + local phase = cmd.phase or 'queued' + + cmd.abandoned = true + cmd.abort_reason = abort_reason + cmd.phase = 'abandoned' + + if phase == 'active' then + if cmd.on_active_abort then + safe.pcall(cmd.on_active_abort, abort_reason, cmd) + else + -- Active stream work may already have consumed request bytes, emitted + -- response bytes, or changed HTTP/2 stream state. Losing interest in + -- the Fibers waiter must therefore close the owning context. + self:terminate(abort_reason) + end + elseif cmd.on_abort then + safe.pcall(cmd.on_abort, abort_reason, cmd) + end + + notify_waitset(self._cmd_waiters, cmd) +end + +function HttpContext:_enqueue_command(label, fn, opts) + opts = opts or {} + if self._closed then return nil, self._close_reason or 'closed' end + assert(type(fn) == 'function', 'run_stream_op expects a function') + + local cmd = { + label = label or 'http_stream_command', + phase = 'queued', + fn = fn, + done = false, + ok = nil, + results = nil, + err = nil, + abandoned = false, + abort_reason = nil, + on_abort = opts.on_abort, + on_active_abort = opts.on_active_abort, + } + + self._commands_started = self._commands_started + 1 + self._cmdq:push(cmd) + -- The onstream coroutine waits on a cqueues condition. Signalling that + -- condition is the precise wake-up; poking the shared HTTP driver here also + -- wakes unrelated listener pumps and can turn a context command into a + -- service-wide busy loop under simple pollable fakes. + signal_condition(self._cmd_cv) + return cmd, nil +end + +function HttpContext:_command_outcome_op(cmd) + local function step() + if cmd.done then + if cmd.ok then + local r = cmd.results or pack() + return true, unpack(r, 1, r.n) + end + return true, nil, cmd.err or 'http stream command failed' + end + if cmd.abandoned then return true, nil, cmd.abort_reason or 'aborted' end + if self._closed then return true, nil, self._close_reason or 'closed' end + return false + end + + local function register(task) + return self._cmd_waiters:add(cmd, task) + end + + return wait.waitable(register, step) +end + +--- Run fn(stream, context) inside the original lua-http onstream coroutine. +function HttpContext:run_stream_op(label, fn, opts) + return op.guard(function () + local cmd, err = self:_enqueue_command(label, fn, opts) + if not cmd then return op.always(nil, err) end + + return self:_command_outcome_op(cmd):on_abort(function () + self:_abandon_command(cmd, 'aborted') + end) + end) +end + +function HttpContext:_run_one_command(cmd) + if cmd.abandoned then return end + + cmd.phase = 'active' + self._active_cmd = cmd + local ok, results_or_err = safe.xpcall(function () + return pack(cmd.fn(self._stream, self)) + end, function (e, tb) + return tb or tostring_error(e) + end) + self._active_cmd = nil + + if cmd.abandoned then return end + if ok then + self:_complete_command(cmd, true, results_or_err) + else + self:_complete_command(cmd, false, results_or_err) + end +end + +-- Test hook: run one queued command without a real cqueues condition loop. +function HttpContext:_drain_one_for_test() + if self._cmdq:empty() then return false end + local cmd = self._cmdq:pop() + self:_run_one_command(cmd) + return true +end + +--- Runs inside lua-http's onstream cqueues coroutine. +function HttpContext:_command_loop() + while not self._closed do + local cmd + if not self._cmdq:empty() then cmd = self._cmdq:pop() end + + if cmd then + self:_run_one_command(cmd) + else + wait_condition(self._cmd_cv) + end + end +end + +function HttpContext:terminate(reason) + if self._closed then return true end + + self._closed = true + self._close_reason = reason or 'closed' + + while not self._cmdq:empty() do + self:_abandon_command(self._cmdq:pop(), self._close_reason) + end + if self._active_cmd then self:_abandon_command(self._active_cmd, self._close_reason) end + + local term = self._terminator + if term then + safe.pcall(term, self._stream, self._close_reason, self) + else + terminate.terminate_stream(self._stream, self._close_reason) + end + + -- Wake only the owning onstream command loop. Listener/server pump + -- termination is handled by Listener:terminate(). + signal_condition(self._cmd_cv) + + self:_notify_public_terminated(self._close_reason) + + local listener = self._listener + if listener then listener._contexts[self] = nil end + return true +end + +-- HTTP stream-shaped Ops ------------------------------------------------------ + +local function context_stream_timeout(ctx) + return ctx._stream_timeout +end + +function HttpContext:get_headers_op() + return self:run_stream_op('http.get_headers', function (stream) + return stream:get_headers(context_stream_timeout(self)) + end) +end + +function HttpContext:read_chunk_op(_max) + return self:run_stream_op('http.get_next_chunk', function (stream) + return stream:get_next_chunk(context_stream_timeout(self)) + end) +end + +function HttpContext:read_chars_op(n) + return self:run_stream_op('http.get_body_chars', function (stream) + return stream:get_body_chars(n, context_stream_timeout(self)) + end) +end + +function HttpContext:read_body_as_string_op() + return self:run_stream_op('http.get_body_as_string', function (stream) + return stream:get_body_as_string(context_stream_timeout(self)) + end) +end + +function HttpContext:write_headers_op(headers, end_stream) + return self:run_stream_op('http.write_headers', function (stream) + return stream:write_headers(headers, not not end_stream, context_stream_timeout(self)) + end) +end + +function HttpContext:write_chunk_op(chunk, end_stream) + return self:run_stream_op('http.write_chunk', function (stream) + return stream:write_chunk(chunk, not not end_stream, context_stream_timeout(self)) + end) +end + +function HttpContext:write_body_from_string_op(str) + return self:run_stream_op('http.write_body_from_string', function (stream) + return stream:write_body_from_string(str, context_stream_timeout(self)) + end) +end + +function HttpContext:peername_op() + return self:run_stream_op('http.peername', function (stream) + if type(stream.peername) ~= 'function' then return nil, 'peername_not_available' end + return stream:peername() + end) +end + +function HttpContext:localname_op() + return self:run_stream_op('http.localname', function (stream) + if type(stream.localname) ~= 'function' then return nil, 'localname_not_available' end + return stream:localname() + end) +end + +function HttpContext:checktls_op() + return self:run_stream_op('http.checktls', function (stream) + if type(stream.checktls) ~= 'function' then return nil, 'checktls_not_available' end + return stream:checktls() + end) +end + +function HttpContext:connection_version_op() + return self:run_stream_op('http.connection_version', function (stream) + if type(stream.connection_version) ~= 'function' then return nil, 'connection_version_not_available' end + return stream:connection_version() + end) +end + +function HttpContext:write_continue_op() + return self:run_stream_op('http.write_continue', function (stream) + if type(stream.write_continue) ~= 'function' then return nil, 'write_continue_not_available' end + return stream:write_continue() + end) +end + +function HttpContext:unget_op(str) + return self:run_stream_op('http.unget', function (stream) + if type(stream.unget) ~= 'function' then return nil, 'unget_not_available' end + return stream:unget(str) + end) +end + +function HttpContext:shutdown_op() + return self:run_stream_op('http.stream_shutdown', function (stream) + return stream:shutdown() + end) +end + +-- Listener ------------------------------------------------------------------- + +function Listener:_notify_accept() + local sched = runtime.current_scheduler + if sched then self._accept_waiters:notify_one('accept', sched) end +end + +function Listener:_notify_admission() + local sched = runtime.current_scheduler + if sched then self._admission_waiters:notify_one('admission', sched) end +end + +function Listener:admission_op() + local function step() + if not self._admissionq:empty() then + return true, self._admissionq:pop(), nil + end + if self._closed then return true, nil, self._close_reason or 'closed' end + return false + end + + local function register(task) + return self._admission_waiters:add('admission', task) + end + + return wait.waitable(register, step) +end + +function Listener:_admit_context(ctx) + if self._closed then + ctx:terminate(self._close_reason or 'listener_closed') + return nil, 'closed' + end + + if self._acceptq:length() >= self._max_accept_queue then + ctx:terminate('accept_queue_full') + return nil, 'accept_queue_full' + end + + self._contexts[ctx] = true + self._acceptq:push(ctx) + self._admissionq:push(ctx) + self:_notify_admission() + self:_notify_accept() + return true, nil +end + +function Listener:_onstream(server, stream) + local ctx = new_context(self, server, stream) + local ok = self:_admit_context(ctx) + if not ok then return end + ctx:_command_loop() +end + +function M.listen(opts) + opts = opts or {} + + local driver = opts.driver + if not driver then driver = assert(cqueues_driver.new(opts.driver_options or {})) end + + local self = setmetatable({ + _driver = driver, + _server = nil, + _acceptq = fifo.new(), + _accept_waiters = wait.new_waitset(), + _admissionq = fifo.new(), + _admission_waiters = wait.new_waitset(), + _closed = false, + _close_reason = nil, + _max_accept_queue = opts.max_accept_queue or 64, + _contexts = {}, + _condition_factory = opts.condition_factory, + _context_terminator = opts.context_terminator, + _backend_timeout = opts.backend_timeout, + _intra_stream_timeout = opts.intra_stream_timeout, + }, Listener) + + local server_opts = copy_table(opts) + if server_opts.cq == nil and driver.controller then server_opts.cq = driver:controller() end + server_opts.onstream = function (server, stream) + return self:_onstream(server, stream) + end + server_opts.driver = nil + server_opts.max_accept_queue = nil + server_opts.condition_factory = nil + server_opts.context_terminator = nil + server_opts.on_context = nil + server_opts.on_context_admitted = nil + server_opts.on_context_transferred = nil + server_opts.on_context_terminated = nil + server_opts.on_terminate = nil + server_opts.driver_options = nil + server_opts.backend_timeout = nil + server_opts.intra_stream_timeout = nil + + local server, err = make_server(server_opts) + if not server then return nil, err end + self._server = server + return self +end + +function Listener:_raw_server_for_test() return self._server end +function Listener:localname() + if self._server and type(self._server.localname) == 'function' then return self._server:localname() end + return nil, 'localname_not_available' +end +function Listener:is_closed() return self._closed end +function Listener:why() return self._close_reason end + +function Listener:accept_op() + local function step() + if not self._acceptq:empty() then + local ctx = self._acceptq:pop() + -- Ownership transfer: once accepted, the caller/request scope owns + -- normal use and graceful response work. The listener keeps only + -- unaccepted contexts for listener-close cleanup. + self._contexts[ctx] = nil + return true, ctx + end + if self._closed then return true, nil, self._close_reason or 'closed' end + return false + end + + local function register(task) + return self._accept_waiters:add('accept', task) + end + + return wait.waitable(register, step) +end + +function Listener:pump_once_op() + return self._driver:pollable_step_op(self._server, function (server) + return server:step(0) + end) +end + +function Listener:run() + local perform = require 'fibers.performer'.perform + while not self._closed do + local ok, err = perform(self:pump_once_op()) + if ok == nil then + self:terminate(err or 'lua-http server pump failed') + return nil, err + end + end + return true, nil +end + +function Listener:start(scope) + assert(scope and scope.spawn, 'Listener:start expects a scope') + local ok, err = scope:spawn(function (owning_scope) + -- Listener:start may be called from a setup or request operation while + -- the listener pump is deliberately owned by another scope, typically + -- the HTTP service scope. Once that target scope has started, its + -- finalisers must be installed from inside it. + owning_scope:finally(function () + self:terminate('scope_finalised') + end) + + return self:run() + end) + if not ok then return nil, err end + return true, nil +end + +function Listener:listen_op() + return self._driver:run_op('http.server.listen', function () + return self._server:listen(self._backend_timeout) + end) +end + +function Listener:pause() + if self._server and type(self._server.pause) == 'function' then return self._server:pause() end + return true +end + +function Listener:resume() + if self._server and type(self._server.resume) == 'function' then return self._server:resume() end + return true +end + +function Listener:terminate(reason) + if self._closed then return true end + + self._closed = true + self._close_reason = reason or 'closed' + + terminate.terminate_server(self._server, self._close_reason) + + local contexts = {} + for ctx in pairs(self._contexts) do contexts[#contexts + 1] = ctx end + for _, ctx in ipairs(contexts) do ctx:terminate(self._close_reason) end + while not self._acceptq:empty() do + local ctx = self._acceptq:pop() + if ctx and (type(ctx.is_closed) ~= 'function' or not ctx:is_closed()) then + ctx:terminate(self._close_reason) + end + end + + local sched = runtime.current_scheduler + if sched then + self._accept_waiters:notify_all('accept', sched) + self._admission_waiters:notify_all('admission', sched) + end + if self._driver and self._driver.poke then self._driver:poke() end + return true +end + +M.Listener = Listener +M.HttpContext = HttpContext +M._new_context_for_test = new_context +M._default_condition_factory = default_condition_factory + +return M diff --git a/src/services/http/transport/request_body.lua b/src/services/http/transport/request_body.lua new file mode 100644 index 00000000..b7ab836b --- /dev/null +++ b/src/services/http/transport/request_body.lua @@ -0,0 +1,153 @@ +-- services/http/transport/request_body.lua +-- Bounded bridge from a Fibers-native body source into lua-http's request body +-- iterator contract. Bytes stay in process-local memory and never cross the +-- bus. The iterator is consumed by lua-http in a cqueues coroutine; the source +-- side is written by a Fibers worker through write_chunk_op()/finish_op(). + +local op = require 'fibers.op' +local cond = require 'fibers.cond' + +local M = {} +local Pipe = {} +Pipe.__index = Pipe + +local function default_condition_factory() + local cc = require 'cqueues.condition' + if type(cc.new) == 'function' then return cc.new() end + if type(cc) == 'function' then return cc() end + return nil, 'cqueues.condition has no constructor' +end + +local function signal_condition(cv) + if cv and type(cv.signal) == 'function' then return cv:signal() end + return nil, 'condition has no signal method' +end + +local function wait_condition(cv) + if cv and type(cv.wait) == 'function' then return cv:wait() end + return nil, 'condition has no wait method' +end + +local function new_space_cond(self) + self._space_cond = cond.new() + return self._space_cond +end + +local function signal_space(self) + local c = self._space_cond + new_space_cond(self) + if c then c:signal() end +end + +local function pop_first(q) + local v = q[1] + for i = 1, #q - 1 do q[i] = q[i + 1] end + q[#q] = nil + return v +end + +function M.new_pipe(opts) + opts = opts or {} + local factory = opts.condition_factory or default_condition_factory + local cv, err = factory() + if not cv then return nil, err or 'request_body_condition_failed' end + + local max_chunks = opts.max_buffered_chunks or 8 + if type(max_chunks) ~= 'number' or max_chunks < 1 then return nil, 'invalid_args' end + + local self = setmetatable({ + _cv = cv, + _q = {}, + _max_chunks = max_chunks, + _closed = false, + _done = false, + _err = nil, + _bytes = 0, + }, Pipe) + new_space_cond(self) + return self, nil +end + +function Pipe:stats() + return { + queued = #self._q, + bytes = self._bytes, + closed = self._closed, + done = self._done, + err = self._err, + } +end + +function Pipe:_push(chunk) + if self._closed then return nil, self._err or 'closed' end + if self._done then return nil, 'request_body_finished' end + if type(chunk) ~= 'string' then return nil, 'invalid_args' end + self._q[#self._q + 1] = chunk + self._bytes = self._bytes + #chunk + signal_condition(self._cv) + return true, nil +end + +function Pipe:write_chunk_op(chunk) + return op.guard(function () + if self._closed then return op.always(nil, self._err or 'closed') end + if #self._q < self._max_chunks then return op.always(self:_push(chunk)) end + + return self._space_cond:wait_op():wrap(function () + if self._closed then return nil, self._err or 'closed' end + if #self._q >= self._max_chunks then return nil, 'request_body_backpressure_wake_failed' end + return self:_push(chunk) + end) + end) +end + +function Pipe:finish_op() + return op.guard(function () + if self._closed then return op.always(nil, self._err or 'closed') end + self._done = true + signal_condition(self._cv) + return op.always(true, nil) + end) +end + +function Pipe:fail(reason) + if self._closed then return true end + self._err = reason or 'request_body_failed' + self._closed = true + self._q = {} + signal_condition(self._cv) + signal_space(self) + return true +end + +function Pipe:terminate(reason) + return self:fail(reason or 'request_body_terminated') +end + +function Pipe:body_iterator() + return function () + while not self._closed and not self._done and #self._q == 0 do + local ok, err = wait_condition(self._cv) + if ok == nil and err ~= nil then error(err, 0) end + end + + if self._closed then error(self._err or 'request_body_closed', 0) end + + if #self._q > 0 then + local chunk = pop_first(self._q) + signal_space(self) + return chunk + end + + return nil + end +end + +function M.new_body(opts) + local pipe, err = M.new_pipe(opts) + if not pipe then return nil, err end + return pipe:body_iterator(), pipe +end + +M.Pipe = Pipe +return M diff --git a/src/services/http/transport/terminate.lua b/src/services/http/transport/terminate.lua new file mode 100644 index 00000000..dc553940 --- /dev/null +++ b/src/services/http/transport/terminate.lua @@ -0,0 +1,94 @@ +-- services/http/transport/terminate.lua +-- Immediate backend termination helpers. These are the only transport-level +-- helpers intended for finaliser-safe shutdown paths. Graceful protocol close +-- remains Op-based work owned by callers. + +local safe = require 'coxpcall' + +local M = {} + +local unpack = rawget(table, 'unpack') or _G.unpack + +local function call(method, obj, ...) + if obj and type(obj[method]) == 'function' then + local args = { n = select('#', ...), ... } + return safe.pcall(function () return obj[method](obj, unpack(args, 1, args.n)) end) + end + return false, 'method_missing' +end + +local function get_field(obj, key) + if not obj then return nil end + local ok, value = safe.pcall(function () return obj[key] end) + if ok then return value end + return nil +end + +local function take_and_close_connection_socket(obj) + local conn = get_field(obj, 'connection') + if not conn then return false, 'connection_missing' end + + -- lua-http exposes h1 stream.connection and h1_connection:take_socket(). In + -- cancellation cleanup we need a transport boundary, not a graceful protocol + -- shutdown. Taking the cqueues socket and closing that socket lets cqueues + -- cancel its descriptor before close, which is the ordering cqueues requires. + if type(conn.take_socket) == 'function' then + local ok, sock = safe.pcall(function () return conn:take_socket() end) + if ok and sock then + local closed = call('close', sock) + if closed then return true end + end + end + + return false, 'take_socket_unavailable' +end + +function M.terminate_stream(stream, reason) + if not stream then return true end + local ok = take_and_close_connection_socket(stream) + if ok then return true end + ok = call('terminate', stream, reason or 'terminated') + if ok then return true end + ok = call('close', stream) + if ok then return true end + -- Last resort only. lua-http shutdown can be graceful and should not be the + -- first action on timeout/finaliser cleanup. + ok = call('shutdown', stream) + if ok then return true end + return true +end + +function M.terminate_server(server, reason) + if not server then return true end + local ok = call('close', server) + if ok then return true end + return true +end + +function M.terminate_websocket(ws, reason) + if not ws then return true end + -- This is deliberately not graceful. It gives lua-http/cqueues a bounded + -- immediate wake/close path for finalisers and service shutdown. + local ok = call('close', ws, 1006, reason or 'terminated', 0) + if ok then return true end + ok = call('shutdown', ws) + if ok then return true end + return true +end + +function M.terminate_request(req, reason) + if not req then return true end + local ok = take_and_close_connection_socket(req) + if ok then return true end + ok = call('terminate', req, reason or 'terminated') + if ok then return true end + ok = call('cancel', req, reason or 'terminated') + if ok then return true end + ok = call('close', req) + if ok then return true end + ok = call('shutdown', req) + if ok then return true end + return true +end + +return M diff --git a/src/services/http/transport/tls.lua b/src/services/http/transport/tls.lua new file mode 100644 index 00000000..cf278208 --- /dev/null +++ b/src/services/http/transport/tls.lua @@ -0,0 +1,20 @@ +-- services/http/transport/tls.lua +local M = {} +local function require_tls() + local ok, mod = pcall(require, 'http.tls') + if not ok then return nil, mod end + return mod +end +function M.new_server_context(opts) + local tls, err = require_tls() + if not tls then return nil, err end + if type(tls.new_server_context) == 'function' then return tls.new_server_context(opts or {}) end + return nil, 'http.tls.new_server_context is not available' +end +function M.new_client_context(opts) + local tls, err = require_tls() + if not tls then return nil, err end + if type(tls.new_client_context) == 'function' then return tls.new_client_context(opts or {}) end + return nil, 'http.tls.new_client_context is not available' +end +return M diff --git a/src/services/http/transport/uri.lua b/src/services/http/transport/uri.lua new file mode 100644 index 00000000..b564d360 --- /dev/null +++ b/src/services/http/transport/uri.lua @@ -0,0 +1,76 @@ +-- services/http/transport/uri.lua +-- lua-http-backed URI parsing boundary for the HTTP capability service. +-- +-- Policy code should not keep a separate hand-written parser for the same URI +-- shape that lua-http will later consume. This module uses lua-http's request +-- constructor and utility helpers to produce the small normalised view needed by +-- service policy without starting any network work. + +local http_request = require 'http.request' +local http_util = require 'http.util' + +local M = {} + +local function normalise_error(err) + local s = tostring(err or ''):lower() + if s:find('scheme not valid', 1, true) or s:find('unknown scheme', 1, true) then + return 'unsupported_scheme' + end + return 'invalid_args' +end + +local function leading_scheme(uri) + return uri:match('^([%a][%w+.-]*)://') +end + +local function lua_http_uri(uri, scheme) + -- http.request.new_from_uri is the authority/path parser used by the + -- backend. WebSocket URIs are HTTP Upgrade requests, but lua-http's + -- request constructor is deliberately HTTP(S)-shaped, so parse ws/wss via + -- the corresponding HTTP scheme and restore the public scheme below. + if scheme == 'ws' then + return 'http://' .. uri:sub(6), 'http' + end + if scheme == 'wss' then + return 'https://' .. uri:sub(7), 'https' + end + return uri, scheme +end + +function M.parse(uri) + if type(uri) ~= 'string' or uri == '' then return nil, 'invalid_args' end + + local public_scheme = leading_scheme(uri) + if type(public_scheme) ~= 'string' then return nil, 'invalid_args' end + public_scheme = public_scheme:lower() + + local parse_uri, authority_scheme = lua_http_uri(uri, public_scheme) + local ok, req_or_err = pcall(function () + return http_request.new_from_uri(parse_uri) + end) + if not ok or not req_or_err then return nil, normalise_error(req_or_err) end + + local req = req_or_err + local headers = req.headers + local parsed_scheme = headers and headers:get(':scheme') or nil + local authority = headers and headers:get(':authority') or nil + local path = headers and headers:get(':path') or nil + if type(parsed_scheme) ~= 'string' or parsed_scheme == '' then return nil, 'invalid_args' end + if type(authority) ~= 'string' or authority == '' then return nil, 'invalid_args' end + + local ok_split, host, port = pcall(http_util.split_authority, authority, authority_scheme) + if not ok_split then return nil, 'invalid_args' end + if type(host) ~= 'string' or host == '' then return nil, 'invalid_args' end + + return { + uri = uri, + scheme = public_scheme, + authority = authority, + host = host, + port = port, + path = path, + request = req, + }, nil +end + +return M diff --git a/src/services/http/transport/websocket.lua b/src/services/http/transport/websocket.lua new file mode 100644 index 00000000..506369d6 --- /dev/null +++ b/src/services/http/transport/websocket.lua @@ -0,0 +1,251 @@ +-- services/http/transport/websocket.lua +-- +-- Fibers-facing WebSocket wrapper for lua-http WebSocket objects. +-- Underlying WebSocket methods run through the owning HttpContext command loop, +-- so service code sees Ops and does not depend on cqueues callbacks. + +local op = require 'fibers.op' +local terminate = require 'services.http.transport.terminate' +local headers_mod = require 'services.http.headers' + +local M = {} + +local WebSocket = {} +WebSocket.__index = WebSocket + +local function websocket_module(opts) + if opts and opts.websocket_module then return opts.websocket_module end + local ok, mod = pcall(require, 'http.websocket') + if not ok then return nil, mod end + return mod +end + +local function apply_headers(ws, fields) + if fields == nil then return true end + local headers = ws and ws.headers + if headers == nil then return nil, 'websocket_headers_not_supported' end + for _, pair in ipairs(headers_mod.to_pairs(fields)) do + local k, v = tostring(pair[1]), tostring(pair[2] or '') + if type(headers.append) == 'function' then headers:append(k, v) + elseif type(headers.upsert) == 'function' then headers:upsert(k, v) + elseif type(headers.set) == 'function' then headers:set(k, v) + else return nil, 'websocket_headers_not_supported' end + end + return true +end + +local function wrap(ctx, ws, opts) + local self = setmetatable({ + _ctx = ctx, + _ws = ws, + _closed = false, + _close_reason = nil, + _on_terminate = opts and opts.on_terminate, + }, WebSocket) + if type(ctx) == 'table' then ctx._http_transport_websocket = self end + return self +end + +function M.from_context_op(ctx, headers, opts) + opts = opts or {} + return op.guard(function () + local websocket, err = websocket_module(opts) + if not websocket then return op.always(nil, err) end + + return ctx:run_stream_op('websocket.new_from_stream', function (stream) + return websocket.new_from_stream(stream, headers) + end):wrap(function (ws, werr) + if not ws then return nil, werr or 'not_websocket' end + return wrap(ctx, ws, opts), nil + end) + end) +end + +function WebSocket:_raw_websocket_for_test() return self._ws end +function WebSocket:context() return self._ctx end + +function WebSocket:_notify_terminated(reason) + if self._closed then return true end + self._closed = true + self._close_reason = reason or 'closed' + local hook = self._on_terminate + self._on_terminate = nil + if hook then pcall(hook, self, self._close_reason) end + return true +end + +function WebSocket:is_closed() + return self._closed or (self._ctx and self._ctx:is_closed()) +end + +function WebSocket:why() + return self._close_reason or (self._ctx and self._ctx:why()) +end + +function WebSocket:accept_op(options) + return self._ctx:run_stream_op('websocket.accept', function () + return self._ws:accept(options or {}) + end) +end + +function WebSocket:receive_op() + return self._ctx:run_stream_op('websocket.receive', function () + return self._ws:receive() + end) +end + +function WebSocket:send_op(data, opcode) + return self._ctx:run_stream_op('websocket.send', function () + return self._ws:send(data, opcode) + end) +end + +function WebSocket:send_ping_op(data) + return self._ctx:run_stream_op('websocket.send_ping', function () + return self._ws:send_ping(data) + end) +end + +function WebSocket:send_pong_op(data) + return self._ctx:run_stream_op('websocket.send_pong', function () + return self._ws:send_pong(data) + end) +end + +function WebSocket:close_op(code, reason) + return self._ctx:run_stream_op('websocket.close', function () + return self._ws:close(code, reason) + end):wrap(function (ok, err) + self:_notify_terminated(reason or err or 'closed') + return ok, err + end) +end + +--- Immediate termination path for finalisers. This is not a graceful protocol +--- close; graceful close belongs in close_op() inside a worker scope. +function WebSocket:terminate(reason) + if self._closed then return true end + local why = reason or 'closed' + if self._ctx then self._ctx:terminate(why) end + return self:_notify_terminated(why) +end + +-- Client-side WebSocket transport -------------------------------------------- + +local ClientWebSocket = {} +ClientWebSocket.__index = ClientWebSocket + +local function wrap_client(driver, ws, opts) + return setmetatable({ + _driver = driver, + _ws = ws, + _closed = false, + _close_reason = nil, + _on_terminate = opts and opts.on_terminate, + }, ClientWebSocket) +end + +function M.connect_op(driver, args, opts) + opts = opts or {} + args = args or {} + return op.guard(function () + if not driver or type(driver.run_op) ~= 'function' then + return op.always(nil, 'driver_required') + end + local websocket, werr = websocket_module(opts) + if not websocket then return op.always(nil, werr) end + if type(args.uri) ~= 'string' then return op.always(nil, 'invalid_args') end + + local active = { ws = nil } + return driver:run_op('http.websocket.connect', function () + local ws, err = websocket.new_from_uri(args.uri) + if not ws then return nil, err end + active.ws = ws + local hok, herr = apply_headers(ws, args.headers) + if hok ~= true then return nil, herr end + local ok, cerr = ws:connect(opts.backend_timeout) + if not ok then return nil, cerr end + return ws + end, { + on_active_abort = function (reason) + if active.ws then + terminate.terminate_websocket(active.ws, reason or 'websocket_connect_aborted') + elseif driver and driver.terminate then + driver:terminate(reason or 'websocket_connect_aborted') + end + end, + }):wrap(function (ws, err) + if not ws then return nil, err end + return wrap_client(driver, ws, opts), nil + end) + end) +end + +function ClientWebSocket:_raw_websocket_for_test() return self._ws end +function ClientWebSocket:is_closed() return self._closed end +function ClientWebSocket:why() return self._close_reason end + +function ClientWebSocket:_run(label, fn) + if self._closed then return op.always(nil, self._close_reason or 'closed') end + return self._driver:run_op(label, fn, { + on_active_abort = function (reason) + self:terminate(reason or 'websocket_op_aborted') + end, + }) +end + +function ClientWebSocket:receive_op() + return self:_run('http.websocket.receive', function () + return self._ws:receive() + end) +end + +function ClientWebSocket:send_op(data, opcode) + return self:_run('http.websocket.send', function () + return self._ws:send(data, opcode) + end) +end + +function ClientWebSocket:send_ping_op(data) + return self:_run('http.websocket.ping', function () + return self._ws:send_ping(data) + end) +end + +function ClientWebSocket:send_pong_op(data) + return self:_run('http.websocket.pong', function () + return self._ws:send_pong(data) + end) +end + +function ClientWebSocket:close_op(code, reason) + return self:_run('http.websocket.close', function () + return self._ws:close(code, reason) + end):wrap(function (ok, err) + self:_notify_terminated(reason or err or 'closed') + return ok, err + end) +end + +function ClientWebSocket:_notify_terminated(reason) + if self._closed then return true end + self._closed = true + self._close_reason = reason or 'closed' + local hook = self._on_terminate + self._on_terminate = nil + if hook then pcall(hook, self, self._close_reason) end + return true +end + +function ClientWebSocket:terminate(reason) + if self._closed then return true end + local why = reason or 'closed' + terminate.terminate_websocket(self._ws, why) + return self:_notify_terminated(why) +end + +M.ClientWebSocket = ClientWebSocket + +M.WebSocket = WebSocket + +return M diff --git a/src/services/http/websocket.lua b/src/services/http/websocket.lua new file mode 100644 index 00000000..df928d9e --- /dev/null +++ b/src/services/http/websocket.lua @@ -0,0 +1,249 @@ +-- services/http/websocket.lua +-- Public WebSocket transport handle boundary above services.http.transport. + +local transport_ws = require 'services.http.transport.websocket' +local op = require 'fibers.op' + +local M = {} + +local WebSocket = {} +WebSocket.__index = WebSocket + +local ClientWebSocket = {} +ClientWebSocket.__index = ClientWebSocket + +local function copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +local function raw_context(ctx) + if ctx and type(ctx._raw_context) == 'function' then return ctx:_raw_context() end + return ctx +end + +local function notify_terminated(self, reason) + if self._closed then return true end + self._closed = true + self._close_reason = reason or 'closed' + local hooks = self._terminate_hooks or {} + self._terminate_hooks = {} + local hook = self._on_terminate + self._on_terminate = nil + if hook then pcall(hook, self, self._close_reason) end + for _, f in ipairs(hooks) do pcall(f, self, self._close_reason) end + return true +end + +local function wrap_server(raw, opts) + if raw == nil then return nil, 'websocket_required' end + if type(raw) == 'table' and raw._http_public_websocket then return raw._http_public_websocket end + local self = setmetatable({ + _raw = raw, + _closed = false, + _close_reason = nil, + _on_terminate = opts and opts.on_terminate, + _terminate_hooks = {}, + }, WebSocket) + if type(raw) == 'table' then raw._http_public_websocket = self end + return self +end + +local function wrap_client(raw, opts) + if raw == nil then return nil, 'websocket_required' end + if type(raw) == 'table' and raw._http_public_client_websocket then return raw._http_public_client_websocket end + local self = setmetatable({ + _raw = raw, + _closed = false, + _close_reason = nil, + _on_terminate = opts and opts.on_terminate, + _terminate_hooks = {}, + }, ClientWebSocket) + if type(raw) == 'table' then raw._http_public_client_websocket = self end + return self +end + +function M.from_context_op(ctx, headers, opts) + opts = opts or {} + return op.guard(function () + local public + local transport_opts = copy(opts) + transport_opts.on_terminate = function (_, reason) + if public then return public:_notify_terminated(reason) end + local hook = opts.on_terminate + if hook then return hook(nil, reason) end + return true + end + return transport_ws.from_context_op(raw_context(ctx), headers, transport_opts):wrap(function (raw_ws, err) + if not raw_ws then return nil, err end + public = assert(wrap_server(raw_ws, opts)) + if ctx and type(ctx._register_server_websocket) == 'function' then + local ok, rerr = ctx:_register_server_websocket(public) + if ok == nil or ok == false then + public:terminate(rerr or 'websocket_registration_failed') + return nil, rerr or 'websocket_registration_failed' + end + end + return public, nil + end) + end) +end + +function M.connect_op(driver, args, opts) + opts = opts or {} + return op.guard(function () + local public + local transport_opts = copy(opts) + transport_opts.on_terminate = function (_, reason) + if public then return public:_notify_terminated(reason) end + local hook = opts.on_terminate + if hook then return hook(nil, reason) end + return true + end + return transport_ws.connect_op(driver, args, transport_opts):wrap(function (raw_ws, err) + if not raw_ws then return nil, err end + public = assert(wrap_client(raw_ws, opts)) + return public, nil + end) + end) +end + +function WebSocket:_raw_websocket_for_test() + return self._raw and self._raw._raw_websocket_for_test and self._raw:_raw_websocket_for_test() +end + +function WebSocket:_notify_terminated(reason) + return notify_terminated(self, reason) +end + +function WebSocket:add_terminate_hook(fn) + if type(fn) ~= 'function' then return nil, 'invalid_args' end + if self._closed then fn(self, self._close_reason or 'closed'); return function () end end + local hooks = self._terminate_hooks or {} + self._terminate_hooks = hooks + hooks[#hooks + 1] = fn + local active = true + return function () + if not active then return end + active = false + for i, h in ipairs(hooks) do + if h == fn then table.remove(hooks, i); break end + end + end +end + + +function WebSocket:context() + local raw_ctx = self._raw and self._raw.context and self._raw:context() + return raw_ctx +end + +function WebSocket:is_closed() + return self._closed or (self._raw and self._raw:is_closed()) +end + +function WebSocket:why() + return self._close_reason or (self._raw and self._raw:why()) +end + +function WebSocket:accept_op(options) + return self._raw:accept_op(options) +end + +function WebSocket:receive_op() + return self._raw:receive_op() +end + +function WebSocket:send_op(data, opcode) + return self._raw:send_op(data, opcode) +end + +function WebSocket:send_ping_op(data) + return self._raw:send_ping_op(data) +end + +function WebSocket:send_pong_op(data) + return self._raw:send_pong_op(data) +end + +function WebSocket:close_op(code, reason) + return self._raw:close_op(code, reason):wrap(function (ok, err) + self:_notify_terminated(reason or err or 'closed') + return ok, err + end) +end + +function WebSocket:terminate(reason) + if self._closed then return true end + if self._raw and self._raw.terminate then self._raw:terminate(reason or 'closed') end + return self:_notify_terminated(reason or 'closed') +end + +function ClientWebSocket:_raw_websocket_for_test() + return self._raw and self._raw._raw_websocket_for_test and self._raw:_raw_websocket_for_test() +end + +function ClientWebSocket:_notify_terminated(reason) + return notify_terminated(self, reason) +end + +function ClientWebSocket:add_terminate_hook(fn) + if type(fn) ~= 'function' then return nil, 'invalid_args' end + if self._closed then fn(self, self._close_reason or 'closed'); return function () end end + local hooks = self._terminate_hooks or {} + self._terminate_hooks = hooks + hooks[#hooks + 1] = fn + local active = true + return function () + if not active then return end + active = false + for i, h in ipairs(hooks) do + if h == fn then table.remove(hooks, i); break end + end + end +end + + +function ClientWebSocket:is_closed() + return self._closed or (self._raw and self._raw:is_closed()) +end + +function ClientWebSocket:why() + return self._close_reason or (self._raw and self._raw:why()) +end + +function ClientWebSocket:receive_op() + return self._raw:receive_op() +end + +function ClientWebSocket:send_op(data, opcode) + return self._raw:send_op(data, opcode) +end + +function ClientWebSocket:send_ping_op(data) + return self._raw:send_ping_op(data) +end + +function ClientWebSocket:send_pong_op(data) + return self._raw:send_pong_op(data) +end + +function ClientWebSocket:close_op(code, reason) + return self._raw:close_op(code, reason):wrap(function (ok, err) + self:_notify_terminated(reason or err or 'closed') + return ok, err + end) +end + +function ClientWebSocket:terminate(reason) + if self._closed then return true end + if self._raw and self._raw.terminate then self._raw:terminate(reason or 'closed') end + return self:_notify_terminated(reason or 'closed') +end + +M.wrap_server = wrap_server +M.wrap_client = wrap_client +M.WebSocket = WebSocket +M.ClientWebSocket = ClientWebSocket +return M diff --git a/src/services/log.lua b/src/services/log.lua deleted file mode 100644 index 5f70cb86..00000000 --- a/src/services/log.lua +++ /dev/null @@ -1,37 +0,0 @@ -local rxilog = require 'rxilog' -local new_msg = require 'bus'.new_msg -local syscall = require 'fibers.utils.syscall' - -local log_service = { - name = "log", -} -log_service.__index = log_service - -for _, mode in ipairs(rxilog.modes) do - local level = mode.name - log_service[level] = function(...) - local msg = rxilog.tostring(...) - rxilog[level](msg) - - if log_service.conn then - local info = debug.getinfo(2, "Sl") - local lineinfo = info.short_src .. ":" .. info.currentline - local formatted_msg = rxilog.format_log_message(level:upper(), lineinfo, msg) - - log_service.conn:publish(new_msg({ "logs", level }, { - message = formatted_msg, - timestamp = syscall.realtime() - })) - end - end -end - -function log_service:start(ctx, conn) - self.ctx = ctx - self.conn = conn - log_service.trace("Starting Log Service") -end - --- Make singleton -package.loaded["services.log"] = log_service -return log_service diff --git a/src/services/mcu_bridge.lua b/src/services/mcu_bridge.lua deleted file mode 100644 index cfbc1787..00000000 --- a/src/services/mcu_bridge.lua +++ /dev/null @@ -1,128 +0,0 @@ -local log = require 'services.log' -local service = require 'service' -local cjson = require 'cjson.safe' -local new_msg = require 'bus'.new_msg -local cache = require 'cache' -local pow = math.pow or function(base, power) - return base ^ power -end - -local mcu_bridge = { - name = 'mcu_bridge' -} -mcu_bridge.__index = mcu_bridge - --- Cache to store previous MCU values (no timeout since we want to keep them indefinitely) -local mcu_value_cache = cache.new(math.huge) - -local UNDERFLOW_THRESHOLD = 1000000 -- threshiold for detection of underflow due to cjson limitations - -local function fix_underflows(tbl) - for k, value in pairs(tbl) do - if type(value) == "number" and value > UNDERFLOW_THRESHOLD then - tbl[k] = value - pow(2, 32) - end - end - return tbl -end - -local function trim(str) - return str:gsub("^%s*(.-)%s*$", "%1") -end - -local function read_uart_data(ctx) - local time_to_next_err_log = os.time() - local err_log_cooldown = 1 - - local uart_cap_sub = mcu_bridge.conn:subscribe({ 'hal', 'capability', 'uart', 'uart0' }) - uart_cap_sub:next_msg_with_context(ctx) - log.trace(string.format( - "%s - %s: Dectected UART capability", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - uart_cap_sub:unsubscribe() - local req = mcu_bridge.conn:request(new_msg( - { 'hal', 'capability', 'uart', 'uart0', 'control', 'open' }, - { { baudrate = 115200, read = true, write = true } } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - if ctx_err or resp.payload.err then - log.error(string.format( - "%s - %s: Failed to open UART port: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - ctx_err or resp.payload.err - )) - return - else - log.info(string.format( - "%s - %s: UART port opened successfully", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - end - req:unsubscribe() - - local uart_data_sub = mcu_bridge.conn:subscribe({ 'hal', 'capability', 'uart', 'uart0', 'info', 'out' }) - while not ctx:err() do - local msg, sub_err = uart_data_sub:next_msg_with_context(ctx) - if sub_err then - log.error(string.format( - "%s - %s: Error receiving UART data: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - sub_err - )) - break - end - if msg and msg.payload then - local json_string = trim(msg.payload) - local decoded, decode_err = cjson.decode(json_string) - -- If pico starts putting out wrong json or the line becomes noisy the logs could be spammed with - -- decoding errors, therefore I have put a cooldown on - if decode_err and os.time() >= time_to_next_err_log then - log.error(string.format( - "%s - %s: Error decoding UART JSON data: %s \"%s\"", - ctx:value("service_name"), - ctx:value("fiber_name"), - decode_err, - msg.payload - )) - time_to_next_err_log = os.time() + err_log_cooldown - err_log_cooldown = 2 * err_log_cooldown - err_log_cooldown = (err_log_cooldown > 60) and 60 or err_log_cooldown - elseif not decode_err then - local casted = fix_underflows(decoded) - for k, v in pairs(casted) do - -- Check if value has changed before publishing - local cached_value = mcu_value_cache:get(k) - - -- Only publish if value changed or doesn't exist in cache - if cached_value ~= v then - local key_table = { 'mcu' } - for segment in string.gmatch(k, "[^/]+") do - table.insert(key_table, segment) - end - mcu_value_cache:set(k, v) - mcu_bridge.conn:publish(new_msg( - key_table, - v - )) - end - end - end - end - end - uart_data_sub:unsubscribe() -end - -function mcu_bridge:start(ctx, conn) - self.ctx = ctx - self.conn = conn - -- Placeholder for future implementation - - service.spawn_fiber('UART Reader', conn, ctx, read_uart_data) -end - -return mcu_bridge diff --git a/src/services/metrics.lua b/src/services/metrics.lua index 84fe0f82..fef32663 100644 --- a/src/services/metrics.lua +++ b/src/services/metrics.lua @@ -1,369 +1,603 @@ -local service = require 'service' -local op = require "fibers.op" -local log = require 'services.log' -local sc = require 'fibers.utils.syscall' -local sleep = require 'fibers.sleep' -local json = require 'cjson.safe' -local senml = require 'services.metrics.senml' -local http = require 'services.metrics.http' -local conf = require 'services.metrics.config' -local unpack = table.unpack or unpack - ----@class metrics_service ----@field default_process ProcessPipeline -local metrics_service = { - name = 'metrics', - metrics = {}, - metric_values = {}, - pipelines = {}, +-- services/metrics.lua +-- +-- Transitional metrics pipeline: long-term, metrics should observe retained +-- domain /state and derive export series from those models. Services may +-- still publish legacy obs metrics during migration, but UI-critical state +-- should live under /state. +-- +-- Metrics service: +-- - subscribes to {'obs', 'v1', '+', 'metric', '+'} for all observable metrics +-- - applies per-pipeline processing (DiffTrigger, DeltaValue, etc.) +-- - maintains per-endpoint processing state (shared pipeline logic, isolated state) +-- - periodically publishes accumulated metrics via http, log, or bus protocol +-- - fetches Mainflux cloud credentials from the HAL filesystem capability +-- +-- Topics consumed: +-- {'obs', 'v1', '+', 'metric', '+'} - incoming metric values +-- {'cfg', 'metrics'} - metrics config (retained) +-- {'state', 'time', 'synced'} - NTP sync status (retained) +-- {'cap', 'fs', 'configs', ...} - HAL filesystem capability (via cap listener) +-- {'cap', 'http', 'main', 'rpc', ...} - HTTP capability service (exchange RPC) +-- +-- Topics produced: +-- {'svc', 'metrics', 'status'} - service lifecycle status (retained) +-- {'obs', 'v1', 'metrics', 'output', ...} - per-metric bus publications (bus protocol) + +local fibers = require 'fibers' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local runtime = require 'fibers.runtime' +local time = require 'fibers.utils.time' +local perform = fibers.perform + +local json = require 'cjson.safe' +local base = require 'devicecode.service_base' +local cap_sdk = require 'services.hal.sdk.cap' +local http_sdk = require('services.http').sdk + +local senml = require 'services.metrics.senml' +local http_m = require 'services.metrics.http' +local conf = require 'services.metrics.config' +local types = require 'services.metrics.types' + + +local unpack = unpack or rawget(table, 'unpack') + +local NAME = 'metrics' + +------------------------------------------------------------------------------- +-- Topic helpers +------------------------------------------------------------------------------- + +---@param service string +---@param name string +---@return table +local function t_obs_metric(service, name) return { 'obs', 'v1', service, 'metric', name } end + +---@param name string +---@return table +local function t_cfg(name) return { 'cfg', name } end + +---@return table +local function t_state_time_synced() return { 'state', 'time', 'synced' } end + +---@param tokens table +---@return table +local function t_obs_metrics_output(tokens) return { 'obs', 'v1', 'metrics', 'output', unpack(tokens) } end + +---@return number +local function now() return runtime.now() end + +---@return number +local function now_real() return time.realtime() end + +------------------------------------------------------------------------------- +-- Metric helpers +------------------------------------------------------------------------------- + +--- Validate a topic array (no gaps, no nils, at least one element). +---@param topic any +---@return boolean +local function validate_topic(topic) + if type(topic) ~= 'table' then return false end + local count = 0 + for k in pairs(topic) do + if type(k) ~= 'number' or k < 1 or k ~= math.floor(k) then return false end + count = count + 1 + end + if count == 0 then return false end + for i = 1, count do + if topic[i] == nil then return false end + end + return true +end + +--- Shift per-endpoint metric timestamps from monotonic to real-time milliseconds. +--- base_time = { real = wall_clock_at_mono_base, mono = mono_at_base } +---@param base_time BaseTime +---@param metrics table +---@return table +local function set_timestamps_realtime_millis(base_time, metrics) + for _, metric in pairs(metrics) do + metric.time = math.floor((base_time.real + (metric.time - base_time.mono)) * 1000) + end + return metrics +end + +------------------------------------------------------------------------------- +-- Service state +------------------------------------------------------------------------------- + +---@type ServiceState +local State = { + conn = nil, + svc = nil, + name = nil, + http_ref = nil, + http_send_ch = nil, + pipelines_map = {}, + metric_states = {}, + endpoint_to_pipe = {}, + metric_values = {}, + publish_period = nil, + cloud_url = nil, + mainflux_config = nil, + cloud_config = nil, + base_time = nil, + fs_cap = nil, } -metrics_service.__index = metrics_service - ---- this is the only publish protocol ---- (for testing purposes) ---- @param data table -function metrics_service:_log_publish(data) - local function print_recursive(t, indent) - for k, v in pairs(t) do - if type(v) == "table" then - print(string.rep(" ", indent) .. k .. ":") - print_recursive(v, indent + 2) - else - print(string.rep(" ", indent) .. k .. ": " .. tostring(v)) - end - end - end - print_recursive(data, 0) + +------------------------------------------------------------------------------- +-- Config warnings (pure: no service state) +------------------------------------------------------------------------------- + +--- Log config warnings and prune invalid entries from the raw config in-place. +---@param warns table +---@param config table +local function process_config_warnings(warns, config) + if #warns == 0 then return end + + local warn_msgs = {} + local dropped_metrics = {} + local dropped_templates = {} + + for _, warn in ipairs(warns) do + table.insert(warn_msgs, warn.msg) + if warn.endpoint then + if warn.type == 'metric' then + config.pipelines[warn.endpoint] = nil + dropped_metrics[warn.endpoint] = true + elseif warn.type == 'template' then + if config.templates then + config.templates[warn.endpoint] = nil + end + dropped_templates[warn.endpoint] = true + end + end + end + + local dm_list = {} + for ep in pairs(dropped_metrics) do dm_list[#dm_list + 1] = ep end + + local dt_list = {} + for ep in pairs(dropped_templates) do dt_list[#dt_list + 1] = ep end + + State.svc:obs_log('warn', { + what = 'config_warnings', + warnings = warn_msgs, + dropped_metrics = dm_list, + dropped_templates = dt_list, + }) end -function metrics_service:_http_publish(data) - local senml_list, err = senml.encode_r("", data) - if err then - log.error(string.format( - "%s - %s: %s", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - err - )) - return - end - if #senml_list == 0 then return end - local body = json.encode(senml_list) - local valid_config, config_err = conf.validate_http_config(self.cloud_config) - if not valid_config then - log.error(string.format( - "%s - %s: HTTP publish failed, reason: %s", - self.ctx:value('service_name'), - self.ctx:value('fiber_name'), - config_err - )) - return - end - local channel_id - for _, channel in ipairs(self.cloud_config.channels) do - if channel.metadata and channel.metadata.channel_type == "data" then - channel_id = channel.id - break - end - end - if channel_id == nil then - log.error(string.format( - "%s - %s: HTTP publish failed, reason: no channel id found", - self.ctx:value('service_name'), - self.ctx:value('fiber_name') - )) - return - end - local uri = string.format("%s/http/channels/%s/messages", - self.cloud_config.url, - channel_id - ) - local auth = "Thing " .. self.cloud_config.thing_key - local http_payload = { - uri = uri, - auth = auth, - body = body - } - self.http_send_q:put_op(http_payload):perform_alt(function() - log.error(string.format( - "%s - %s: HTTP publish failed, reason: HTTP send queue is full", - self.ctx:value('service_name'), - self.ctx:value('fiber_name') - )) - end) +------------------------------------------------------------------------------- +-- Cloud config +------------------------------------------------------------------------------- + +local function rebuild_cloud_config() + local mf = State.mainflux_config + if not mf or not State.cloud_url or not mf.thing_key or not mf.channels then + State.cloud_config = nil + return + end + local cfg, cfg_err = types.new.CloudConfig(State.cloud_url, mf.thing_key, mf.channels) + if not cfg then + State.svc:obs_log('warn', { what = 'cloud_config_build_failed', err = tostring(cfg_err) }) + State.cloud_config = nil + return + end + State.cloud_config = cfg + State.svc:obs_log('debug', 'cloud config ready') end -local function reset_pipelines(pipelines, metrics) - for endpoint, _ in pairs(metrics) do - if pipelines[endpoint] then - pipelines[endpoint]:reset() - end - end +local function fetch_mainflux_config() + local read_opts, opts_err = cap_sdk.args.new.FilesystemReadOpts('mainflux.cfg') + if not read_opts then + State.svc:obs_log('warn', { what = 'mainflux_read_opts_failed', err = tostring(opts_err) }) + return + end + + local reply, err = State.fs_cap:call_control('read', read_opts) + if not reply then + State.svc:obs_log('warn', { what = 'mainflux_read_failed', err = tostring(err) }) + return + end + if reply.ok ~= true then + State.svc:obs_log('warn', { what = 'mainflux_read_error', err = tostring(reply.reason) }) + return + end + + local raw, decode_err = json.decode(reply.reason) + if not raw then + State.svc:obs_log('warn', { what = 'mainflux_decode_failed', err = tostring(decode_err) }) + return + end + + State.mainflux_config = conf.standardise_config(raw) + State.svc:obs_log('debug', 'mainflux config loaded') + rebuild_cloud_config() end -local function set_timestamps_realtime_millis(base_time, metrics) - for _, metric in pairs(metrics) do - metric.time = math.floor((base_time.real + (metric.time - base_time.mono))*1000) - end - return metrics +------------------------------------------------------------------------------- +-- Protocol publish handlers +------------------------------------------------------------------------------- + +---@param data table +local function bus_publish(data) + for endpoint_str, metric in pairs(data) do + local tokens = {} + for part in endpoint_str:gmatch('[^.]+') do + tokens[#tokens + 1] = part + end + State.conn:publish(t_obs_metrics_output(tokens), { value = metric.value, time = metric.time }) + end end ---- Validates that a topic array is properly formed with no gaps or nil values ---- @param topic table The topic array to validate ---- @return boolean true if the topic is valid (contiguous array with no nils), false otherwise -local function validate_topic(topic) - if #topic == 0 then return false end - - -- Count all keys using pairs - local pairs_count = 0 - for k, v in pairs(topic) do - pairs_count = pairs_count + 1 - if type(k) ~= "number" or k < 1 or k ~= math.floor(k) then - return false -- non-integer or non-positive key - end - if v == nil then - return false -- explicit nil value - end - end - - -- If pairs_count equals #topic, array is contiguous with no gaps - return pairs_count == #topic +---@param data table +local function log_publish(data) + for endpoint_str, metric in pairs(data) do + State.svc:obs_log('trace', { what = 'metric_value', + endpoint = endpoint_str, value = metric.value, time = metric.time }) + end end ----iterates over a table of data to be published ----@param data table -function metrics_service:_publish_all(data) - -- all first keys are the names of the publish protocols - for protocol, values in pairs(data) do - -- reset all pipelines after publish - reset_pipelines(self.pipelines, values) - values = set_timestamps_realtime_millis(self.base_time, values) - local protocol_fn = self["_" .. protocol .. "_publish"] - if protocol_fn == nil then - log.error(string.format('Failed to publish for %s, no function associated with protocol', protocol)) - else - local ok, err = pcall(protocol_fn, self, values) - if not ok then - log.error(string.format("failed to publish with protocol %s: %s", protocol, err)) - end - end - end +---@param data table +local function http_publish(data) + local senml_list, encode_err = senml.encode_r('', data) + if encode_err then + State.svc:obs_log('error', { what = 'senml_encode_failed', err = tostring(encode_err) }) + return + end + if #senml_list == 0 then return end + + local body = json.encode(senml_list) + + local valid, config_err = conf.validate_http_config(State.cloud_config) + if not valid then + State.svc:obs_log('error', { what = 'http_publish_skipped', err = tostring(config_err) }) + return + end + + local channel_id + for _, ch in ipairs(State.cloud_config.channels) do + if ch.metadata and ch.metadata.channel_type == 'data' then + channel_id = ch.id + break + end + end + if channel_id == nil then + State.svc:obs_log('error', { what = 'http_publish_failed', err = 'no data channel id found' }) + return + end + + local uri = string.format('%s/http/channels/%s/messages', + State.cloud_config.url, channel_id) + local auth = 'Thing ' .. State.cloud_config.thing_key + + -- Non-blocking enqueue: drop and log if the channel is at capacity. + local full = perform(State.http_send_ch:put_op({ uri = uri, auth = auth, body = body }) + :or_else(function() return true end)) + + if full then + State.svc:obs_log('error', { what = 'http_queue_full', err = 'dropping publish payload' }) + end end -function metrics_service:_handle_metric(metric, msg) - local protocol = metric.protocol - local field = metric.field - local rename = metric.rename - local base_pipeline = metric.base_pipeline - - local value = msg.payload - if value == nil then return end - if field then - value = value[field] - end - if value == nil then return end - - local topic = rename or msg.topic - if not validate_topic(topic) then - log.warn(string.format( - "%s - %s: Invalid topic array (nil value or gap detected), skipping metric", - self.ctx:value('service_name'), - self.ctx:value('fiber_name') - )) - return - end - - local str_endpoint = table.concat(topic, '.') - - if self.pipelines[str_endpoint] == nil then - self.pipelines[str_endpoint] = base_pipeline:clone() - end - - local pipeline = self.pipelines[str_endpoint] - - local ret, short, err = pipeline:run(value) - if err then - log.error(string.format( - "%s - %s: Metric processing error for endpoint %s: %s", - self.ctx:value('service_name'), - self.ctx:value('fiber_name'), - str_endpoint, - err - )) - return - end - if not short then - self.metric_values[protocol] = self.metric_values[protocol] or {} - self.metric_values[protocol][str_endpoint] = { - value = ret, - time = sc.monotime() - } - end + +local function count_pipelines() + local n = 0 + for _ in pairs(State.pipelines_map or {}) do n = n + 1 end + return n end ----Process validation warnings and remove invalid configs ----@param warns table ----@param config table -function metrics_service:_process_config_warnings(warns, config) - if #warns == 0 then return end - - local warn_msgs = {} - local dropped_metrics = {} - local dropped_templates = {} - - for _, warn in ipairs(warns) do - table.insert(warn_msgs, warn.msg) - if warn.endpoint then - if warn.type == "metric" then - config.collections[warn.endpoint] = nil - dropped_metrics[warn.endpoint] = true - elseif warn.type == "template" then - config.templates[warn.endpoint] = nil - dropped_templates[warn.endpoint] = true - end - end - end - - local summary_parts = {} - local dropped_metric_list = {} - for endpoint, _ in pairs(dropped_metrics) do - table.insert(dropped_metric_list, endpoint) - end - if #dropped_metric_list > 0 then - table.insert(summary_parts, string.format("Dropped %d metric(s): %s", - #dropped_metric_list, table.concat(dropped_metric_list, ", "))) - end - - local dropped_template_list = {} - for endpoint, _ in pairs(dropped_templates) do - table.insert(dropped_template_list, endpoint) - end - if #dropped_template_list > 0 then - table.insert(summary_parts, string.format("Dropped %d template(s): %s", - #dropped_template_list, table.concat(dropped_template_list, ", "))) - end - - log.warn(string.format( - "%s - %s: Metrics config warnings (invalid configs will be dropped):\n\t%s\n\nSummary: %s", - self.ctx:value('service_name'), - self.ctx:value('fiber_name'), - table.concat(warn_msgs, "\n\t"), - table.concat(summary_parts, "; ") - )) +local function log_metrics_summary(reason, extra) + local parts = {} + parts[#parts + 1] = 'publishing=' .. ((State.publish_period and State.base_time and State.base_time.synced) and 'enabled' or 'waiting') + if State.publish_period then parts[#parts + 1] = 'interval=' .. tostring(State.publish_period) .. 's' end + parts[#parts + 1] = 'pipelines=' .. tostring(count_pipelines()) + local summary = 'metrics summary ' .. table.concat(parts, ' ') + local tnow = now() + if State._operator_summary_key == summary and (tnow - (State._operator_summary_at or 0)) < 600 then return end + State._operator_summary_key = summary + State._operator_summary_at = tnow + State.svc:obs_log('info', { what = 'metrics_summary', summary = summary, reason = reason }) end ----use config to build cache and processing pipelines ----@param config table -function metrics_service:_handle_config(config) - local valid, warns, err = conf.validate_config(config) - if not valid then - log.error(string.format( - "%s - %s: Metrics config invalid: %s", - self.ctx:value('service_name'), - self.ctx:value('fiber_name'), - err - )) - return - end - - self:_process_config_warnings(warns, config) - - local metrics, publish_period, merged_cloud_config = conf.apply_config(self.conn, config, self.cloud_config) - if #metrics == 0 then - log.warn(string.format( - "%s - %s: No valid metrics created, metrics service will be idle", - self.ctx:value('service_name'), - self.ctx:value('fiber_name') - )) - end - - -- clean up any old metric pipelines - for _, metric in ipairs(self.metrics) do - metric.sub:unsubscribe() - end - self.pipelines = {} - self.metrics = metrics - self.publish_period = publish_period - self.cloud_config = merged_cloud_config +local publish_fns = { bus = bus_publish, log = log_publish, http = http_publish } + +---@param values table> +local function publish_all(values) + for protocol, pv in pairs(values) do + -- Reset per-endpoint pipeline states for published endpoints. + for endpoint_str, _ in pairs(pv) do + local metric_name = State.endpoint_to_pipe[endpoint_str] + if metric_name then + local pipe_cfg = State.pipelines_map[metric_name] + if pipe_cfg and State.metric_states[endpoint_str] then + pipe_cfg.pipeline:reset(State.metric_states[endpoint_str]) + end + end + end + + pv = set_timestamps_realtime_millis(State.base_time, pv) + + local fn = publish_fns[protocol] + if fn == nil then + State.svc:obs_log('error', { what = 'unknown_protocol', protocol = tostring(protocol) }) + else + fn(pv) + end + end +end + +------------------------------------------------------------------------------- +-- Metric handling +------------------------------------------------------------------------------- + +---@param msg Message? +local function handle_metric(msg) + if not msg then return end + + -- Topic layout: {'obs', 'v1', , 'metric', } + local metric_name = msg.topic and msg.topic[5] + if not metric_name then return end + + local pipe_cfg = State.pipelines_map[metric_name] + if not pipe_cfg then return end -- no matching pipeline, drop silently + + local payload = msg.payload + if type(payload) ~= 'table' then return end + + local value = payload.value + if value == nil then return end + + -- Optional namespace overrides the topic used as the SenML name and state key. + local topic = payload.namespace or msg.topic + if not validate_topic(topic) then + State.svc:obs_log('warn', { what = 'metric_invalid_topic', metric = metric_name }) + return + end + + local endpoint_str = table.concat(topic, '.') + + -- Get-or-create per-endpoint processing state. + if not State.metric_states[endpoint_str] then + State.metric_states[endpoint_str] = pipe_cfg.pipeline:new_state() + State.endpoint_to_pipe[endpoint_str] = metric_name + end + + local ret, short, err = pipe_cfg.pipeline:run(value, State.metric_states[endpoint_str]) + if err then + State.svc:obs_log('error', { what = 'pipeline_error', endpoint = endpoint_str, err = tostring(err) }) + return + end + + if not short then + State.metric_values[pipe_cfg.protocol] = State.metric_values[pipe_cfg.protocol] or {} + State.metric_values[pipe_cfg.protocol][endpoint_str] = types.new.MetricSample(ret, now()) + end +end + +------------------------------------------------------------------------------- +-- Config handling +------------------------------------------------------------------------------- + +---@param payload table? +---@return number next_publish_time +local function handle_config(payload) + if not payload then return math.huge end + + local valid, warns, err = conf.validate_config(payload) + if not valid then + State.svc:obs_log('error', { what = 'config_invalid', err = tostring(err) }) + State.svc:obs_event('config_rejected', { err = tostring(err) }) + return math.huge + end + + process_config_warnings(warns, payload) + + local log_fn = function(level, msg) State.svc:obs_log(level, msg) end + local new_pipelines_map, new_publish_period = conf.apply_config(payload, log_fn) + + if next(new_pipelines_map) == nil then + State.svc:obs_log('warn', { what = 'config_no_pipelines' }) + end + + -- Cache cloud_url from the metrics config and rebuild cloud_config. + State.cloud_url = payload.data and payload.data.cloud_url + rebuild_cloud_config() + + -- Replace all pipeline state (logic may have changed). + State.pipelines_map = new_pipelines_map + State.metric_states = {} + State.endpoint_to_pipe = {} + State.publish_period = new_publish_period + + if State.base_time.synced and State.publish_period then + return now() + State.publish_period + end + return math.huge end ----setup initial cache and config then loop over config, metrics and timed cache operations -function metrics_service:_main(ctx) - self.ctx = ctx - self.http_send_q = http.start_http_publisher(self.ctx, self.conn) - self.base_time = { - synced = false, - real = sc.realtime(), - mono = sc.monotime() - } - - local config_sub = self.conn:subscribe({ 'config', 'metrics' }) - local cloud_config_sub = self.conn:subscribe({ 'config', 'mainflux' }) - local time_sync_sub = self.conn:subscribe({ 'time', 'ntp_synced' }) - - local next_publish_time = math.huge - - -- local device_info_sub = self.conn:subscribe({'system', 'device', 'idenitity'}) idk what this will be but - -- it is a reminder to get system info that canopy will need for reporting - - while not self.ctx:err() do - local metric_ops = {} - for _, metric in ipairs(self.metrics) do - table.insert(metric_ops, metric.sub:next_msg_op():wrap(function (msg) - self:_handle_metric(metric, msg) - end)) - end - op.choice( - self.ctx:done_op(), - config_sub:next_msg_op():wrap(function(config_msg) - self:_handle_config(config_msg.payload) - next_publish_time = self.base_time.synced and (sc.monotime() + self.publish_period) or math.huge - end), - cloud_config_sub:next_msg_op():wrap(function(config_msg) - local config = conf.standardise_config(config_msg.payload) - self.cloud_config = conf.merge_config(self.cloud_config, config) - end), - time_sync_sub:next_msg_op():wrap(function(msg) - if msg.payload == true then - if not self.base_time.synced then - -- First time sync - calculate real time at base and schedule first publish - self.base_time.synced = true - local real = sc.realtime() - local mono = sc.monotime() - local real_at_base = real - (mono - self.base_time.mono) - self.base_time.real = real_at_base - if self.publish_period then - next_publish_time = mono + self.publish_period - end - end - else - self.base_time.synced = false - next_publish_time = math.huge - end - end), - sleep.sleep_until_op(next_publish_time):wrap(function() - local values = self.metric_values - self.metric_values = {} - next_publish_time = self.base_time.synced and (sc.monotime() + self.publish_period) or math.huge - self:_publish_all(values) - end), - unpack(metric_ops) - ):perform() - end - config_sub:unsubscribe() - cloud_config_sub:unsubscribe() - log.info("Metrics Service Ending") +---@param synced boolean +---@return boolean first_sync +local function handle_time_sync(synced) + if synced == true then + if not State.base_time.synced then + State.base_time.synced = true + local real = now_real() + local mono = now() + -- Compute the wall-clock time that corresponds to base_time.mono. + State.base_time.real = real - (mono - State.base_time.mono) + return true -- first sync + end + else + State.base_time.synced = false + end + return false end ----Creates the metrics fiber ----@param ctx Context +------------------------------------------------------------------------------- +-- Main loop +------------------------------------------------------------------------------- + +local function main() + -- Subscribe to all observable metrics. + local obs_sub = State.conn:subscribe( + t_obs_metric('+', '+'), + { queue_len = 100, full = 'drop_oldest' }) + + -- Subscribe to the metrics config (retained; first message is current config). + local cfg_sub = State.conn:subscribe( + t_cfg(NAME), + { queue_len = 10, full = 'drop_oldest' }) + + -- Subscribe to NTP sync status. + local time_sub = State.conn:subscribe( + t_state_time_synced(), + { queue_len = 5, full = 'drop_oldest' }) + + local next_publish_time = math.huge + + while true do + local which, a, b = perform(op.named_choice({ + config = cfg_sub:recv_op(), + metric = obs_sub:recv_op(), + timesync = time_sub:recv_op(), + -- timesync = State.base_time.synced and op.never() or op.always({ payload = true }), + tick = sleep.sleep_until_op(next_publish_time), + })) + + + if which == 'config' then + local msg, err = a, b + if not msg then + State.svc:obs_log('warn', { what = 'config_sub_closed', err = tostring(err) }) + break + end + State.svc:obs_log('debug', 'config received, applying') + next_publish_time = handle_config(msg.payload) + -- Re-read mainflux.cfg in case cloud_url or credentials changed. + fetch_mainflux_config() + local next_s = next_publish_time == math.huge and nil or (next_publish_time - now()) + State.svc:obs_event('config_applied', { next_publish_s = next_s }) + State.svc:obs_log('debug', next_s + and string.format('config applied, next publish in %.1fs', next_s) + or 'config applied, publishing suspended (waiting for NTP sync)') + elseif which == 'metric' then + local msg = a + if msg then + handle_metric(msg) + end + elseif which == 'timesync' then + local msg = a + if msg then + local first_sync = handle_time_sync(msg.payload) + if first_sync and State.publish_period then + next_publish_time = now() + State.publish_period + State.svc:obs_event('ntp_synced', { first = true, next_publish_s = State.publish_period }) + State.svc:obs_log('info', { what = 'metrics_ready', summary = string.format('metrics ready; next publish in %ss', tostring(State.publish_period)), next_publish_s = State.publish_period }) + elseif first_sync then + State.svc:obs_event('ntp_synced', { first = true }) + State.svc:obs_log('debug', 'NTP synced, waiting for config before scheduling publish') + elseif not State.base_time.synced then + next_publish_time = math.huge + State.svc:obs_event('ntp_lost', {}) + State.svc:obs_log('debug', { what = 'ntp_lost' }) + end + end + elseif which == 'tick' then + local values = State.metric_values + State.metric_values = {} + + if State.base_time.synced and State.publish_period then + next_publish_time = now() + State.publish_period + else + next_publish_time = math.huge + end + + local total = 0 + for _, pv in pairs(values) do + for _ in pairs(pv) do total = total + 1 end + end + State.svc:obs_event('publish', { count = total }) + if total > 0 then + State.svc:obs_log('debug', string.format('publishing %d metric(s)', total)) + end + publish_all(values) + end + end + + obs_sub:unsubscribe() + cfg_sub:unsubscribe() + time_sub:unsubscribe() + State.svc:obs_log('debug', 'service stopping') +end + +------------------------------------------------------------------------------- +-- Module entry point +------------------------------------------------------------------------------- + +local M = {} + ---@param conn Connection -function metrics_service:start(ctx, conn) - log.info("Metrics Service Starting") - self.conn = conn - service.spawn_fiber('Main Fiber', conn, ctx, function(metrics_ctx) - metrics_service:_main(metrics_ctx) - end) +---@param opts table? +function M.start(conn, opts) + opts = opts or {} + local name = opts.name or NAME + local heartbeat_s = (type(opts.heartbeat_s) == 'number') and opts.heartbeat_s or 30.0 + + local svc = base.new(conn, { name = name, env = opts.env }) + + svc:obs_state('boot', { at = svc:wall(), ts = svc:now(), state = 'entered' }) + svc:obs_log('debug', 'service start() entered') + svc:announce({}) + svc:starting() + svc:spawn_heartbeat(heartbeat_s, 'tick') + + State.conn = conn + State.svc = svc + State.name = name + State.http_ref = http_sdk.new_ref(conn, opts.http_service_id or 'main') + State.http_send_ch = http_m.start_http_publisher(State.http_ref, function(level, payload) + svc:obs_log(level, payload) + end) + State.pipelines_map = {} + State.metric_states = {} + State.endpoint_to_pipe = {} + State.metric_values = {} + State.publish_period = nil + State.cloud_url = nil + State.mainflux_config = nil + State.cloud_config = nil + State.base_time = types.new.BaseTime(now_real(), now()) + State.fs_cap = nil + + fibers.current_scope():finally(function() + local _, primary = fibers.current_scope():status() + svc:lifecycle('stopped', { ready = false, reason = tostring(primary or 'scope_exit') }) + svc:obs_log('debug', 'service stopped') + end) + + svc:obs_log('debug', 'waiting for filesystem capability') + local fs_listener = cap_sdk.new_cap_listener(conn, 'fs', 'credentials') + local fs_cap, cap_err = fs_listener:wait_for_cap() + fs_listener:close() + if not fs_cap then + svc:failed('filesystem capability unavailable') + svc:obs_log('error', { what = 'start_failed', err = tostring(cap_err) }) + return + end + State.fs_cap = fs_cap + + svc:obs_event('fs_ready', {}) + svc:obs_log('debug', 'fetching mainflux config') + fetch_mainflux_config() + + svc:running() + svc:obs_log('debug', 'service is live') + + main() end -return metrics_service +return M diff --git a/src/services/metrics/config.lua b/src/services/metrics/config.lua index b686c9cb..761ba024 100644 --- a/src/services/metrics/config.lua +++ b/src/services/metrics/config.lua @@ -1,437 +1,469 @@ -local log = require "services.log" -local processing = require "services.metrics.processing" - -local VALID_PROTOCOLS = { http = true, log = true } -local VALID_PROCESS_TYPES = { DiffTrigger = true, TimeTrigger = true, DeltaValue = true } +-- services/metrics/config.lua +-- +-- Configuration validation and application for the metrics service. +-- +-- validate_config(config) +-- Returns (ok, warnings, error). Validates structure, protocols, process +-- blocks, templates and pipelines. Invalid templates and any pipelines that +-- reference them are collected as warnings rather than hard failures so that +-- the service can continue with the valid subset. +-- +-- apply_config(config) +-- Returns (pipelines_map, publish_period). +-- pipelines_map[metric_name] = { pipeline, protocol } +-- The pipeline object contains only logic; per-endpoint state is created +-- externally with pipeline:new_state(). + +local processing = require 'services.metrics.processing' +local _types = require 'services.metrics.types' -- luacheck: ignore (imported for annotations) + +local TARGET_SCHEMA = "devicecode.config/metrics/1" + +local VALID_PROTOCOLS = { http = true, log = true, bus = true } +local VALID_PROCESS_TYPES = { DiffTrigger = true, TimeTrigger = true, DeltaValue = true } local VALID_TEMPLATE_FIELDS = { - field = true, protocol = true, - process = true, + process = true, } -local VALID_METRIC_FIELDS = { - field = true, +local VALID_METRIC_FIELDS = { protocol = true, - process = true, - rename = true, + process = true, template = true, } +------------------------------------------------------------------------------- +-- Helpers +------------------------------------------------------------------------------- + +---@param t any +---@return boolean local function is_array(t) - if type(t) ~= "table" then - return false - end - local i = 1 - for k, _ in pairs(t) do - if k ~= i then + if type(t) ~= 'table' then return false end + local count = 0 + for k in pairs(t) do + if type(k) ~= 'number' or math.floor(k) ~= k or k < 1 then return false end - i = i + 1 + count = count + 1 + end + for i = 1, count do + if t[i] == nil then return false end end - return true end -local function merge_config(base_config, override_vals) - if not base_config then return override_vals end - if not override_vals then return base_config end - +---@param base table? +---@param override table? +---@return table +local function merge_config(base, override) + if not base then return override or {} end + if not override then return base end local result = {} - - -- Copy base_config values - for k, v in pairs(base_config) do - result[k] = v - end - - -- Merge in override_vals - for k, v in pairs(override_vals) do + for k, v in pairs(base) do result[k] = v end + for k, v in pairs(override) do if type(v) == 'table' and type(result[k]) == 'table' then result[k] = merge_config(result[k], v) else result[k] = v end end - return result end +--- Normalise a raw mainflux config table to a consistent field set. +--- Accepts both legacy (`mainflux_*`) and current (`thing_*`) naming. +---@param config table +---@return table local function standardise_config(config) - local standard_config = {} - - standard_config.thing_id = config.mainflux_id or config.thing_id - standard_config.thing_key = config.mainflux_key or config.thing_key - standard_config.channels = config.mainflux_channels or config.channels - for _, channel in ipairs(standard_config.channels) do + local out = {} + out.thing_key = config.mainflux_key or config.thing_key + out.channels = config.mainflux_channels or config.channels + for _, channel in ipairs(out.channels or {}) do channel.metadata = channel.metadata or {} - if type(channel.metadata) == "userdata" then channel.metadata = {} end - if string.find(channel.name, "data") then - channel.metadata.channel_type = "data" - elseif string.find(channel.name, "control") then - channel.metadata.channel_type = "events" - end - end - - standard_config.content = config.content - return standard_config -end - ---- Uses process configs to build a processing pipeline made of ---- process blocks ---- @param endpoint string ---- @param process_config table ---- @return ProcessPipeline? ---- @return string? Error -local function build_metric_pipeline(endpoint, process_config) - -- processing blocks take a value and output another value - -- build up a processing pipeline between the cache and publish steps - local process, process_err = processing.new_process_pipeline() - if process_err or not process then - return nil, process_err - end - if process_config == nil then - return nil, "process config is nil" - end - for _, process_block_config in ipairs(process_config) do - local process_type = process_block_config.type - if process_type == nil then - return nil, string.format('Metric config [%s] has process with no type', endpoint) - end - local proc_class = processing[process_type] - - if proc_class == nil then - return nil, string.format('Metric config [%s] has invalid process [%s]', endpoint, process_type) - end - - local proc, proc_err = proc_class.new(process_block_config) - - if proc_err or not proc then - return nil, string.format('Metric config [%s] failed to create process block [%s]', endpoint, process_type) - end - local add_err = process:add(proc) - if add_err then - return nil, add_err + if type(channel.metadata) == 'userdata' then channel.metadata = {} end + if type(channel.name) == 'string' then + if string.find(channel.name, 'data') then + channel.metadata.channel_type = 'data' + elseif string.find(channel.name, 'control') then + channel.metadata.channel_type = 'events' + end end end - - return process, nil + out.content = config.content + return out end +--- Basic sanity-check for the cloud (Mainflux) config used for HTTP publish. +---@param config CloudConfig? +---@return boolean ok +---@return string? error local function validate_http_config(config) if not config then - return false, "No cloud config set" - elseif not config.url then - return false, "No cloud url set" - elseif not config.thing_key or not config.channels then - return false, "No cloud config set" + return false, 'No cloud config set' end - - if type(config.url) ~= "string" then - return false, "Cloud url is not a string" + if not config.url then + return false, 'No cloud url set' + end + if type(config.url) ~= 'string' then + return false, 'Cloud url is not a string' + end + if not config.thing_key or not config.channels then + return false, 'Cloud thing_key / channels missing' end return true, nil end +------------------------------------------------------------------------------- +-- Internal validation helpers +------------------------------------------------------------------------------- + +---@param process_block any +---@param endpoint string +---@param index number +---@return string? error local function validate_process_block(process_block, endpoint, index) - if type(process_block) ~= "table" then - return string.format("Metric config [%s] process block %d is not a table", + if type(process_block) ~= 'table' then + return string.format('Metric config [%s] process block %d is not a table', tostring(endpoint), index) - elseif process_block.type == nil then - return string.format("Metric config [%s] process block %d has no type field", + end + if process_block.type == nil then + return string.format('Metric config [%s] process block %d has no type field', tostring(endpoint), index) - elseif not VALID_PROCESS_TYPES[process_block.type] then - local valid_types = table.concat({ "DiffTrigger", "TimeTrigger", "DeltaValue" }, ", ") + end + if not VALID_PROCESS_TYPES[process_block.type] then return string.format( "Metric config [%s] process block %d has invalid type '%s' (valid: %s)", - tostring(endpoint), index, tostring(process_block.type), valid_types) + tostring(endpoint), index, tostring(process_block.type), + table.concat({ 'DiffTrigger', 'TimeTrigger', 'DeltaValue' }, ', ')) end return nil end +---@param name any +---@param template_config any +---@return table warnings local function validate_template(name, template_config) local warnings = {} - for field, _ in pairs(template_config) do - if not VALID_TEMPLATE_FIELDS[field] then - table.insert(warnings, { - msg = string.format("Template config [%s] has invalid field '%s'", - tostring(name), tostring(field)), - endpoint = name, - type = "template" - }) - end - end - if type(name) ~= "string" then + + if type(name) ~= 'string' then table.insert(warnings, { - msg = "Template name is not a string", + msg = 'Template name is not a string', endpoint = name, - type = "template" + type = 'template', }) end - if type(template_config) ~= "table" then + if type(template_config) ~= 'table' then table.insert(warnings, { - msg = string.format("Template config [%s] is not a table", tostring(name)), + msg = string.format('Template config [%s] is not a table', tostring(name)), endpoint = name, - type = "template" + type = 'template', }) + return warnings + end + + for field in pairs(template_config) do + if not VALID_TEMPLATE_FIELDS[field] then + table.insert(warnings, { + msg = string.format("Template config [%s] has invalid field '%s'", + tostring(name), tostring(field)), + endpoint = name, + type = 'template', + }) + end end - -- Validate protocol - if template_config.protocol and (not VALID_PROTOCOLS[template_config.protocol]) then + if template_config.protocol and not VALID_PROTOCOLS[template_config.protocol] then table.insert(warnings, { - msg = string.format("Template config [%s] has invalid protocol '%s' (valid: http, log)", + msg = string.format( + "Template config [%s] has invalid protocol '%s' (valid: http, log, bus)", tostring(name), tostring(template_config.protocol)), endpoint = name, - type = "template" + type = 'template', }) end - -- Validate process pipeline if template_config.process ~= nil then if not is_array(template_config.process) then table.insert(warnings, { - msg = string.format("Template config [%s] process must be an array", tostring(name)), + msg = string.format('Template config [%s] process must be an array', tostring(name)), endpoint = name, - type = "template" + type = 'template', }) else - -- Validate each process block - for i, process_block in ipairs(template_config.process) do - local proc_err = validate_process_block(process_block, name, i) - if proc_err then - table.insert(warnings, { - msg = proc_err, - endpoint = name, - type = "template" - }) + for i, blk in ipairs(template_config.process) do + local err = validate_process_block(blk, name, i) + if err then + table.insert(warnings, { msg = err, endpoint = name, type = 'template' }) end end end end - -- Validate field - local field_type = type(template_config.field) - if template_config.field and (field_type ~= "string" and field_type ~= "number") then - table.insert(warnings, { - msg = string.format("Template config [%s] field must be string or number", - tostring(name)), - endpoint = name, - type = "template" - }) - end - return warnings end -local function validate_metric(endpoint, metric_config, renames) +---@param endpoint any +---@param metric_config any +---@return table warnings +local function validate_metric(endpoint, metric_config) local warnings = {} - for field, _ in pairs(metric_config) do - if not VALID_METRIC_FIELDS[field] then - table.insert(warnings, { - msg = string.format("Metric config [%s] has invalid field '%s'", - tostring(endpoint), tostring(field)), - endpoint = endpoint, - type = "metric" - }) - end - end - if type(endpoint) ~= "string" then + + if type(endpoint) ~= 'string' then table.insert(warnings, { - msg = "Metric endpoint is not a string", + msg = 'Metric endpoint is not a string', endpoint = endpoint, - type = "metric" + type = 'metric', }) end - if type(metric_config) ~= "table" then + if type(metric_config) ~= 'table' then table.insert(warnings, { - msg = string.format("Metric config [%s] is not a table", tostring(endpoint)), + msg = string.format('Metric config [%s] is not a table', tostring(endpoint)), endpoint = endpoint, - type = "metric" + type = 'metric', }) + return warnings + end + + for field in pairs(metric_config) do + if not VALID_METRIC_FIELDS[field] then + table.insert(warnings, { + msg = string.format("Metric config [%s] has invalid field '%s'", + tostring(endpoint), tostring(field)), + endpoint = endpoint, + type = 'metric', + }) + end end - -- Validate protocol if metric_config.protocol == nil then table.insert(warnings, { - msg = string.format("Metric config [%s] has no defined protocol", tostring(endpoint)), + msg = string.format('Metric config [%s] has no defined protocol', tostring(endpoint)), endpoint = endpoint, - type = "metric" + type = 'metric', }) elseif not VALID_PROTOCOLS[metric_config.protocol] then table.insert(warnings, { - msg = string.format("Metric config [%s] has invalid protocol '%s' (valid: http, log)", + msg = string.format( + "Metric config [%s] has invalid protocol '%s' (valid: http, log, bus)", tostring(endpoint), tostring(metric_config.protocol)), endpoint = endpoint, - type = "metric" + type = 'metric', }) end - -- Validate process pipeline if metric_config.process ~= nil then if not is_array(metric_config.process) then table.insert(warnings, { - msg = string.format("Metric config [%s] process must be an array", tostring(endpoint)), + msg = string.format('Metric config [%s] process must be an array', tostring(endpoint)), endpoint = endpoint, - type = "metric" + type = 'metric', }) else - -- Validate each process block - for i, process_block in ipairs(metric_config.process) do - local proc_err = validate_process_block(process_block, endpoint, i) - if proc_err then - table.insert(warnings, { - msg = proc_err, - endpoint = endpoint, - type = "metric" - }) + for i, blk in ipairs(metric_config.process) do + local err = validate_process_block(blk, endpoint, i) + if err then + table.insert(warnings, { msg = err, endpoint = endpoint, type = 'metric' }) end end end end - -- Validate rename - if metric_config.rename then - if (not is_array(metric_config.rename)) then - table.insert(warnings, { - msg = string.format("Metric config [%s] rename is not of expected type: table", tostring(endpoint)), - endpoint = endpoint, - type = "metric" - }) - else - local rename = table.concat(metric_config.rename, ",") - if renames[rename] then - table.insert(warnings, { - msg = string.format("Metric config [%s] has duplicate rename definition", tostring(endpoint)), - endpoint = endpoint, - type = "metric" - }) - else - renames[rename] = true - end - end + return warnings +end + +------------------------------------------------------------------------------- +-- Pipeline builder +------------------------------------------------------------------------------- + +--- Build a ProcessPipeline from a process_config array. +---@param endpoint string +---@param process_config table +---@return ProcessPipeline? +---@return string? error +local function build_metric_pipeline(endpoint, process_config) + local pipeline, pipeline_err = processing.new_process_pipeline() + if not pipeline then + return nil, string.format('Metric config [%s] failed to create pipeline: %s', endpoint, tostring(pipeline_err)) end - -- Validate field - local field_type = type(metric_config.field) - if metric_config.field and (field_type ~= "string" and field_type ~= "number") then - table.insert(warnings, { - msg = string.format("Metric config [%s] field must be string or number", - tostring(endpoint)), - endpoint = endpoint, - type = "metric" - }) + if process_config == nil then + -- An empty pipeline (pass-through) is valid. + return pipeline, nil end - return warnings + + for _, blk_cfg in ipairs(process_config) do + local ptype = blk_cfg.type + if ptype == nil then + return nil, string.format('Metric config [%s] has process block with no type', endpoint) + end + + local proc_class = processing[ptype] + if proc_class == nil then + return nil, string.format('Metric config [%s] has invalid process block type [%s]', + endpoint, tostring(ptype)) + end + + local proc, proc_err = proc_class.new(blk_cfg) + if not proc or proc_err then + return nil, string.format( + 'Metric config [%s] failed to create process block [%s]: %s', + endpoint, tostring(ptype), tostring(proc_err)) + end + + local add_err = pipeline:add(proc) + if add_err then + return nil, add_err + end + end + + return pipeline, nil end +------------------------------------------------------------------------------- +-- Public: validate_config +------------------------------------------------------------------------------- + +--- Validate a raw metrics config table. +---@param config table +---@return boolean ok +---@return table warnings +---@return string? error local function validate_config(config) + if type(config) ~= 'table' then + return false, {}, 'Config is not a table' + end + + local data = config.data or {} local warnings = {} - if type(config) ~= "table" then - return false, warnings, "Invalid configuration message" + if type(data) ~= 'table' then + return false, warnings, 'Invalid configuration message' end - if type(config.publish_period) ~= "number" or tonumber(config.publish_period) == nil then - return false, warnings, "Publish period must be of number type, found " .. type(config.publish_period) + if data.schema ~= TARGET_SCHEMA then + return false, {}, string.format( + 'Unsupported config schema [%s], expected [%s]', tostring(data.schema), TARGET_SCHEMA) end - if config.publish_period <= 0 then - return false, warnings, "Publish period must be greater than 0" + if type(data.publish_period) ~= 'number' then + return false, warnings, + 'Publish period must be of number type, found ' .. type(data.publish_period) + end + if data.publish_period <= 0 then + return false, warnings, 'Publish period must be greater than 0' end - if type(config.collections) ~= "table" then - return false, warnings, "No metric collections defined in config" + if type(data.pipelines) ~= 'table' then + return false, warnings, 'No metric pipelines defined in config' end local dropped_templates = {} - for name, template in pairs(config.templates or {}) do - local template_warnings = validate_template(name, template) - if #template_warnings > 0 then - for _, warn in ipairs(template_warnings) do - table.insert(warnings, warn) + for name, tmpl in pairs(data.templates or {}) do + local tmpl_warns = validate_template(name, tmpl) + if #tmpl_warns > 0 then + for _, w in ipairs(tmpl_warns) do + table.insert(warnings, w) end dropped_templates[name] = true end end - local renames = {} - for endpoint, metric_config in pairs(config.collections) do - if metric_config.template and ((not config.templates) or (not config.templates[metric_config.template])) then - table.insert(warnings, { - msg = string.format("Metric config [%s] uses template [%s] that does not exist", - tostring(endpoint), tostring(metric_config.template)), - endpoint = endpoint, - type = "metric" - }) - end - if metric_config.template and dropped_templates[metric_config.template] then - table.insert(warnings, { - msg = string.format("Metric config [%s] uses invalid template [%s]", - tostring(endpoint), tostring(metric_config.template)), - endpoint = endpoint, - type = "metric" - }) + for endpoint, metric_config in pairs(data.pipelines) do + -- Check template existence + if metric_config.template then + if (not data.templates) or (not data.templates[metric_config.template]) then + table.insert(warnings, { + msg = string.format( + 'Metric config [%s] uses template [%s] that does not exist', + tostring(endpoint), tostring(metric_config.template)), + endpoint = endpoint, + type = 'metric', + }) + end + if dropped_templates[metric_config.template] then + table.insert(warnings, { + msg = string.format( + 'Metric config [%s] uses invalid template [%s]', + tostring(endpoint), tostring(metric_config.template)), + endpoint = endpoint, + type = 'metric', + }) + end end - local full_metric_config = merge_config( - config.templates and metric_config.template and config.templates[metric_config.template] or {}, + + -- Merge template then validate the resulting config + local full_cfg = merge_config( + (data.templates and metric_config.template + and data.templates[metric_config.template]) or {}, metric_config ) - local metric_warnings = validate_metric(endpoint, full_metric_config, renames) - for _, warn in ipairs(metric_warnings) do - table.insert(warnings, warn) + local metric_warns = validate_metric(endpoint, full_cfg) + for _, w in ipairs(metric_warns) do + table.insert(warnings, w) end end return true, warnings, nil end -local function apply_config(conn, config, cloud_config) - local merged_cloud_config = merge_config(cloud_config, { url = config.cloud_url }) - - local publish_period = config.publish_period - local metrics = {} - - -- iterate over each bus topic in our config - -- and build up a pipeline for each one - for endpoint, metric_config in pairs(config.collections) do - local sub_topic = {} - for part in endpoint:gmatch("[^/]+") do - sub_topic[#sub_topic + 1] = part +------------------------------------------------------------------------------- +-- Public: apply_config +------------------------------------------------------------------------------- + +--- Apply a validated config and return a pipelines_map. +--- Does not create any bus subscriptions. +--- +---@param config table +---@param log_fn? fun(level: string, payload: any) optional logger; defaults to log.warn/error +---@return PipelineMap pipelines_map keyed by metric_name +---@return number publish_period +local function apply_config(config, log_fn) + local data = config.data + log_fn = log_fn or function() end + + local publish_period = data.publish_period + local pipelines_map = {} + + for metric_name, metric_config in pairs(data.pipelines) do + local resolved = metric_config + if resolved.template and data.templates and data.templates[resolved.template] then + resolved = merge_config(data.templates[resolved.template], resolved) end - local sub = conn:subscribe(sub_topic) - if metric_config.template then - metric_config = merge_config( - config.templates[metric_config.template], - metric_config - ) - end - - -- protocol defines the publish method to be used - local protocol = metric_config.protocol - - -- create our processing pipeline for this endpoint - local base_pipeline, pipeline_err = build_metric_pipeline(endpoint, metric_config.process) - if pipeline_err then - log.error(pipeline_err) + local protocol = resolved.protocol + if not protocol or not VALID_PROTOCOLS[protocol] then + log_fn('warn', { + what = 'pipeline_skipped', + pipeline = tostring(metric_name), + reason = 'invalid or missing protocol', + }) else - local metric_inst = { - sub = sub, - field = metric_config.field, - rename = metric_config.rename, - protocol = protocol, - base_pipeline = base_pipeline - } - - table.insert(metrics, metric_inst) + local pipeline, pipeline_err = build_metric_pipeline( + metric_name, resolved.process or {}) + if pipeline_err then + log_fn('error', { + what = 'pipeline_skipped', + pipeline = tostring(metric_name), + err = pipeline_err, + }) + else + pipelines_map[metric_name] = { + pipeline = pipeline, + protocol = protocol, + } + end end end - return metrics, publish_period, merged_cloud_config + + return pipelines_map, publish_period end return { - merge_config = merge_config, - standardise_config = standardise_config, - validate_http_config = validate_http_config, - validate_config = validate_config, - apply_config = apply_config, - build_metric_pipeline = build_metric_pipeline + merge_config = merge_config, + standardise_config = standardise_config, + validate_http_config = validate_http_config, + validate_config = validate_config, + apply_config = apply_config, + build_metric_pipeline = build_metric_pipeline, } diff --git a/src/services/metrics/http.lua b/src/services/metrics/http.lua index 060ffdab..36d032f1 100644 --- a/src/services/metrics/http.lua +++ b/src/services/metrics/http.lua @@ -1,79 +1,126 @@ -local context = require "fibers.context" -local queue = require "fibers.queue" -local op = require "fibers.op" -local sleep = require "fibers.sleep" -local service = require "service" -local request = require 'http.request' -local log = require "services.log" +-- services/metrics/http.lua +-- +-- HTTP publisher for the metrics service. +-- +-- Starts a dedicated fiber that drains a bounded channel of HTTP payloads and +-- sends them to the Mainflux cloud endpoint with exponential-backoff retry on +-- network failure. +-- +-- Uses the HTTP capability service (services.http.sdk) so outbound requests +-- are handled via fiber-native Ops and do not block the cqueues event loop. +-- +-- Public API: +-- start_http_publisher(http_ref, log_fn?) -> channel +-- Must be called from inside a running fiber scope. Returns the send +-- channel (capacity QUEUE_SIZE). The caller enqueues payloads with a +-- non-blocking put; if the channel is full the payload is dropped and an +-- error is logged. +-- http_ref: an http SDK ref obtained via http_sdk.new_ref(conn, id) +-- log_fn(level, payload) is an optional logger; defaults to log.debug/info. -local QUEUE_SIZE = 10 +local fibers = require 'fibers' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local channel = require 'fibers.channel' +local blob_source = require 'devicecode.blob_source' -local function send_http(ctx, data) - local uri = data.uri - local body = data.body - local auth = data.auth +local QUEUE_SIZE = 10 +local HTTP_TIMEOUT = 60 - local response_headers - local sleep_duration = 1 - while not response_headers and not ctx:err() do - local req = request.new_from_uri(uri) - req.headers:upsert(":method", "POST") - req.headers:upsert("authorization", auth) - req.headers:upsert("content-type", "application/senml+json") - req:set_body(body) - req.headers:delete("expect") - local headers, _ = req:go(10) - response_headers = headers - if not response_headers then - log.debug(string.format( - "%s - %s: HTTP publish failed, retrying in %s seconds", - ctx:value("service_name"), - ctx:value("fiber_name"), - sleep_duration - )) - sleep.sleep(sleep_duration) - sleep_duration = math.min(sleep_duration * 2, 60) -- Exponential backoff, max 60 seconds - end - end +--- Send a single HTTP payload to the cloud via the HTTP capability service, +--- retrying with exponential backoff on failure. Returns only when the send +--- succeeds or the enclosing scope is cancelled. +--- +---@param http_ref table HTTP SDK ref (services.http.sdk Ref) +---@param data table { uri: string, auth: string, body: string } +---@param log_fn fun(level: string, payload: any) +local function send_http(http_ref, data, log_fn) + local uri = data.uri + local body = data.body + local auth = data.auth - if response_headers:get(":status") ~= "202" then - local header_msgs = {} - for k, v in response_headers:each() do - table.insert(header_msgs, string.format("\t%s: %s", k, v)) - end + local sleep_duration = 1 + local reply - log.debug(string.format( - "%s - %s: HTTP publish failed, header responses:\n%s", - ctx:value('service_name'), - ctx:value('fiber_name'), - table.concat(header_msgs, "\n") - )) - else - log.info(string.format( - "%s - %s: HTTP publish success, response: %s", - ctx:value('service_name'), - ctx:value('fiber_name'), - response_headers:get(":status") - )) - end + while not reply do + log_fn('trace', { what = 'http_publish_attempt', status = 'started' }) + local which, result, err = fibers.perform(op.named_choice({ + response = http_ref:exchange_op({ + method = 'POST', + uri = uri, + headers = { + authorization = auth, + ['content-type'] = 'application/senml+json', + }, + body_source = blob_source.from_string(body), + }), + timeout = sleep.sleep_op(HTTP_TIMEOUT), + })) + log_fn('trace', { what = 'http_publish_attempt', status = which }) + + if which == 'timeout' or not result then + local err_msg = which == 'timeout' and 'timeout' or tostring(err) + log_fn('debug', { what = 'http_retry', retry_in_s = sleep_duration, err = err_msg }) + sleep.sleep(sleep_duration) + sleep_duration = math.min(sleep_duration * 2, 60) + else + reply = result + end + end + + local status = reply.result and reply.result.status + if status ~= '202' then + local parts = {} + for k, v in pairs(reply.result and reply.result.headers or {}) do + table.insert(parts, string.format('\t%s: %s', k, v)) + end + log_fn('warn', { + what = 'http_publish_failed', + status = tostring(status), + headers = table.concat(parts, '\n'), + }) + else + log_fn('trace', { what = 'http_publish_ok', status = status }) + end end -local function start_http_publisher(ctx, conn) - local http_ctx = context.with_cancel(ctx) - local to_send_queue = queue.new(QUEUE_SIZE) +--- Start the HTTP publisher fiber in the current scope. +--- Returns the send channel. Payloads must be enqueued with a non-blocking +--- select (see http_publish in metrics.lua); if the channel is full the +--- payload should be dropped by the caller. +--- +---@param http_ref table HTTP SDK ref +---@param log_fn? fun(level: string, payload: any) optional logger +---@return table channel +local function start_http_publisher(http_ref, log_fn) + log_fn = log_fn or function() end + + local send_ch = channel.new(QUEUE_SIZE) + + fibers.spawn(function() + fibers.run_scope(function(s) + s:finally(function(aborted, _, sc_err) + if aborted then + log_fn('fatal', 'HTTP publisher fiber: ' .. tostring(sc_err)) + else + log_fn('trace', 'HTTP publisher fiber: exiting') + end + end) - service.spawn_fiber("HTTP Publish", conn, ctx, function () - while not http_ctx:err() do - op.choice( - to_send_queue:get_op():wrap(function (data) send_http(http_ctx, data) end), - http_ctx:done_op() - ):perform() - end - end) + while true do + local which, payload = fibers.perform(op.named_choice({ + msg = send_ch:get_op(), + })) + if which == 'msg' and payload ~= nil then + send_http(http_ref, payload, log_fn) + end + end + end) + end) - return to_send_queue + return send_ch end return { - start_http_publisher = start_http_publisher + start_http_publisher = start_http_publisher, } diff --git a/src/services/metrics/processing.lua b/src/services/metrics/processing.lua index 42746504..27656f64 100644 --- a/src/services/metrics/processing.lua +++ b/src/services/metrics/processing.lua @@ -1,248 +1,297 @@ -local sc = require 'fibers.utils.syscall' +-- services/metrics/processing.lua +-- +-- Processing pipeline blocks for the metrics service. +-- +-- Each block class holds only configuration (logic). Runtime state is kept in a +-- separate table created by :new_state() and passed explicitly to :run() and +-- :reset(). This allows a single pipeline object to be shared across many +-- metric endpoints while maintaining fully isolated per-endpoint state. +-- +-- Block interface: +-- block:new_state() -> state_table +-- block:run(value, state) -> value, short_circuit, error +-- block:reset(state) -> nil +-- +-- ProcessPipeline wraps a list of blocks and exposes the same interface: +-- pipeline:new_state() -> state_table {full_run, blocks={...}} +-- pipeline:run(value, state) -> value, short_circuit, error +-- pipeline:reset(state) -> nil (only resets when full_run == true) +-- pipeline:force_reset(state) -> nil (reset unconditionally) +-- pipeline:add(block) -> error? ---- Base Process class setup ----@class Process -local Process = {} -Process.__index = Process -- Setup instance->class lookup +local runtime = require 'fibers.runtime' -function Process.new() - return setmetatable({}, Process) -end +------------------------------------------------------------------------------- +-- Base Process (documentation only; not used at runtime) +------------------------------------------------------------------------------- ----@param value any ----@return any ----@return boolean # Short-circuit flag ----@return string? Error -function Process:run(value) - return nil, true, "run must be implemented by derived class" -end - ----@return Process? -function Process:clone() - error("clone must be implemented by derived class") -end +---@class Process +---@field new_state fun(self: Process): table +---@field run fun(self: Process, value: any, state: table): any, boolean, string? +---@field reset fun(self: Process, state: table) -function Process:reset() - error("reset must be implemented by derived class") -end +------------------------------------------------------------------------------- +-- DiffTrigger +------------------------------------------------------------------------------- ---- @class DiffTrigger: Process ---- @field threshold number ---- @field diff_method function ---- @field config table +--- Passes a value only when the change from the last-published value exceeds a +--- threshold. Three diff methods are supported: +--- "absolute" - abs(curr - last) >= threshold +--- "percent" - abs((curr - last) / last) * 100 >= threshold +--- "any-change" - curr ~= last +--- +---@class DiffTrigger +---@field threshold number +---@field diff_fn function +---@field config table local DiffTrigger = {} DiffTrigger.__index = DiffTrigger -setmetatable(DiffTrigger, { __index = Process }) local function check_diff_args_valid(config) - if config.initial_val and type(config.initial_val) ~= 'number' then + if config.initial_val ~= nil and type(config.initial_val) ~= 'number' then return 'Initial value must be a number' end if config.diff_method ~= 'any-change' and type(config.threshold) ~= 'number' then return 'Threshold must be a number' end end + ---@param config table ---@return DiffTrigger? ----@return string? Error +---@return string? error function DiffTrigger.new(config) - local initial_val = config.initial_val - local diff_method = config.diff_method - local threshold = config.threshold local valid_err = check_diff_args_valid(config) if valid_err then return nil, valid_err end + local self = setmetatable({}, DiffTrigger) - if diff_method == 'absolute' then - self.diff_method = function(curr, last, threshold) return math.abs(curr - last) >= threshold end - elseif diff_method == 'percent' then - self.diff_method = function(curr, last, threshold) return (math.abs((curr - last) / last) * 100) >= threshold end - elseif diff_method == 'any-change' then - self.diff_method = function (curr, last) return curr ~= last end + self.config = config + self.threshold = config.threshold + + local dm = config.diff_method + if dm == 'absolute' then + self.diff_fn = function(curr, last, threshold) + return math.abs(curr - last) >= threshold + end + elseif dm == 'percent' then + self.diff_fn = function(curr, last, threshold) + return (math.abs((curr - last) / last) * 100) >= threshold + end + elseif dm == 'any-change' then + self.diff_fn = function(curr, last) + return curr ~= last + end else return nil, "Diff method must be 'absolute', 'percent' or 'any-change'" end - self.empty = (initial_val == nil) - self.threshold = threshold - self.last_val = initial_val or 0 - self.curr_val = initial_val - self.config = config return self, nil end +---@return table +function DiffTrigger:new_state() + return { + empty = (self.config.initial_val == nil), + last_val = self.config.initial_val or 0, + curr_val = nil, + } +end + ---@param value any +---@param state table ---@return any ----@return boolean # Short-circuit flag ----@return string? Error -function DiffTrigger:run(value) - self.curr_val = value - if self.empty or self.diff_method(self.curr_val, self.last_val, self.threshold) then - self.last_val = value - self.empty = false +---@return boolean short_circuit +---@return string? error +function DiffTrigger:run(value, state) + state.curr_val = value + if state.empty or self.diff_fn(state.curr_val, state.last_val, self.threshold) then + state.last_val = value + state.empty = false return value, false, nil end return nil, true, nil end ----@return DiffTrigger? ----@return string? -function DiffTrigger:clone() - return DiffTrigger.new(self.config) -end +--- No-op: DiffTrigger does not reset last_val on publish. +---@param state table +function DiffTrigger:reset(state) end -- luacheck: ignore -function DiffTrigger:reset() -end +------------------------------------------------------------------------------- +-- TimeTrigger +------------------------------------------------------------------------------- ---- @class TimeTrigger: Process ---- @field duration number ---- @field config table +--- Passes a value only when the elapsed time since the last pass exceeds +--- `duration` seconds. +--- +---@class TimeTrigger +---@field duration number +---@field config table local TimeTrigger = {} TimeTrigger.__index = TimeTrigger -setmetatable(TimeTrigger, { __index = Process }) ---@param config table ---@return TimeTrigger? ----@return string? Error +---@return string? error function TimeTrigger.new(config) - local duration = config.duration - if duration == nil or type(duration) ~= "number" then return nil, "Duration must be a number" end - local self = setmetatable({}, TimeTrigger) - self.duration = duration - self.timeout = sc.monotime() + duration - self.config = config + if type(config.duration) ~= 'number' then + return nil, 'Duration must be a number' + end + local self = setmetatable({}, TimeTrigger) + self.duration = config.duration + self.config = config return self, nil end +---@return table +function TimeTrigger:new_state() + return { timeout = runtime.now() + self.duration } +end + ---@param value any +---@param state table ---@return any ----@return boolean # Short-circuit flag ----@return string? Error -function TimeTrigger:run(value) - if sc.monotime() >= self.timeout then - self.timeout = sc.monotime() + self.duration +---@return boolean short_circuit +---@return string? error +function TimeTrigger:run(value, state) + if runtime.now() >= state.timeout then + state.timeout = runtime.now() + self.duration return value, false, nil end return nil, true, nil end ----@return TimeTrigger? ----@return string? Error -function TimeTrigger:clone() - return TimeTrigger.new(self.config) -end +---@param state table +function TimeTrigger:reset(state) end -- luacheck: ignore -function TimeTrigger:reset() -end +------------------------------------------------------------------------------- +-- DeltaValue +------------------------------------------------------------------------------- ----@class DeltaValue: Process +--- Replaces the raw value with the difference from the last-published value. +--- Requires numeric input. +--- +---@class DeltaValue ---@field config table local DeltaValue = {} DeltaValue.__index = DeltaValue -setmetatable(DeltaValue, { __index = Process }) ---@param config table ----@return DeltaValue +---@return DeltaValue? +---@return string? error function DeltaValue.new(config) + if config.initial_val ~= nil and type(config.initial_val) ~= 'number' then + return nil, 'Initial value must be a number' + end local self = setmetatable({}, DeltaValue) - self.last_val = config.initial_val or 0 self.config = config - return self + return self, nil +end + +---@return table +function DeltaValue:new_state() + return { + last_val = self.config.initial_val or 0, + curr_val = nil, + } end ---@param value any +---@param state table ---@return any ----@return boolean # Short-circuit flag ----@return string? Error -function DeltaValue:run(value) - if type(value) ~= 'number' then return nil, nil, 'Value must be a number' end - local difference = value - self.last_val - self.curr_val = value +---@return boolean short_circuit +---@return string? error +function DeltaValue:run(value, state) + if type(value) ~= 'number' then + return nil, false, 'Value must be a number' + end + local difference = value - state.last_val + state.curr_val = value return difference, false, nil end -function DeltaValue:reset() - self.last_val = self.curr_val or 0 +--- On reset, advance last_val to curr_val so the next delta is computed from +--- the most-recently-published sample. +---@param state table +function DeltaValue:reset(state) + state.last_val = state.curr_val or 0 end ----@return DeltaValue -function DeltaValue:clone() - return DeltaValue.new(self.config) -end +------------------------------------------------------------------------------- +-- ProcessPipeline +------------------------------------------------------------------------------- ----@class ProcessPipeline: Process ----@field process_blocks Process[] +---@class ProcessPipeline +---@field process_blocks table local ProcessPipeline = {} ProcessPipeline.__index = ProcessPipeline ----comment ----@param processing_block Process ----@return string? Error -function ProcessPipeline:add(processing_block) - if processing_block == nil then return 'processing block cannot be nil' end - table.insert(self.process_blocks, processing_block) +---@return ProcessPipeline +local function new_process_pipeline() + return setmetatable({ process_blocks = {} }, ProcessPipeline) +end + +--- Append a processing block to the pipeline. +---@param block any +---@return string? error +function ProcessPipeline:add(block) + if block == nil then return 'processing block cannot be nil' end + table.insert(self.process_blocks, block) +end + +--- Create a fresh state table for this pipeline (and all its blocks). +---@return table +function ProcessPipeline:new_state() + local state = { full_run = false, blocks = {} } + for i, block in ipairs(self.process_blocks) do + state.blocks[i] = block:new_state() + end + return state end +--- Run the pipeline, passing value through each block sequentially. +--- Stops early if any block short-circuits or returns an error. ---@param value any +---@param state table ---@return any ----@return boolean # Short-circuit flag ----@return string? Error -function ProcessPipeline:run(value) - local val = value +---@return boolean short_circuit +---@return string? error +function ProcessPipeline:run(value, state) + local val = value local short = false - local err = nil - for _, process in ipairs(self.process_blocks) do - val, short, err = process:run(val) + local err = nil + + for i, block in ipairs(self.process_blocks) do + val, short, err = block:run(val, state.blocks[i]) if err or short then break end end - if (not short) and (not err) then self.full_run = true end - -- if not err and not short then self:reset() end - return val, short, err -end ---- Reset only if the pipeline has output a value ---- without short circuiting -function ProcessPipeline:reset() - if self.full_run then - for _, process in ipairs(self.process_blocks) do - process:reset() - end - self.full_run = false + if not short and not err then + state.full_run = true end -end ---- Reset whether the pipeline has output or not -function ProcessPipeline:force_reset() - self.full_run = true - self:reset() + return val, short, err end ----@return ProcessPipeline? ----@return string? Error -function ProcessPipeline:clone() - local pipeline = setmetatable({ full_run = false, process_blocks = {} }, ProcessPipeline) - for _, process_block in ipairs(self.process_blocks) do - local process, err = process_block:clone() - if not process or err then - return nil, err +--- Reset block states, but only when the pipeline produced a published value +--- (i.e. ran to completion without short-circuiting). +---@param state table +function ProcessPipeline:reset(state) + if state.full_run then + for i, block in ipairs(self.process_blocks) do + block:reset(state.blocks[i]) end - pipeline:add(process) + state.full_run = false end - - return pipeline, nil end ---- @return ProcessPipeline? ---- @return string? Error -local function new_process_pipeline() - -- if process_config == nil then return nil, 'Cannot create a process pipeline with no config' end - local self = setmetatable({}, ProcessPipeline) - self.process_blocks = {} - self.full_run = false - return self +--- Reset regardless of whether the pipeline produced a published value. +---@param state table +function ProcessPipeline:force_reset(state) + state.full_run = true + self:reset(state) end return { - DiffTrigger = DiffTrigger, - TimeTrigger = TimeTrigger, - DeltaValue = DeltaValue, - new_process_pipeline = new_process_pipeline + DiffTrigger = DiffTrigger, + TimeTrigger = TimeTrigger, + DeltaValue = DeltaValue, + new_process_pipeline = new_process_pipeline, } diff --git a/src/services/metrics/senml.lua b/src/services/metrics/senml.lua index 05273dbf..f493c86c 100644 --- a/src/services/metrics/senml.lua +++ b/src/services/metrics/senml.lua @@ -1,51 +1,84 @@ +-- services/metrics/senml.lua +-- +-- SenML (Sensor Markup Language) encoder for the metrics service. +-- Converts a nested metric-values table into a flat array of SenML objects +-- ready for JSON encoding and HTTP publication. + +--- Encode a single (name, value, time) triple as a SenML record. +---@param topic string +---@param value number|string|boolean +---@param time number? milliseconds since epoch (optional) +---@return SenMLRecord? senml_obj +---@return string? error local function encode(topic, value, time) + if type(topic) ~= 'string' or topic == '' then + return nil, 'topic must be a non-empty string' + end local vtype = type(value) - if vtype ~= 'number' and vtype ~= 'string' and vtype ~= "boolean" then - return nil, "value must be number, string or boolean, found " .. vtype + if vtype ~= 'number' and vtype ~= 'string' and vtype ~= 'boolean' then + return nil, 'value must be number, string or boolean, found ' .. vtype end - local senml_obj = {n = topic} + local obj = { n = topic } - if vtype == 'string' then senml_obj.vs = value - elseif vtype == 'number' then senml_obj.v = value - elseif vtype == 'boolean' then senml_obj.vb = value + if vtype == 'number' then + obj.v = value + elseif vtype == 'string' then + obj.vs = value + elseif vtype == 'boolean' then + obj.vb = value end if time and type(time) == 'number' then - senml_obj.t = time + obj.t = time end - return senml_obj + + return obj, nil end +--- Recursively encode a nested values table into a flat SenML array. +--- +--- Each leaf may be either: +--- {value = v, time = t} - a metric sample +--- a plain value - treated as {value = v} +--- +--- Tables without both 'value' and 'time' fields are recursed into with the +--- key appended to the topic (dot-separated). The special key '__value' does +--- not append anything to the topic. +--- +---@param base_topic string +---@param values table +---@param output table accumulator array (created automatically on top call) +---@return table? output +---@return string? error local function encode_r(base_topic, values, output) for k, v in pairs(values) do local topic = base_topic if k ~= '__value' then - if base_topic == '' then + if base_topic == '' then topic = k else - topic = topic .. "." .. k + topic = topic .. '.' .. k end end - -- check if v is a table with value and time - if type(v) == 'table' and not (v.value and v.time) then - -- if v is a table with no value and time, recurse + if type(v) == 'table' and (v.value == nil or v.time == nil) then local _, err = encode_r(topic, v, output) if err then return nil, err end else + local clean_metric = v if type(v) ~= 'table' then - v = {value = v} + clean_metric = { value = v } end - local senml_obj, err = encode(topic, v.value, v.time) + local obj, err = encode(topic, clean_metric.value, clean_metric.time) if err then return nil, err end - table.insert(output, senml_obj) + table.insert(output, obj) end end return output, nil end return { - encode = encode, - encode_r = function(base_topic, values) return encode_r(base_topic, values, {}) end + encode = encode, + encode_r = function(base_topic, values) return encode_r(base_topic, values, {}) end, } diff --git a/src/services/metrics/state_projection.lua b/src/services/metrics/state_projection.lua new file mode 100644 index 00000000..273ad4ce --- /dev/null +++ b/src/services/metrics/state_projection.lua @@ -0,0 +1,30 @@ +-- services/metrics/state_projection.lua +-- +-- Transitional mapping table for the future metrics pipeline. +-- +-- The intended direction is for metrics to observe retained domain /state and +-- derive gauges/counters from that public model, rather than requiring each +-- service to actively publish UI-critical operational facts as metrics. + +return { + { + topic = { 'state', 'system', 'stats' }, + metrics = { + { path = { 'cpu', 'utilisation' }, name = 'system.cpu_util' }, + { path = { 'memory', 'utilisation' }, name = 'system.mem_util' }, + { path = { 'thermal', 'zone0', 'temp_c' }, name = 'system.temp' }, + }, + }, + { + topic = { 'state', 'net', 'backhaul' }, + metrics = { + -- Future projection: per-uplink availability, usable state and uptime. + }, + }, + { + topic = { 'state', 'gsm', 'uplink', '+' }, + metrics = { + -- Future projection: cellular connectivity and SIM state. + }, + }, +} diff --git a/src/services/metrics/types.lua b/src/services/metrics/types.lua new file mode 100644 index 00000000..372d8a95 --- /dev/null +++ b/src/services/metrics/types.lua @@ -0,0 +1,178 @@ +-- services/metrics/types.lua +-- +-- Shared type definitions and constructors for the metrics service. +-- Follows the same pattern as services/hal/types/external.lua. + +---@class TypeConstructors +local new = {} + +------------------------------------------------------------------------------- +-- BaseTime +------------------------------------------------------------------------------- + +---@class BaseTime +---@field synced boolean +---@field real number wall-clock seconds at the monotonic base +---@field mono number monotonic seconds at the base +local BaseTime = {} +BaseTime.__index = BaseTime + +---Create a new BaseTime anchored to the supplied real/monotonic pair. +---@param real number +---@param mono number +---@return BaseTime? +---@return string error +function new.BaseTime(real, mono) + if type(real) ~= 'number' then + return nil, "real must be a number" + end + if type(mono) ~= 'number' then + return nil, "mono must be a number" + end + return setmetatable({ + synced = false, + real = real, + mono = mono, + }, BaseTime), "" +end + +------------------------------------------------------------------------------- +-- CloudConfig +------------------------------------------------------------------------------- + +---@class CloudChannel +---@field id string +---@field name string +---@field metadata table + +---@class CloudConfig +---@field url string +---@field thing_key string +---@field channels CloudChannel[] +local CloudConfig = {} +CloudConfig.__index = CloudConfig + +---Create a new CloudConfig. +---@param url string +---@param thing_key string +---@param channels CloudChannel[] +---@return CloudConfig? +---@return string error +function new.CloudConfig(url, thing_key, channels) + if type(url) ~= 'string' or url == '' then + return nil, "url must be a non-empty string" + end + if type(thing_key) ~= 'string' or thing_key == '' then + return nil, "thing_key must be a non-empty string" + end + if type(channels) ~= 'table' then + return nil, "channels must be a table" + end + return setmetatable({ + url = url, + thing_key = thing_key, + channels = channels, + }, CloudConfig), "" +end + +------------------------------------------------------------------------------- +-- MetricSample (per-endpoint cache entry) +------------------------------------------------------------------------------- + +---@class MetricSample +---@field value number|string|boolean +---@field time number monotonic seconds when the value was recorded +local MetricSample = {} +MetricSample.__index = MetricSample + +---Create a new MetricSample. +---@param value number|string|boolean +---@param time number +---@return MetricSample? +---@return string error +function new.MetricSample(value, time) + local vt = type(value) + if vt ~= 'number' and vt ~= 'string' and vt ~= 'boolean' then + return nil, "value must be number, string or boolean" + end + if type(time) ~= 'number' then + return nil, "time must be a number" + end + return setmetatable({ + value = value, + time = time, + }, MetricSample), "" +end + +------------------------------------------------------------------------------- +-- SenMLRecord (single SenML object ready for JSON encoding) +------------------------------------------------------------------------------- + +---@class SenMLRecord +---@field n string SenML name +---@field v number? numeric value +---@field vs string? string value +---@field vb boolean? boolean value +---@field t number? time in milliseconds since epoch +local SenMLRecord = {} +SenMLRecord.__index = SenMLRecord + +---Create a new SenMLRecord. +---@param name string +---@param value number|string|boolean +---@param time number? milliseconds since epoch +---@return SenMLRecord? +---@return string? error +function new.SenMLRecord(name, value, time) + if type(name) ~= 'string' or name == '' then + return nil, "name must be a non-empty string" + end + local vt = type(value) + if vt ~= 'number' and vt ~= 'string' and vt ~= 'boolean' then + return nil, "value must be number, string or boolean, found " .. vt + end + if time ~= nil and type(time) ~= 'number' then + return nil, "time must be a number" + end + + local obj = setmetatable({ n = name }, SenMLRecord) + if vt == 'number' then obj.v = value end + if vt == 'string' then obj.vs = value end + if vt == 'boolean' then obj.vb = value end + if time then obj.t = time end + return obj, nil +end + +------------------------------------------------------------------------------- +-- ServiceState (the live mutable state table held in metrics.lua) +------------------------------------------------------------------------------- + +---@alias PipelineEntry { pipeline: ProcessPipeline, protocol: string } +---@alias PipelineMap table +---@alias MetricStates table +---@alias MetricValues table> + +---@class ServiceState +---@field svc ServiceBase? +---@field conn Connection? nil before M.start() is called +---@field name string? nil before M.start() is called +---@field http_ref CapabilityReference? +---@field http_send_ch Channel? nil before M.start() is called +---@field pipelines_map PipelineMap +---@field metric_states MetricStates +---@field endpoint_to_pipe table +---@field metric_values MetricValues +---@field publish_period number? +---@field cloud_url string? +---@field mainflux_config table? +---@field cloud_config CloudConfig? +---@field base_time BaseTime? nil before M.start() is called +---@field fs_cap CapabilityReference? nil before filesystem cap is resolved + +return { + new = new, + BaseTime = BaseTime, + CloudConfig = CloudConfig, + MetricSample = MetricSample, + SenMLRecord = SenMLRecord, +} diff --git a/src/services/monitor.lua b/src/services/monitor.lua new file mode 100644 index 00000000..b8e1c931 --- /dev/null +++ b/src/services/monitor.lua @@ -0,0 +1,939 @@ +-- services/monitor.lua +-- +-- Monitor service: +-- * operator mode subscribes to observability but prints structured logs only +-- * keeps a boot buffer and rolling in-memory ring of normalised log records +-- * raw mode preserves the previous firehose behaviour for development +-- * legacy obs-plane detection is retained, but no longer pollutes operator mode + +local OBS_VER = 'v1' + +local DEFAULT_BOOT_RECORDS = 200 +local DEFAULT_RING_RECORDS = 50 +-- A value <= 0 means 'capture the first DEFAULT_BOOT_RECORDS records regardless of elapsed time'. +local DEFAULT_BOOT_WINDOW_S = 0 + +local fibers = require 'fibers' +local runtime = require 'fibers.runtime' +local file = require 'fibers.io.file' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' + +local queue = require 'devicecode.support.queue' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local config_watch = require 'devicecode.support.config_watch' + +local perform = fibers.perform +local named_choice = fibers.named_choice + +local base = require 'devicecode.service_base' + +local M = {} + +local LEVEL_VALUE = { + trace = 10, + debug = 20, + info = 30, + warn = 40, + error = 50, + fatal = 60, +} + +local function env(name, default) + local v = os.getenv(name) + if v == nil or v == '' then return default end + return v +end + +local function env_num(name, default) + local n = tonumber(env(name, nil)) + if n == nil then return default end + return n +end + +local function first_num(default, ...) + for i = 1, select('#', ...) do + local v = select(i, ...) + local n = tonumber(v) + if n ~= nil then return n end + end + return default +end + +local function norm_level(level) + level = tostring(level or 'info'):lower() + if level == 'warning' then level = 'warn' end + if LEVEL_VALUE[level] then return level end + return 'info' +end + +local function level_enabled(level, min_level) + return (LEVEL_VALUE[norm_level(level)] or LEVEL_VALUE.info) >= (LEVEL_VALUE[norm_level(min_level)] or LEVEL_VALUE.info) +end + +-- pretty printer kept intentionally small; this is an operator/dev tool, not a +-- serialisation contract. +local function is_array(t) + local n = 0 + for _ in ipairs(t) do n = n + 1 end + for k in pairs(t) do + if type(k) ~= 'number' or k < 1 or k % 1 ~= 0 or k > n then return false end + end + return true +end + +local function sort_keys(t) + local ks = {} + for k in pairs(t) do ks[#ks + 1] = k end + table.sort(ks, function(a, b) + local ta, tb = type(a), type(b) + if ta ~= tb then return ta < tb end + if ta == 'number' then return a < b end + return tostring(a) < tostring(b) + end) + return ks +end + +local function pretty(v, opts, depth, seen) + opts = opts or {} + depth = depth or 0 + seen = seen or {} + local max_depth = opts.max_depth or 5 + local max_items = opts.max_items or 30 + + local tv = type(v) + if tv == 'string' then return string.format('%q', v) end + if tv == 'number' or tv == 'boolean' or tv == 'nil' then return tostring(v) end + if tv ~= 'table' then return ('<%s:%s>'):format(tv, tostring(v)) end + if seen[v] then return '' end + seen[v] = true + if depth >= max_depth then seen[v] = nil; return '{...}' end + + local out, count = {}, 0 + if is_array(v) then + out[#out + 1] = '[' + for i = 1, #v do + count = count + 1 + if count > max_items then out[#out + 1] = '...'; break end + out[#out + 1] = pretty(v[i], opts, depth + 1, seen) + end + out[#out + 1] = ']' + else + out[#out + 1] = '{' + local keys = sort_keys(v) + for i = 1, #keys do + count = count + 1 + if count > max_items then out[#out + 1] = '...'; break end + local k = keys[i] + local kk = (type(k) == 'string') and k or ('[' .. tostring(k) .. ']') + out[#out + 1] = tostring(kk) .. '=' .. pretty(v[k], opts, depth + 1, seen) + end + out[#out + 1] = '}' + end + seen[v] = nil + return table.concat(out, ' ') +end + +local function topic_to_string(topic) + local parts = {} + for i = 1, #(topic or {}) do parts[#parts + 1] = tostring(topic[i]) end + return table.concat(parts, '/') +end + +local function monitor_cap_topic(id) + return { 'cap', 'monitor', id or 'main' } +end + +local function monitor_rpc_topic(method, id) + return { 'cap', 'monitor', id or 'main', 'rpc', method } +end + +local function monitor_state_topic(name) + return { 'state', 'monitor', name } +end + +local function fmt_time(mono) + mono = mono or (runtime.now and runtime.now()) or 0 + return os.date('%Y-%m-%d %H:%M:%S') .. string.format(' (mono=%.3f)', mono) +end + +local function classify_canonical(msg) + local t = msg.topic or {} + -- canonical service_base log: obs/v1//event/log + local svc, kind, name = t[3] or 'unknown', t[4], t[5] + if kind == 'event' and name == 'log' then + local level = (type(msg.payload) == 'table' and msg.payload.level) or 'info' + return 'log', svc, level + end + -- newer direct log shape used by http: obs/v1//log// + if kind == 'log' then + return 'log', svc, t[6] or t[5] or (type(msg.payload) == 'table' and msg.payload.level) or 'info' + end + if kind == 'event' then return 'event', svc, name or 'event' end + if kind == 'metric' then return 'metric', svc, nil end + if kind == 'counter' then return 'counter', svc, name or 'counter' end + return 'obs', svc, nil +end + +local function classify_legacy(msg) + local t = msg.topic or {} + local kind, svc = t[2], t[3] or 'unknown' + if kind == 'log' then return 'log', svc, t[4] or (type(msg.payload) == 'table' and msg.payload.level) or 'info' end + if kind == 'event' then return 'event', svc, t[4] or 'event' end + if kind == 'state' then return 'state', svc, t[4] or 'state' end + return 'obs', svc, nil +end + +local function payload_string(payload, max_depth, max_items) + return (type(payload) == 'string') and payload + or pretty(payload, { max_depth = max_depth or 6, max_items = max_items or 40 }) +end + +local DETAIL_SKIP_KEYS = { + service = true, env = true, run_id = true, ts = true, at = true, level = true, + what = true, summary = true, message = true, +} + +local DETAIL_PRIORITY = { + 'entity', 'id', 'role', 'modem', 'address', 'interface', 'uplink_id', 'device', + 'from', 'to', 'state', 'ready', 'reason', 'ok', 'err', 'peak_mbps', 'elapsed_ms', + 'generation', 'apply_id', 'weight_apply_id', 'links', 'components', 'backend', +} + +local function compact_value(v) + local tv = type(v) + if tv == 'string' then return v end + if tv == 'number' then + if math.floor(v) == v then return tostring(v) end + return string.format('%.3f', v):gsub('0+$', ''):gsub('%.$', '') + end + if tv == 'boolean' then return v and 'true' or 'false' end + if tv == 'nil' then return nil end + return pretty(v, { max_depth = 2, max_items = 5 }) +end + +local function summarise_log_payload(payload) + if type(payload) ~= 'table' then return tostring(payload or '') end + if type(payload.summary) == 'string' and payload.summary ~= '' then return payload.summary end + if type(payload.message) == 'string' and payload.message ~= '' then return payload.message end + + local parts, used = {}, {} + for _, k in ipairs(DETAIL_PRIORITY) do + local v = payload[k] + local s = compact_value(v) + if s ~= nil and s ~= '' then + parts[#parts + 1] = tostring(k) .. '=' .. s + used[k] = true + end + end + local keys = sort_keys(payload) + for _, k in ipairs(keys) do + if #parts >= 8 then break end + if type(k) == 'string' and not used[k] and not DETAIL_SKIP_KEYS[k] then + local s = compact_value(payload[k]) + if s ~= nil and s ~= '' then parts[#parts + 1] = tostring(k) .. '=' .. s end + end + end + local what = payload.what and tostring(payload.what) or nil + if #parts > 0 then + return what and (what .. ' ' .. table.concat(parts, ' ')) or table.concat(parts, ' ') + end + if what ~= nil then return what end + return payload_string(payload, 4, 12) +end + +local function format_log_line(rec) + return string.format('%s LOG %-10s %-5s %-24s %s', + fmt_time(rec.mono), tostring(rec.service), tostring(rec.level), tostring(rec.what or ''), summarise_log_payload(rec.payload)) +end + +local function format_canonical_line(msg) + local kind, svc, lvl_or_name = classify_canonical(msg) + local payload_s = payload_string(msg.payload, 6, 40) + if kind == 'log' then + local rec = { + mono = (runtime.now and runtime.now()) or 0, + service = svc, + level = norm_level(lvl_or_name), + what = type(msg.payload) == 'table' and msg.payload.what or nil, + payload = msg.payload, + } + return format_log_line(rec) + end + if kind == 'metric' then + return string.format('%s MET %-10s %-24s %s', fmt_time(), tostring(svc), topic_to_string(msg.topic), payload_s) + end + if kind == 'event' then + return string.format('%s EVT %-10s %-12s %s', fmt_time(), tostring(svc), tostring(lvl_or_name or 'event'), payload_s) + end + if kind == 'counter' then + return string.format('%s CNT %-10s %-12s %s', fmt_time(), tostring(svc), tostring(lvl_or_name or 'counter'), payload_s) + end + return string.format('%s OBS %-10s %-24s %s', fmt_time(), tostring(svc), topic_to_string(msg.topic), payload_s) +end + +local function format_legacy_warn(msg, reason) + local t = msg.topic or {} + local _, svc = classify_legacy(msg) + return string.format('%s WARN %-10s %-20s topic=%s %s', + fmt_time(), tostring(svc), tostring(reason), topic_to_string(t), payload_string(msg.payload, 4, 20)) +end + +local function normalise_log_record(msg, plane) + local kind, svc, level + if plane == 'canonical' then kind, svc, level = classify_canonical(msg) else kind, svc, level = classify_legacy(msg) end + if kind ~= 'log' then return nil end + local payload = msg.payload + if type(payload) ~= 'table' then payload = { message = tostring(payload or '') } end + return { + mono = (runtime.now and runtime.now()) or 0, + wall = os.date('%Y-%m-%d %H:%M:%S'), + service = tostring(payload.service or svc or 'unknown'), + level = norm_level(payload.level or level), + what = payload.what, + summary = payload.summary or payload.message, + payload = payload, + topic = msg.topic, + plane = plane, + } +end + + +local function shallow_copy_record_payload(payload) + if type(payload) ~= 'table' then return payload end + local out = {} + for k, v in pairs(payload) do out[k] = v end + return out +end + +local function serialise_record(rec) + if type(rec) ~= 'table' then return rec end + return { + id = rec.id, + mono = rec.mono, + wall = rec.wall, + service = rec.service, + level = rec.level, + what = rec.what, + summary = rec.summary or summarise_log_payload(rec.payload), + payload = shallow_copy_record_payload(rec.payload), + topic = topic_to_string(rec.topic), + plane = rec.plane, + } +end + +local function record_matches(rec, filter) + filter = filter or {} + if filter.min_level and not level_enabled(rec.level, filter.min_level) then return false end + if filter.service and tostring(filter.service) ~= '' and tostring(rec.service) ~= tostring(filter.service) then return false end + if filter.since_id and tonumber(rec.id or 0) <= tonumber(filter.since_id) then return false end + local contains = filter.contains or filter.q + if contains and tostring(contains) ~= '' then + local hay = table.concat({ tostring(rec.service or ''), tostring(rec.level or ''), tostring(rec.what or ''), tostring(rec.summary or ''), summarise_log_payload(rec.payload) }, ' '):lower() + if not hay:find(tostring(contains):lower(), 1, true) then return false end + end + return true +end + + +local function inc_count(t, key, n) + key = tostring(key or 'unknown') + t[key] = (t[key] or 0) + (n or 1) +end + +local function top_counts(t, max_items) + local items = {} + for k, v in pairs(t or {}) do items[#items + 1] = { k = k, v = v } end + table.sort(items, function(a, b) + if a.v ~= b.v then return a.v > b.v end + return tostring(a.k) < tostring(b.k) + end) + local out = {} + for i = 1, math.min(max_items or 5, #items) do + out[#out + 1] = tostring(items[i].k) .. '=' .. tostring(items[i].v) + end + return (#out > 0) and table.concat(out, ' ') or '-' +end + +local function new_period_counts() + return { + received = 0, + logs = 0, + stored = 0, + printed = 0, + dropped = 0, + suppressed = 0, + suppressed_by_category = {}, + suppressed_by_service = {}, + received_by_category = {}, + received_by_service = {}, + } +end + +local function log_category(level) + level = norm_level(level) + return 'log_' .. level +end + +local function ring_append(buf, rec, max_records) + if max_records <= 0 then return false, 0 end + buf[#buf + 1] = rec + local evicted = 0 + while #buf > max_records do + table.remove(buf, 1) + evicted = evicted + 1 + end + return true, evicted +end + +local function trim_first_records(buf, max_records) + if max_records <= 0 then + local n = #buf + for i = n, 1, -1 do buf[i] = nil end + return n + end + local removed = 0 + while #buf > max_records do + table.remove(buf) + removed = removed + 1 + end + return removed +end + +local function trim_latest_records(buf, max_records) + if max_records <= 0 then + local n = #buf + for i = n, 1, -1 do buf[i] = nil end + return n + end + local removed = 0 + while #buf > max_records do + table.remove(buf, 1) + removed = removed + 1 + end + return removed +end + +local function monitor_storage_cfg(raw) + if type(raw) ~= 'table' then return nil end + local cfg = raw.storage + if type(cfg) ~= 'table' then return nil end + return cfg +end + +local function field_num(cfg, ...) + if type(cfg) ~= 'table' then return nil end + for i = 1, select('#', ...) do + local k = select(i, ...) + local n = tonumber(cfg[k]) + if n ~= nil then + if n < 0 then n = 0 end + return math.floor(n) + end + end + return nil +end + +local function storage_env_cfg(prefix) + local cfg = {} + local any = false + local function set_num(field, suffix) + local n = tonumber(env(prefix .. suffix, nil)) + if n ~= nil then + if n < 0 then n = 0 end + cfg[field] = math.floor(n) + any = true + end + end + set_num('boot_records', '_BOOT_RECORDS') + set_num('ring_records', '_RING_RECORDS') + set_num('boot_seconds', '_BOOT_SECONDS') + if any then return cfg end + return nil +end + +function M.start(conn, ctx) + ctx = ctx or {} + local svc = base.new(conn, { name = ctx.name or 'monitor', env = ctx.env }) + local name = svc.name + + local profile = tostring(ctx.profile or env('DEVICECODE_MONITOR_PROFILE', 'operator')) + local min_level = tostring(ctx.min_level or env('DEVICECODE_MONITOR_LEVEL', profile == 'debug' and 'debug' or profile == 'trace' and 'trace' or 'info')) + local raw = profile == 'raw' + local legacy_detect = raw or env('DEVICECODE_MONITOR_LEGACY_DETECT', '0') == '1' + local summary_period_s = env_num('DEVICECODE_MONITOR_SUMMARY_PERIOD_S', 60) + -- Buffer sizing is deliberately not a service-start option. Defaults are + -- compiled into monitor; DEVICECODE_INITIAL_MONITOR_* can override those + -- earliest boot values before config is available. cfg/monitor may then set + -- the normal configured values, and DEVICECODE_MONITOR_* can force an + -- environment override over config for deployment/emergency use. + local initial_storage = storage_env_cfg('DEVICECODE_INITIAL_MONITOR') + local env_override_storage = storage_env_cfg('DEVICECODE_MONITOR') + local boot_max = field_num(env_override_storage, 'boot_records') + or field_num(initial_storage, 'boot_records') + or DEFAULT_BOOT_RECORDS + local ring_max = field_num(env_override_storage, 'ring_records') + or field_num(initial_storage, 'ring_records') + or DEFAULT_RING_RECORDS + local boot_window_s = field_num(env_override_storage, 'boot_seconds') + or field_num(initial_storage, 'boot_seconds') + or DEFAULT_BOOT_WINDOW_S + local storage_source = env_override_storage and 'env_override' or (initial_storage and 'initial_env' or 'default') + + local out = file.fdopen(1, 'w', 'stdout') + out:setvbuf('line') + + local function write_line(line) + local which, a, b = perform(named_choice { + wrote = out:write_op(line, '\n'), + timeout = sleep.sleep_op(0.5):wrap(function () return nil, 'write timeout' end), + }) + if which == 'wrote' then + local n, err = a, b + if n == nil and err ~= nil then error(err) end + end + end + + local start_mono = (runtime.now and runtime.now()) or 0 + local last_summary = start_mono + local ring, boot = {}, {} + local counts = { received = 0, logs = 0, stored = 0, printed = 0, dropped = 0, suppressed = 0, ring_evicted = 0 } + local period = new_period_counts() + local next_record_id = 0 + local followers = {} + local next_follower_id = 0 + local last_summary_payload = nil + + local function public_summary(extra) + local payload = { + profile = profile, + min_level = min_level, + ring_records = #ring, + ring_max_records = ring_max, + boot_records = #boot, + boot_max_records = boot_max, + boot_window_s = boot_window_s, + storage_source = storage_source, + counts = counts, + followers = 0, + last_record_id = next_record_id, + } + local n_followers = 0 + for _ in pairs(followers) do n_followers = n_followers + 1 end + payload.followers = n_followers + if type(extra) == 'table' then for k, v in pairs(extra) do payload[k] = v end end + last_summary_payload = payload + return payload + end + + local function retain_summary(extra) + conn:retain(monitor_state_topic('summary'), public_summary(extra)) + conn:retain(monitor_cap_topic('main'), { + kind = 'cap.monitor', + class = 'monitor', + id = 'main', + owner = name, + methods = { ['query-logs'] = true, ['follow-logs'] = true, ['set-profile'] = true }, + state = { summary = monitor_state_topic('summary') }, + }) + return last_summary_payload + end + + local function query_records(payload) + payload = type(payload) == 'table' and payload or {} + local source = payload.boot and boot or ring + local default_limit = payload.boot and boot_max or ring_max + if default_limit <= 0 then default_limit = 50 end + local limit = tonumber(payload.limit) or default_limit + if limit < 0 then limit = 0 end + if limit > 1000 then limit = 1000 end + local filter = { + min_level = payload.min_level, + service = payload.service, + since_id = payload.since_id, + contains = payload.contains or payload.q, + } + local out = {} + for i = #source, 1, -1 do + local rec = source[i] + if record_matches(rec, filter) then + table.insert(out, 1, serialise_record(rec)) + if #out >= limit then break end + end + end + return { ok = true, records = out, count = #out, last_record_id = next_record_id, summary = last_summary_payload or public_summary() } + end + + local function make_feed(payload) + payload = type(payload) == 'table' and payload or {} + next_follower_id = next_follower_id + 1 + local tx, rx = mailbox.new(tonumber(payload.queue_len) or 128, { full = payload.full or 'drop_oldest' }) + local id = next_follower_id + local feed = { _id = id, _tx = tx, _rx = rx, dropped = 0, closed = false } + local filter = { + min_level = payload.min_level, + service = payload.service, + since_id = payload.since_id, + contains = payload.contains or payload.q, + } + function feed:recv_op() return self._rx:recv_op() end + function feed:close(reason) + if self.closed then return true end + self.closed = true + followers[id] = nil + if self._tx and type(self._tx.close) == 'function' then self._tx:close(reason or 'monitor_follow_closed') end + return true + end + followers[id] = { feed = feed, filter = filter } + local replay = payload.replay ~= false + if replay then + local result = query_records({ + limit = payload.limit, + boot = payload.boot, + min_level = payload.min_level, + service = payload.service, + since_id = payload.since_id, + contains = payload.contains or payload.q, + }) + for _, rec in ipairs(result.records or {}) do + local ok = queue.try_send_now(tx, { kind = 'log', replay = true, record = rec }) + if ok ~= true then feed.dropped = feed.dropped + 1 end + end + end + queue.try_send_now(tx, { kind = 'ready', follower_id = id, last_record_id = next_record_id }) + return feed + end + + local function publish_status(extra) + local payload = { + profile = profile, + min_level = min_level, + ring_records = #ring, + ring_max_records = ring_max, + boot_records = #boot, + boot_max_records = boot_max, + boot_window_s = boot_window_s, + storage_source = storage_source, + counts = counts, + } + if type(extra) == 'table' then for k, v in pairs(extra) do payload[k] = v end end + retain_summary(extra) + svc:status('running', payload) + end + + local function apply_storage_config(raw, reason) + local cfg = monitor_storage_cfg(raw) + local override = storage_env_cfg('DEVICECODE_MONITOR') + if not cfg and not override then return false end + cfg = cfg or {} + + local new_boot = field_num(override, 'boot_records') or field_num(cfg, 'boot_records') + local new_ring = field_num(override, 'ring_records') or field_num(cfg, 'ring_records') + local new_window = field_num(override, 'boot_seconds') or field_num(cfg, 'boot_seconds') + local new_source = override and 'env_override' or 'config' + + local changed = false + if new_boot ~= nil and new_boot ~= boot_max then + boot_max = new_boot + trim_first_records(boot, boot_max) + changed = true + end + if new_ring ~= nil and new_ring ~= ring_max then + ring_max = new_ring + local evicted = trim_latest_records(ring, ring_max) + if evicted > 0 then counts.ring_evicted = counts.ring_evicted + evicted end + changed = true + end + if new_window ~= nil and new_window ~= boot_window_s then + boot_window_s = new_window + changed = true + end + if storage_source ~= new_source then + storage_source = new_source + changed = true + end + + if changed then + publish_status({ storage_configured = true, storage_reason = reason or 'config_changed' }) + svc:info('monitor_storage_configured', { + summary = string.format('monitor storage configured boot=%d ring=%d source=%s', boot_max, ring_max, storage_source), + boot_records = boot_max, + ring_records = ring_max, + boot_seconds = boot_window_s, + storage_source = storage_source, + reason = reason or 'config_changed', + }) + else + svc:debug('monitor_storage_config_unchanged', { + boot_records = boot_max, + ring_records = ring_max, + boot_seconds = boot_window_s, + storage_source = storage_source, + reason = reason or 'config_changed', + }) + end + return changed + end + + local function store_record(rec) + next_record_id = next_record_id + 1 + rec.id = next_record_id + counts.logs = counts.logs + 1 + period.logs = period.logs + 1 + local appended, evicted = ring_append(ring, rec, ring_max) + if appended then + counts.stored = counts.stored + 1 + period.stored = period.stored + 1 + if evicted and evicted > 0 then counts.ring_evicted = counts.ring_evicted + evicted end + else + counts.dropped = counts.dropped + 1 + period.dropped = period.dropped + 1 + end + local age = rec.mono - start_mono + local capture_boot = boot_max > 0 and (boot_window_s <= 0 or age <= boot_window_s) + if capture_boot and #boot < boot_max then boot[#boot + 1] = rec end + local serialised = nil + for id, follower in pairs(followers) do + if record_matches(rec, follower.filter) then + serialised = serialised or serialise_record(rec) + local ok = queue.try_send_now(follower.feed._tx, { kind = 'log', replay = false, record = serialised }) + if ok ~= true then follower.feed.dropped = (follower.feed.dropped or 0) + 1 end + end + end + end + + local function note_received(category, service) + counts.received = counts.received + 1 + period.received = period.received + 1 + inc_count(period.received_by_category, category or 'unknown') + inc_count(period.received_by_service, service or 'unknown') + end + + local function note_printed() + counts.printed = counts.printed + 1 + period.printed = period.printed + 1 + end + + local function note_suppressed(category, service) + counts.suppressed = counts.suppressed + 1 + period.suppressed = period.suppressed + 1 + inc_count(period.suppressed_by_category, category or 'unknown') + inc_count(period.suppressed_by_service, service or 'unknown') + end + + local function maybe_summary() + local now = (runtime.now and runtime.now()) or 0 + if summary_period_s <= 0 or now - last_summary < summary_period_s then return end + local elapsed = now - last_summary + last_summary = now + if period.received > 0 or period.suppressed > 0 or period.dropped > 0 then + write_line(string.format('%s STA %-10s %-12s +%.0fs received=%d printed=%d stored=%d suppressed=%d dropped=%d', + fmt_time(now), name, 'summary', elapsed, period.received, period.printed, period.stored, period.suppressed, period.dropped)) + if period.suppressed > 0 then + write_line(string.format('%s STA %-10s %-12s suppressed_by_category=%s suppressed_by_service=%s', + fmt_time(now), name, 'summary', top_counts(period.suppressed_by_category, 6), top_counts(period.suppressed_by_service, 6))) + end + end + period = new_period_counts() + publish_status() + end + + local endpoints = {} + local function add_endpoint(label, method, opts) + local ep, berr = bus_cleanup.bind(conn, monitor_rpc_topic(method, 'main'), opts or { queue_len = 32, full = 'reject_newest' }) + if not ep then error(berr or ('monitor endpoint bind failed: ' .. tostring(method)), 0) end + endpoints[#endpoints + 1] = { label = label, method = method, ep = ep } + end + add_endpoint('rpc_query_logs', 'query-logs') + add_endpoint('rpc_follow_logs', 'follow-logs') + add_endpoint('rpc_set_profile', 'set-profile') + + local cfg_watch = nil + if conn then + local werr + cfg_watch, werr = config_watch.open(conn, name, { + queue_len = 4, + full = 'reject_newest', + changed_kind = 'monitor_config_changed', + closed_kind = 'monitor_config_closed', + }) + if not cfg_watch then error(werr or 'monitor config watch failed', 2) end + end + + local subscriptions = {} + local subscribed_topics = {} + local function add_sub(label, plane, topic, opts) + subscriptions[#subscriptions + 1] = { + label = label, + plane = plane, + topic = topic, + sub = conn:subscribe(topic, opts or { queue_len = 500, full = 'drop_oldest' }), + } + subscribed_topics[#subscribed_topics + 1] = topic_to_string(topic) + end + + if raw then + add_sub('canonical_raw', 'canonical', { 'obs', OBS_VER, '#' }, { queue_len = 500, full = 'drop_oldest' }) + else + add_sub('canonical_event_log', 'canonical', { 'obs', OBS_VER, '+', 'event', 'log' }, { queue_len = 500, full = 'drop_oldest' }) + add_sub('canonical_direct_log', 'canonical', { 'obs', OBS_VER, '+', 'log', '#' }, { queue_len = 500, full = 'drop_oldest' }) + end + if legacy_detect then + add_sub('legacy', 'legacy', { 'obs', '#' }, { queue_len = 200, full = 'drop_oldest' }) + end + + local subscribed = table.concat(subscribed_topics, ',') + publish_status({ subscribed = subscribed }) + + write_line(string.format('%s STA %-10s %-12s profile=%s min_level=%s subscribed=%s boot=%d ring=%d storage=%s', + fmt_time(start_mono), name, 'start', profile, min_level, subscribed, boot_max, ring_max, storage_source)) + + local canonical_seen = {} + local legacy_count = {} + local LEGACY_WARN_THRESHOLD = 2 + local legacy_to_canonical_kind = { log = 'event', event = 'event', state = 'metric' } + + while true do + local choices = {} + for i, entry in ipairs(subscriptions) do + choices['sub:' .. entry.label] = entry.sub:recv_op():wrap(function(msg, err) return 'sub', i, msg, err end) + end + for i, entry in ipairs(endpoints) do + choices['rpc:' .. entry.label] = entry.ep:recv_op():wrap(function(req, err) return 'rpc', i, req, err end) + end + if cfg_watch then + choices['cfg:monitor'] = cfg_watch:recv_op():wrap(function(ev) return 'config', 0, ev, nil end) + end + local _, source, idx, msg, err = perform(named_choice(choices)) + maybe_summary() + if source == 'config' then + local ev = msg + if type(ev) == 'table' and ev.kind == 'monitor_config_changed' then + apply_storage_config(ev.raw, 'config_changed') + elseif type(ev) == 'table' and ev.kind == 'monitor_config_closed' then + svc:warn('monitor_config_watch_closed', { summary = 'monitor config watch closed', err = ev.err }) + end + elseif source == 'rpc' then + local endpoint = idx and endpoints[idx] or nil + if not endpoint then + write_line(string.format('%s STA %-10s %-12s monitor rpc dispatch failed', fmt_time(), name, 'stop')) + break + end + local req = msg + if req == nil then + write_line(string.format('%s STA %-10s %-12s %s endpoint ended: %s', fmt_time(), name, 'stop', tostring(endpoint.label), tostring(err or 'closed'))) + break + end + local payload = type(req.payload) == 'table' and req.payload or {} + if endpoint.method == 'query-logs' then + req:reply(query_records(payload)) + elseif endpoint.method == 'follow-logs' then + local feed = make_feed(payload) + req:reply({ ok = true, feed = feed, follower_id = feed._id, summary = last_summary_payload or public_summary() }) + elseif endpoint.method == 'set-profile' then + local requested_profile = payload.profile and tostring(payload.profile) or profile + local requested_level = payload.min_level and norm_level(payload.min_level) or nil + if requested_profile == 'raw' then + req:reply({ ok = false, err = 'raw_profile_requires_restart', profile = profile, min_level = min_level }) + else + local changed = false + if requested_profile == 'operator' or requested_profile == 'debug' or requested_profile == 'trace' then + profile = requested_profile + min_level = requested_level or (profile == 'debug' and 'debug' or profile == 'trace' and 'trace' or 'info') + changed = true + elseif requested_level then + min_level = requested_level + changed = true + end + if changed then + publish_status({ profile_changed = true }) + svc:info('monitor_profile_changed', { summary = string.format('monitor profile changed profile=%s min_level=%s', tostring(profile), tostring(min_level)), profile = profile, min_level = min_level }) + req:reply({ ok = true, profile = profile, min_level = min_level }) + else + req:reply({ ok = false, err = 'invalid_profile', profile = profile, min_level = min_level }) + end + end + else + req:reply({ ok = false, err = 'unknown_monitor_method', method = endpoint.method }) + end + else + local entry = idx and subscriptions[idx] or nil + if not entry then + write_line(string.format('%s STA %-10s %-12s monitor subscription dispatch failed', fmt_time(), name, 'stop')) + break + end + if msg == nil then + write_line(string.format('%s STA %-10s %-12s %s subscription ended: %s', + fmt_time(), name, 'stop', tostring(entry.label), tostring(err or 'closed'))) + break + end + + if entry.plane == 'canonical' then + local kind, svc_name, lvl_or_name = classify_canonical(msg) + note_received(kind == 'log' and log_category(lvl_or_name) or kind, svc_name) + if type(svc_name) == 'string' and type(kind) == 'string' then + canonical_seen[svc_name] = canonical_seen[svc_name] or {} + canonical_seen[svc_name][kind] = true + end + local rec = normalise_log_record(msg, 'canonical') + if rec then + store_record(rec) + if raw or level_enabled(rec.level, min_level) then + write_line(format_log_line(rec)) + note_printed() + else + note_suppressed(log_category(rec.level), rec.service) + end + elseif raw then + write_line(format_canonical_line(msg)) + note_printed() + else + note_suppressed(kind or 'obs', svc_name) + end + elseif entry.plane == 'legacy' then + local topic = msg.topic or {} + local kind, svc_name = topic[2], topic[3] + note_received(kind == 'log' and 'legacy_log' or ('legacy_' .. tostring(kind or 'obs')), svc_name) + if kind ~= OBS_VER then + local rec = normalise_log_record(msg, 'legacy') + -- service_base dual-publishes legacy and canonical logs. Operator/debug + -- profiles therefore ignore legacy log payloads to avoid duplicate lines; + -- raw mode keeps the old firehose behaviour. + if rec then + if raw then + write_line(format_log_line(rec)) + note_printed() + else + note_suppressed('legacy_log', rec.service) + end + elseif legacy_detect then + if legacy_to_canonical_kind[kind] then + local can_kind = legacy_to_canonical_kind[kind] + local svc_seen = canonical_seen[svc_name] + if not (svc_seen and svc_seen[can_kind]) then + legacy_count[svc_name] = legacy_count[svc_name] or {} + local count = (legacy_count[svc_name][kind] or 0) + 1 + legacy_count[svc_name][kind] = count + if count >= LEGACY_WARN_THRESHOLD then + write_line(format_legacy_warn(msg, 'legacy-only')) + note_printed() + else + note_suppressed('legacy_probe', svc_name) + end + else + note_suppressed('legacy_duplicate', svc_name) + end + else + write_line(format_legacy_warn(msg, 'unknown-endpoint')) + note_printed() + end + else + note_suppressed('legacy_' .. tostring(kind or 'obs'), svc_name) + end + end + end + end + end +end + +return M diff --git a/src/services/net.lua b/src/services/net.lua index b6b0d376..159a2949 100644 --- a/src/services/net.lua +++ b/src/services/net.lua @@ -1,1217 +1,30 @@ -local file = require 'fibers.stream.file' -local fiber = require "fibers.fiber" -local sleep = require "fibers.sleep" -local queue = require "fibers.queue" -local channel = require "fibers.channel" -local cond = require "fibers.cond" -local op = require "fibers.op" -local sc = require "fibers.utils.syscall" -local context = require "fibers.context" -local cjson = require "cjson.safe" -local log = require "services.log" -local shaping = require "services.net.shaping" -local speedtest = require "services.net.speedtest" -local new_msg = require "bus".new_msg --- local cursor = uci.cursor("/tmp/test", "/tmp/.uci") -- runtime-safe! - --- top-level service -local net_service = { - name = 'net' +-- services/net.lua +-- Public entry point for the NET service. + +local M = { + service = require 'services.net.service', + config = require 'services.net.config', + model = require 'services.net.model', + topics = require 'services.net.topics', + projection = require 'services.net.projection', + publisher = require 'services.net.publisher', + events = require 'services.net.events', + backpressure = require 'services.net.backpressure', + generation = require 'services.net.generation', + apply_runtime = require 'services.net.apply_runtime', + hal_client = require 'services.net.hal_client', + observer_manager = require 'services.net.observer_manager', + wan_manager = require 'services.net.wan_manager', + wan_policy = require 'services.net.wan_policy', + drift = require 'services.net.drift', } -net_service.__index = net_service -------------------------------------------------------- --- Constants -local tracking_ips = { "8.8.8.8", "1.1.1.1" } -local BUS_TIMEOUT = 10 -------------------------------------------------------- --- Helper functions - -local function add_to_uci_list(main, section_name, list_name, list_value) - -- Read current networks assigned to the zone - local uci_get_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'get' }, - { main, section_name, list_name } - )) - local ret, ctx_err = uci_get_sub:next_msg_with_context(context.with_timeout(net_service.ctx, BUS_TIMEOUT)) - uci_get_sub:unsubscribe() - if ctx_err then - log.error(string.format( - "%s - %s: Get (%s, %s, %s) failed, reason: %s", - net_service.ctx:value("service_name"), - net_service.ctx:value("fiber_name"), - main, - section_name, - list_name, - ctx_err - )) - end - local list_elements = ret.payload.result - - -- Convert to table if necessary - local network_list = {} - if type(list_elements) == "string" then - network_list = { list_elements } - elseif type(list_elements) == "table" then - network_list = list_elements - end - - -- Add this value if not already included - local already_present = false - for _, v in ipairs(network_list) do - if v == list_value then - already_present = true - break - end - end - - if not already_present then - table.insert(network_list, list_value) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { main, section_name, list_name, network_list } - )) - log.debug("NET: Added", list_value, "to", main, section_name, list_name) - end -end - -local function round(num, numDecimalPlaces) - local mult = 10 ^ (numDecimalPlaces or 0) - return math.floor(num * mult + 0.5) / mult -end -local function make_counter() - local count = 10 - return { - next = function() - count = count + 1 - return count - end, - reset = function() - count = 10 - end - } -end - -local metric_counter = make_counter() - --- Channel definitions -local config_channel = channel.new() -- For config updates -local interface_channel = channel.new() -- For gsm/interface mappings -local speedtest_result_channel = channel.new() -- For speedtest requests -local wan_status_channel = channel.new() -- For wan status supdates -local modem_on_connected_channel = channel.new() -- For wan status supdates -local report_period_channel = channel.new() -- For config updates - --- Queue definitions -local speedtest_queue = queue.new() -- Unbounded queue for holding speedtest requests -local shaping_queue = queue.new() - ---- Conditional variable definitions -local config_signal = cond.new() -- Conditional varibale to signal inital config - -local function config_receiver(ctx) - log.trace("NET: Config receiver starting") - local sub = net_service.conn:subscribe({ "config", "net" }) - while not ctx:err() do - op.choice( - sub:next_msg_op():wrap(function(msg, err) - if err then - log.error("NET: Config receive error:", err) - end - log.info("NET: Config Received") - config_channel:put(msg.payload) - end), - ctx:done_op() - ):perform() - end - sub:unsubscribe() -end - -local function interface_listener(ctx) - config_signal:wait() -- Block interface listener until initial configs - log.trace("NET: Interface listener starting") - local sub = net_service.conn:subscribe({ "gsm", "modem", "+", "interface" }) - while not ctx:err() do - op.choice( - sub:next_msg_op():wrap(function(msg, err) - if err then - log.error("NET: Interface listen error:", err) - else - -- Extract modem_id from topic gsm/modem//interface - local modem_id = msg.topic[3] - interface_channel:put { - modem_id = modem_id, - interface = msg.payload - } - end - end), - ctx:done_op() - ):perform() - end - sub:unsubscribe() -end - -local function set_firewall_base_config(fw_cfg) - log.info("NET: Applying base firewall config") - - -- Set default policies - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "firewall", "defaults", "defaults" } - )) - for k, v in pairs(fw_cfg.defaults or {}) do - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "firewall", "defaults", k, v } - )) - end - -- Zones - for _, zone in ipairs(fw_cfg.zones or {}) do - local id = zone.config.name - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "firewall", id, "zone" } - )) - for k, v in pairs(zone.config) do - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "firewall", id, k, v } - )) - end - for _, fwrule in ipairs(zone.forwarding or {}) do - local forwarding_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'add' }, - { "firewall", "forwarding" } - )) - local ret = forwarding_sub:next_msg_with_context(context.with_timeout(net_service.ctx, BUS_TIMEOUT)) - forwarding_sub:unsubscribe() - local fw_id, err = ret.payload.result, ret.payload.err - if err then - log.error("NET: Failed to get forwarding ID:", err) - return - end - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "firewall", fw_id, "src", zone.config.name } - )) - for k, v in pairs(fwrule) do - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "firewall", fw_id, k, v } - )) - end - end - end - - -- Firewall rules - for _, rule in ipairs(fw_cfg.rules or {}) do - local rule_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'add' }, - { "firewall", "rule" } - )) - local ret = rule_sub:next_msg_with_context(context.with_timeout(net_service.ctx, BUS_TIMEOUT)) - rule_sub:unsubscribe() - local rule_id, err = ret.payload.result, ret.payload.err - if err then - log.error("NET: Failed to get rule ID:", err) - return - end - for k, v in pairs(rule.config) do - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "firewall", rule_id, k, v } - )) - end - end - - log.info("NET: Base firewall configured") -end - -local function set_network_base_config(net_cfg) - log.info("NET: Applying base network config") - - -- Globals - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", "globals", "globals" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", "globals", "ula_prefix", "auto" } - )) - - -- Static Routes - for i, route in ipairs(net_cfg.static_routes or {}) do - local id = "route_" .. i - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", id, "route" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", id, "target", route.target } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", id, "interface", route.interface } - )) - if route.netmask then - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", id, "netmask", route.netmask } - )) - end - if route.gateway then - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", id, "gateway", route.gateway } - )) - end - end - - log.info("NET: Base network configured") -end - -local function set_mwan3_base_config(multiwan_cfg) - log.info("NET: Applying base mwan3 config") - - -- Globals - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "globals", "globals" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "globals", "mmx_mask", "0x3F00" } - )) - - -- HTTPS Rule - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "https", "rule" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "https", "sticky", "1" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "https", "dest_port", "443" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "https", "proto", "tcp" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "https", "use_policy", "def_pol" } - )) - - -- Default Rule - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "default_ipv4", "rule" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "default_ipv4", "dest_ip", "0.0.0.0/0" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "default_ipv4", "use_policy", "def_pol" } - )) - - -- Policy - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "def_pol", "policy" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", "def_pol", "last_resort", "unreachable" } - )) - - log.info("NET: Base mwan3 configured") -end - -local function set_dhcp_base_config(dhcp_cfg) - log.info("NET: Applying base dhcp config") - - -- DHCP domains - for _, domain in ipairs(dhcp_cfg.domains or {}) do - local id_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'add' }, - { "dhcp", "domain" } - )) - local ret = id_sub:next_msg_with_context(context.with_timeout(net_service.ctx, BUS_TIMEOUT)) - id_sub:unsubscribe() - local id, err = ret.payload.result, ret.payload.err - if err then - log.error("NET: Failed to add domain:", err) - else - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "name", domain.name } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "ip", domain.ip } - )) - log.info("NET: Added static DNS", domain.name, "->", domain.ip) - end - end - - log.info("NET: Base dhcp configured") -end - -local function set_network_config(instance) - local net_cfg = instance.cfg - assert(net_cfg.id, "network config must have an 'id'") - local net_id = net_cfg.id - - log.info("NET: Applying network config for:", net_id) - - -- 1. Network interface config - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", net_id, "interface" } - )) - - if net_cfg.type == "local" then - if net_cfg.is_bridge then - local devicename_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'add' }, - { "network", "device" } - )) - local ret = devicename_sub:next_msg_with_context(context.with_timeout(net_service.ctx, BUS_TIMEOUT)) - devicename_sub:unsubscribe() - local devicename, err = ret.payload.result, ret.payload.err - if err then - log.error("NET: Failed to add network device:", err) - return - end - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", devicename, "name", "br-" .. net_id } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", devicename, "type", "bridge" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", devicename, "ports", net_cfg.interfaces or {} } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", net_id, "device", "br-" .. net_id } - )) - else - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", net_id, "device", net_cfg.interfaces[1] } - )) - end - end - - if net_cfg.type == "backhaul" then - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", net_id, "peerdns", "0" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", net_id, "device", net_cfg.interfaces[1] } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", net_id, "metric", metric_counter.next() } - )) - end - - if net_cfg.ipv4.proto == "dhcp" then - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", net_id, "proto", "dhcp" } - )) - elseif net_cfg.ipv4.proto == "static" then - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", net_id, "proto", "static" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", net_id, "ipaddr", net_cfg.ipv4.ip_address } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", net_id, "netmask", net_cfg.ipv4.netmask } - )) - if net_cfg.ipv4.gateway then - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "network", net_id, "gateway", net_cfg.ipv4.gateway } - )) - end - end - - -- 2. DHCP - if net_cfg.type=='local' and net_cfg.dhcp_server then - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", net_id, "dhcp" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", net_id, "interface", net_id } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", net_id, "start", net_cfg.dhcp_server.range_skip or "10" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", net_id, "limit", net_cfg.dhcp_server.range_extent or "240" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", net_id, "leasetime", net_cfg.dhcp_server.lease_time or "12h" } - )) - elseif net_cfg.type=='backhaul' then - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", net_id, "dhcp" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", net_id, "interface", net_id } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", net_id, "ignore", "1" } - )) - end - - -- connects DHCP to instance and DNS to interface - if instance.dns_id then - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", net_id, "instance", instance.dns_id } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", instance.dns_id, "local", "/" .. net_id .. "/" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", instance.dns_id, "domain", net_id } - )) - add_to_uci_list("dhcp", instance.dns_id, "listen_address", net_cfg.ipv4.ip_address) - -- add_to_uci_list("dhcp", instance.dns_id, "interface", net_id) - end - - -- 3. Associate this network with an existing firewall zone - if net_cfg.firewall and net_cfg.firewall.zone then - local zone_id = net_cfg.firewall.zone - - add_to_uci_list("firewall", zone_id, "network", net_id) - end - - -- 4. Add MWAN3 configuration - if net_cfg.type == 'backhaul' then - -- first, add the interface - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", net_id, "interface" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", net_id, "enabled", 1 } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", net_id, "interval", 1 } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", net_id, "up", 1 } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", net_id, "down", 2 } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", net_id, "family", "ipv4" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", net_id, "track_ip", tracking_ips } - )) - -- now add the member - local member_id = net_id .. "_member" - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", member_id, "member" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", member_id, "interface", net_id } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", member_id, "metric", instance.speed and net_cfg.multiwan.metric or 99 } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", member_id, "weight", instance.speed and instance.speed * 10 or 1 } - )) - -- now add member to policy - add_to_uci_list("mwan3", "def_pol", "use_member", member_id) - end - log.info("NET: Network config applied successfully for:", net_id) -end - -local function set_network_speed(instance) - local net_cfg = instance.cfg - local net_id = net_cfg.id - - log.info("NET: Applying network speed for:", net_id) - - -- now add the member - local member_id = net_id .. "_member" - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "mwan3", member_id, "weight", instance.speed and round(instance.speed * 10) or 1 } - )) - -- now add member to policy - add_to_uci_list("mwan3", "def_pol", "use_member", member_id) - log.debug("NET: Committing changes for: mwan3") - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'commit' }, - { "mwan3" } - )) - log.info("NET: Speed config applied successfully for:", net_id) -end -local dnsmasq_instances = {} -- maps host filter keys to instance names -local dnsmasq_counter = 0 - -local function get_dnsmasq_id(default_hosts) - local id - - if not default_hosts or #default_hosts == 0 then - default_hosts = {} - id = "standard" - else - -- Generate a key: sorted to canonicalise combinations - table.sort(default_hosts) - id = table.concat(default_hosts, "_") - end - - if dnsmasq_instances[id] then - return id - end - - dnsmasq_counter = dnsmasq_counter + 1 - dnsmasq_instances[id] = true - - -- Create dnsmasq config block - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "dnsmasq" } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "domainneeded", '1' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "boguspriv", '1' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "localise_queries", '1' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "rebind_protection", '1' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "rebind_localhost", '1' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "expandhosts", '1' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "nonegcache", '0' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "cachesize", '1000' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "authoritative", '1' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "readethers", '1' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "leasefile", "/tmp/dhcp.leases." .. id } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "resolvfile", '/tmp/resolv.conf.d/resolv.conf.auto' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "nonwildcard", '1' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "localservice", '1' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "port", '53' } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "server", { "8.8.8.8", "1.1.1.1" } } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "noresolv", "1" } - )) - - -- Add any blocklist host files - if #default_hosts > 0 then - local path_list = {} - for _, list in ipairs(default_hosts) do - table.insert(path_list, "/etc/jng/hosts/" .. list) - end - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set' }, - { "dhcp", id, "addnhosts", path_list } - )) - end - return id +function M.start(conn, opts) + return M.service.start(conn, opts) end -local function uci_manager(ctx) - log.trace("NET: UCI manager starting") - - local uci_sub = net_service.conn:subscribe({ 'hal', 'capability', 'uci', '1' }) - uci_sub:next_msg_with_context(ctx) -- wait for uci capability to appear - uci_sub:unsubscribe() - - -- setup restart policies for each config - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set_restart_actions' }, - { "network", { { "/etc/init.d/network", "reload" } } } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set_restart_actions' }, - { "firewall", { { "/etc/init.d/firewall", "restart" } } } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set_restart_actions' }, - { "dhcp", { - { "/etc/init.d/dnsmasq", "restart" }, - { "/etc/init.d/odhcpd", "restart" } - } } - )) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'set_restart_actions' }, - { "mwan3", { { "/etc/init.d/mwan3", "restart" } } } - )) - - local networks = {} - local resolved_interfaces = {} -- key = modem_id or ifname, value = resolved interface string - - local function on_config(cfg) - local start = sc.monotime() - -- cleanup - local areas = { "network", "firewall", "dhcp", "mwan3" } - for _, area in ipairs(areas) do - -- Delete all sections in each UCI config area - local foreach_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'foreach' }, - { area, nil, - function(cursor, s) - cursor:delete(area, s[".name"]) - end - } - )) - local ret = foreach_sub:next_msg_with_context(context.with_timeout(net_service.ctx, BUS_TIMEOUT)) - foreach_sub:unsubscribe() - if ret.payload.err then - log.error("NET: Failed to clear UCI area", area, ":", ret.payload.err) - end - end - metric_counter.reset() - -- config application - set_firewall_base_config(cfg.firewall) - set_network_base_config(cfg.network) - set_mwan3_base_config(cfg.multiwan) -- NEED TO CREATE MULTIWAN GLOBALS SECTION - set_dhcp_base_config(cfg.dhcp) - - for _, net_cfg in ipairs(cfg.network or {}) do - local net_id = net_cfg.id - networks[net_id] = { - cfg = net_cfg, - dns_id = net_cfg.type == "local" and - get_dnsmasq_id(net_cfg.dns_server and net_cfg.dns_server.default_hosts or {}), - status = networks[net_id] and networks[net_id].status or "offline", - speed = networks[net_id] and networks[net_id].speed, - resolved = net_cfg.interfaces and net_cfg.interfaces[1] and true or false - } - local modem_id = net_cfg.modem_id - local iface = net_cfg.interfaces and net_cfg.interfaces[1] - - if modem_id and not resolved_interfaces[modem_id] then - -- Modem-dependent and unresolved - log.debug("NET: Deferring config for", net_id, "— waiting for modem", modem_id) - elseif iface or (modem_id and resolved_interfaces[modem_id]) then - -- Ready to apply - if modem_id then net_cfg.interfaces = { resolved_interfaces[modem_id] } end - set_network_config(networks[net_id]) - else - log.warn("NET: Network", net_id, "has no usable interface or modem_id") - end - end - for _, area in ipairs(areas) do - log.debug("NET: Committing changes for:", area) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'commit' }, - { area } - )) - end - print("That took:", sc.monotime() - start) - - for _, net_cfg in ipairs(cfg.network or {}) do - shaping_queue:put(net_cfg) - end - -- If this is the initial config, signal config applied - config_signal:signal() - - local report_period = cfg.report_period - if report_period ~= nil then - report_period_channel:put(report_period) - end - end - - local function on_wan_status(msg) - local network, status = msg.network, msg.status - if not networks[network] then - networks[network] = { status = status } - log.info("NET: Status for unknown network", network, "set to", status, ", skipping speedtest") - return - elseif (not networks[network].cfg) or (not networks[network].cfg.interfaces) then - networks[network].status = status - log.info("NET: Status for network", network, "with unknown interface set to", status, ", skipping speedtest") - return - end - local old_status = networks[network].status - networks[network].status = status - if old_status ~= status then - log.debug("NET: WAN state change", network, status) - - -- Trigger speedtest only on new connections - if status == "online" then - local SPEEDTEST_CACHE_DURATION = 600 - if not networks[network].speed or sc.monotime() - networks[network].speed_time > SPEEDTEST_CACHE_DURATION then - local interface = networks[network].cfg.interfaces[1] - log.info("NET: network", network, ": newly online, scheduling speedtest") - speedtest_queue:put({ network = network, interface = interface }) - else - log.info("NET: network", network, ": has recent speed, skipping speedtest") - end - end - end - end - - local function on_interface(msg) - local areas = { "network", "firewall", "dhcp", "mwan3" } - local modem_id = msg.modem_id - local iface = msg.interface - if not modem_id or not iface then return end - - -- Check if this interface was already resolved and configured - if resolved_interfaces[modem_id] == iface then - log.debug("NET: Interface", iface, "already resolved for modem", modem_id) - return - end - - resolved_interfaces[modem_id] = iface - log.info("NET: Interface", iface, "resolved for modem", modem_id) - - -- Check pending networks that depend on this modem - for net_id, net in pairs(networks) do - if net.cfg and net.cfg.modem_id == modem_id then - net.cfg.interfaces = { iface } - set_network_config(net) - log.info("NET: Applied deferred network config for", net_id) - for _, area in ipairs(areas) do - log.debug("NET: Committing changes for:", area) - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'commit' }, - { area } - )) - end - end - end - end - - local function on_speedtest_result(result) - networks[result.network].speed = result.speed - networks[result.network].speed_time = sc.monotime() - set_network_speed(networks[result.network]) - net_service.conn:publish(new_msg({ "net", result.network, "download_speed" }, result.speed)) - end - - local function on_modem_connected(modem_id) - for _, net in pairs(networks) do - if net.cfg and net.cfg.modem_id == modem_id then - local ifup_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'ifup' }, - { net.cfg.id } - )) - local ret, ctx_err = ifup_sub:next_msg_with_context(context.with_timeout(net_service.ctx, BUS_TIMEOUT)) - if ctx_err or ret.payload.err then - log.error(string.format( - "%s - %s: Failed to bring up network %s: %s", - net_service.ctx:value("service_name"), - net_service.ctx:value("fiber_name"), - net.cfg.id, - ctx_err or ret.payload.err - )) - return - else - log.trace(string.format( - "%s - %s: Successfully brought up network %s", - net_service.ctx:value("service_name"), - net_service.ctx:value("fiber_name"), - net.cfg.id - )) - end - ifup_sub:unsubscribe() - end - end - end - - ---Periodic gathering a publish of net information - local function report_metrics(ctx) - local function read_interface_file_numeric(id, stat_name) - local path = "/sys/class/net/" .. id .. "/statistics/" .. stat_name - local f = file.open(path, "r") - if not f then return nil, "Failed to open interface file: " .. path end - - f:seek(0) -- Rewind to beginning - local metric = tonumber(f:read_all_chars()) - f:close() - return metric - end - - local report_period = report_period_channel:get() - while not ctx:err() do - local net_metrics = {} - for net_id, net in pairs(networks) do - if net.cfg and net.cfg.interfaces ~= nil and #net.cfg.interfaces > 0 then - local iface_metrics = {} - - local rx_bytes, e = read_interface_file_numeric(net.cfg.interfaces[1], "rx_bytes") - if e == nil then iface_metrics.rx_bytes = rx_bytes end - - local rx_packets, e = read_interface_file_numeric(net.cfg.interfaces[1], "rx_packets") - if e == nil then iface_metrics.rx_packets = rx_packets end - - local rx_dropped, e = read_interface_file_numeric(net.cfg.interfaces[1], "rx_dropped") - if e == nil then iface_metrics.rx_dropped = rx_dropped end - - local rx_errors, e = read_interface_file_numeric(net.cfg.interfaces[1], "rx_errors") - if e == nil then iface_metrics.rx_errors = rx_errors end - - local tx_bytes, e = read_interface_file_numeric(net.cfg.interfaces[1], "tx_bytes") - if e == nil then iface_metrics.tx_bytes = tx_bytes end - - local tx_packets, e = read_interface_file_numeric(net.cfg.interfaces[1], "tx_packets") - if e == nil then iface_metrics.tx_packets = tx_packets end - - local tx_dropped, e = read_interface_file_numeric(net.cfg.interfaces[1], "tx_dropped") - if e == nil then iface_metrics.tx_dropped = tx_dropped end - - local tx_errors, e = read_interface_file_numeric(net.cfg.interfaces[1], "tx_errors") - if e == nil then iface_metrics.tx_errors = tx_errors end - - if next(iface_metrics) ~= nil then - net_metrics[net_id] = iface_metrics - end - end - end - - if next(net_metrics) ~= nil then - net_service.conn:publish_multiple({ 'net' }, net_metrics, { retained = true }) - end - - op.choice( - sleep.sleep_op(report_period), - report_period_channel:get_op():wrap(function(new_period) - report_period = new_period - end), - ctx:done_op() - ):perform() - end - end - - fiber.spawn(function() report_metrics(ctx) end) - - while ctx:err() == nil do - op.choice( - config_channel:get_op():wrap(on_config), - interface_channel:get_op():wrap(on_interface), - speedtest_result_channel:get_op():wrap(on_speedtest_result), - wan_status_channel:get_op():wrap(on_wan_status), - modem_on_connected_channel:get_op():wrap(on_modem_connected), - ctx:done_op() - ):perform() - end -end - -local function wan_monitor(ctx) - config_signal:wait() -- Block wan monitor until initial configs - local ubus_active_sub = net_service.conn:subscribe({ 'hal', 'capability', 'ubus', '1' }) - ubus_active_sub:next_msg() -- Block wan monitor until ubus capability is active - - log.trace("NET: WAN monitor starting") - - -- first, we get the initial state of the interfaces, we use command line ubus for consistency with `ubus listen` - local status_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'ubus', '1', 'control', 'call' }, - { "mwan3", "status" } - )) - local status_response, ctx_err = status_sub:next_msg_with_context(context.with_timeout(ctx, BUS_TIMEOUT)) - status_sub:unsubscribe() - - if ctx_err or status_response.payload.err then - local err = ctx_err or status_response.payload.err - log.error("NET: WAN monitor: could not get initial state: " .. err) - return - end - - local res = status_response.payload.result - - for network, data in pairs(res.interfaces or {}) do - local status = data.status == "online" and "online" or "offline" - wan_status_channel:put({ network = network, status = status }) - - net_service.conn:publish(new_msg({ "net", network, "status" }, status)) - net_service.conn:publish(new_msg( - { "net", data.interface, "curr_uptime" }, - status == "online" and os.time() or 0 - )) - end - - -- now we start a continuous loop to monitor for changes in interface state - local listen_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'ubus', '1', 'control', 'listen' }, - { 'hotplug.mwan3' } - )) - local listen_response, ctx_err = listen_sub:next_msg_with_context(context.with_timeout(ctx, BUS_TIMEOUT)) - listen_sub:unsubscribe() - if ctx_err or listen_response.payload.err then - local err = ctx_err or listen_response.payload.err - log.error("NET: ubus listen failed:", err) - return - end - local stream_id = listen_response.payload.result.stream_id - local hotplug_sub = net_service.conn:subscribe( - { 'hal', 'capability', 'ubus', '1', 'info', 'stream', stream_id } - ) - local stream_end_sub = net_service.conn:subscribe( - { 'hal', 'capability', 'ubus', '1', 'info', 'stream', stream_id, 'closed' } - ) - - local process_ended = false - - while not ctx:err() and not process_ended do - op.choice( - hotplug_sub:next_msg_op():wrap(function(msg) - local event = msg.payload - if not event then return end - local data = event["hotplug.mwan3"] - log.debug("MWAN3 hotplug event received!") - - local status = data.action == "connected" and "online" or "offline" - wan_status_channel:put({ - network = data.interface, - status = status - }) - net_service.conn:publish(new_msg({ "net", data.interface, "status" }, status)) - net_service.conn:publish(new_msg( - { "net", data.interface, "curr_uptime" }, - status == "online" and os.time() or 0 - )) - end), - stream_end_sub:next_msg_op():wrap(function(stream_ended) - if stream_ended.payload then - process_ended = true - end - end), - ctx:done_op():wrap(function() - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'ubus', '1', 'control', 'stop_stream' }, - { stream_id } - )) - end) - ):perform() - end -end - -local function speedtest_worker(ctx) - log.trace("NET: Speedtest worker starting") - while not ctx:err() do - log.trace("NET: Speedtest worker waiting for jobs") - local msg = speedtest_queue:get() - -- halt any config restarts from happening during speedtest - local halt_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'halt_restarts' }, - {} - )) - halt_sub:next_msg_with_context(ctx) - halt_sub:unsubscribe() - local results, err = speedtest.run(ctx, msg.network, msg.interface) - -- allow config restarts to take place - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'continue_restarts' }, - {} - )) - if err then - log.error("NET:", "Speedtest error:", err) - else - log.info(string.format( - "Speedtest complete for network: %s. %.2f Mbps, data used: %.2f MB in %.2f Secs", - msg.network, results.peak, results.data, results.time - )) - speedtest_result_channel:put({ - network = msg.network, - speed = results.peak - }) - end - end -end - -local function shaping_worker(ctx) - log.trace("NET: Shaping worker starting") - while not ctx:err() do - local net_cfg = shaping_queue:get() - if net_cfg.shaping and net_cfg.interfaces then - local halt_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'halt_restarts' }, - {} - )) - halt_sub:next_msg_with_context(ctx) - halt_sub:unsubscribe() - - log.trace("NET: Shaping worker got config for:", net_cfg.id) - - local ifup_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'ifup' }, - { net_cfg.id } - )) - local ret, ctx_err = ifup_sub:next_msg_with_context(net_service.ctx) - ifup_sub:unsubscribe() - - if ctx_err or ret.payload.err then - log.error(string.format( - "%s - %s: Failed to bring up network %s: %s", - net_service.ctx:value("service_name"), - net_service.ctx:value("fiber_name"), - net_cfg.id, - ctx_err or ret.payload.err - )) - fiber.spawn(function() - sleep.sleep(2) - shaping_queue:put(net_cfg) - end) - else - log.trace(string.format( - "%s - %s: Successfully brought up network %s", - net_service.ctx:value("service_name"), - net_service.ctx:value("fiber_name"), - net_cfg.id - )) - - local iface = (net_cfg.type == "local" and net_cfg.is_bridge) and ("br-" .. net_cfg.id) or net_cfg.interfaces[1] - local device_status_sub = net_service.conn:request(new_msg( - { 'hal', 'capability', 'ubus', '1', 'control', 'call' }, - { "network.device", "status", string.format('{"name":"%s"}', iface) } - )) - local device_ret, device_ctx_err = device_status_sub:next_msg_with_context(ctx) - device_status_sub:unsubscribe() - local info = device_ret and device_ret.payload and device_ret.payload.result - local present = info and info.present - - if (not device_ctx_err) and (not device_ret.err) and present then - shaping.apply(net_cfg) - else - log.warn("NET: Interface " .. iface .. " not yet ready; retrying shaping later") - fiber.spawn(function() - sleep.sleep(2) - shaping_queue:put(net_cfg) - end) - end - end - - net_service.conn:publish(new_msg( - { 'hal', 'capability', 'uci', '1', 'control', 'continue_restarts' }, - {} - )) - end - end -end - -local function modem_state_listener(ctx) - log.trace("NET: Interface listener starting") - local sub = net_service.conn:subscribe({ "gsm", "modem", "+", "state" }) - while not ctx:err() do - op.choice( - sub:next_msg_op():wrap(function(msg, err) - if err then - log.error("NET: Interface listen error:", err) - else - local payload = msg.payload - if payload and payload.prev_state ~= "connected" and payload.curr_state == "connected" then - -- Extract modem_id from topic gsm/modem//state - local modem_id = msg.topic[3] - modem_on_connected_channel:put(modem_id) - end - end - end), - ctx:done_op() - ):perform() - end - sub:unsubscribe() -end - -function net_service:start(ctx, conn) - log.trace("Starting NET Service") - self.ctx = ctx - self.conn = conn - - -- Spawn core components - fiber.spawn(function() config_receiver(ctx) end) - fiber.spawn(function() interface_listener(ctx) end) - fiber.spawn(function() uci_manager(ctx) end) - fiber.spawn(function() wan_monitor(ctx) end) - fiber.spawn(function() speedtest_worker(ctx) end) - fiber.spawn(function() shaping_worker(ctx) end) - fiber.spawn(function() modem_state_listener(ctx) end) +function M.run(scope, params) + return M.service.run(scope, params) end -return net_service +return M diff --git a/src/services/net/apply_runtime.lua b/src/services/net/apply_runtime.lua new file mode 100644 index 00000000..c149f98c --- /dev/null +++ b/src/services/net/apply_runtime.lua @@ -0,0 +1,64 @@ +-- services/net/apply_runtime.lua +-- Scoped apply work for NET. + +local fibers = require 'fibers' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' + +local M = {} + +local function make_identity(service_id, generation, apply_id) + return { + kind = 'net_apply_done', + service_id = service_id, + generation = generation, + apply_id = apply_id, + } +end + +function M.start_apply(spec) + if type(spec) ~= 'table' then return nil, 'start_apply spec required' end + local lifetime_scope = spec.lifetime_scope + local hal = spec.hal + local intent = spec.intent + local done_tx = spec.done_tx + local generation = spec.generation + local apply_id = spec.apply_id + local service_id = spec.service_id or 'net' + + if not lifetime_scope then return nil, 'lifetime_scope required' end + if not hal then return nil, 'hal client required' end + if not intent then return nil, 'intent required' end + if not done_tx then return nil, 'done_tx required' end + if type(generation) ~= 'number' then return nil, 'generation required' end + if type(apply_id) ~= 'number' then return nil, 'apply_id required' end + + return scoped_work.start { + lifetime_scope = lifetime_scope, + reaper_scope = spec.reaper_scope or lifetime_scope, + report_scope = spec.report_scope or lifetime_scope, + identity = make_identity(service_id, generation, apply_id), + + run = function () + local result = fibers.perform(hal:apply_intent_op(intent, { + generation = generation, + apply_id = apply_id, + })) + return { + intent_rev = intent.rev, + ok = result and result.ok == true, + result = result, + } + end, + + report = function (ev) + return queue.try_admit_required( + done_tx, + ev, + 'net_apply_completion_report_failed' + ) + end, + } +end + +return M diff --git a/src/services/net/backhaul_model.lua b/src/services/net/backhaul_model.lua new file mode 100644 index 00000000..b4a3551e --- /dev/null +++ b/src/services/net/backhaul_model.lua @@ -0,0 +1,243 @@ +-- services/net/backhaul_model.lua +-- +-- Pure reducer from HAL-observed host facts and GSM uplink sources into NET's +-- product-level backhaul semantics. OpenWrt/mwan3 vocabulary is confined to +-- HAL-observed provenance; policy and UI should consume the semantic fields in +-- this model. + +local tablex = require 'shared.table' + +local M = {} +local copy = tablex.deep_copy + +local function is_table(v) return type(v) == 'table' end + +local function observed_multiwan(snapshot) + local observed = snapshot and snapshot.observed or nil + local mw = observed and observed.snapshot and observed.snapshot.multiwan or nil + if not is_table(mw) then mw = observed and observed.multiwan or nil end + return is_table(mw) and mw or nil +end + +local function status_by_interface(snapshot, iface) + if type(iface) ~= 'string' or iface == '' then return nil end + local mw = observed_multiwan(snapshot) + if not mw then return nil end + local by_sem = mw.interfaces_by_semantic + if is_table(by_sem) and is_table(by_sem[iface]) then return by_sem[iface] end + local ifaces = mw.interfaces + if is_table(ifaces) and is_table(ifaces[iface]) then return ifaces[iface] end + return nil +end + +local function observed_live(snapshot) + local observed = snapshot and snapshot.observed or nil + local live = observed and observed.snapshot and observed.snapshot.live or nil + if not is_table(live) then live = observed and observed.live or nil end + return is_table(live) and live or nil +end + +local function live_interface(snapshot, iface, ifname) + local live = observed_live(snapshot) + local interfaces = live and live.interfaces or nil + if not is_table(interfaces) then return nil end + if type(iface) == 'string' and iface ~= '' and is_table(interfaces[iface]) then return interfaces[iface] end + if type(ifname) == 'string' and ifname ~= '' and is_table(interfaces[ifname]) then return interfaces[ifname] end + return nil +end + +local function valid_address(addr) + if type(addr) ~= 'string' or addr == '' then return nil end + if addr == '0.0.0.0' or addr == '::' then return nil end + return addr +end + +local function first_address(list) + if not is_table(list) then return nil end + for i = 1, #list do + local rec = list[i] + local addr = is_table(rec) and valid_address(rec.address or rec.ip or rec.addr) or valid_address(rec) + if addr then return addr end + end + return nil +end + +local function select_path_address(st, source) + if not is_table(st) then return nil end + local ipv4 = first_address(st.ipv4 or st.addresses_ipv4 or st.address) + or valid_address(st.ipv4_address or st.ipaddr or st.ip or st.address) + if ipv4 then return { family = 'ipv4', address = ipv4, source = source } end + local ipv6 = first_address(st.ipv6 or st.addresses_ipv6 or st['ipv6-address'] or st.ipv6_address) + or valid_address(st.ipv6addr or st.ip6addr) + if ipv6 then return { family = 'ipv6', address = ipv6, source = source } end + return nil +end + +local function path_address(snapshot, iface, ifname, st) + return select_path_address(live_interface(snapshot, iface, ifname), 'hal-network-live') + or select_path_address(st, 'multiwan-status') +end + +local function semantic_state_from_host(st) + if not is_table(st) then return 'unknown', false end + local state = tostring(st.state or st.status or ''):lower() + if state == '' then + if st.usable == true or st.online == true then state = 'online' + elseif st.enabled == false then state = 'disabled' + else state = 'unknown' end + end + local usable = st.usable == true or state == 'online' + if state == 'up' or state == 'connected' then state = 'online'; usable = true end + if state ~= 'online' then usable = false end + return state, usable +end + +local function interface_endpoint(snapshot, iface) + local rec = snapshot and snapshot.interfaces and snapshot.interfaces[iface] or nil + return is_table(rec) and is_table(rec.endpoint) and rec.endpoint or {} +end + +local function member_interface(member, uplink_id) + member = member or {} + return member.interface or uplink_id +end + +local function member_metric(member) + return tonumber(member and member.metric) or 1 +end + +local function vlan_id(vlan) + if type(vlan) == 'number' then return vlan end + if is_table(vlan) then return tonumber(vlan.id) end + return nil +end + +local function first_kind(segment, endpoint) + local kind = (is_table(segment) and segment.kind) or (is_table(endpoint) and endpoint.kind) + if kind == 'wan' or kind == 'wired' then return 'wired' end + return kind or 'wired' +end + +local function link_view(snapshot, iface, member, source_kind) + local segment = snapshot and snapshot.segments and snapshot.segments[iface] or nil + local endpoint = interface_endpoint(snapshot, iface) + local vid = vlan_id(member and member.vlan) or vlan_id(segment and segment.vlan) or vlan_id(endpoint and endpoint.vlan) + local kind = source_kind == 'gsm-uplink' and 'cellular' or first_kind(segment, endpoint) + return { + kind = kind, + segment = iface, + vlan = vid, + } +end + +local function status_source(snapshot, st) + local mw = observed_multiwan(snapshot) or {} + return { + kind = 'host-multiwan', + backend = st and (st.backend or mw.backend) or mw.backend, + tool = st and (st.tool or mw.source) or mw.source, + } +end + +local function gsm_source(member) + local src = member and member.source or nil + if is_table(src) and src.kind == 'gsm-uplink' then return src.id end + return nil +end + +local function gsm_uplink(snapshot, member) + local id = gsm_source(member) + if not id then return nil end + return snapshot and snapshot.sources and snapshot.sources.gsm_uplinks and snapshot.sources.gsm_uplinks[id] or nil +end + +local function reduce_gsm(snapshot, uplink_id, member, opts) + local gsm = gsm_uplink(snapshot, member) + local linux = is_table(gsm and gsm.linux) and gsm.linux or {} + local iface = member_interface(member, uplink_id) + local st = status_by_interface(snapshot, iface) + local state, usable = semantic_state_from_host(st) + local endpoint = interface_endpoint(snapshot, iface) + local ifname = st and st.ifname or linux.ifname or member.device or member.ifname or endpoint.ifname or endpoint.device or endpoint.name + return { + id = tostring(uplink_id), + role = member.role or uplink_id, + interface = iface, + ifname = ifname, + state = state, + usable = usable, + observed = st ~= nil, + metric = member_metric(member), + path_address = path_address(snapshot, iface, ifname, st), + source = { kind = 'gsm-uplink', id = gsm_source(member) }, + status_source = status_source(snapshot, st), + link = link_view(snapshot, iface, member, 'gsm-uplink'), + observed_at = st and (st.observed_at or ((observed_multiwan(snapshot) or {}).observed_at)) + or (is_table(gsm) and (gsm.updated_at or gsm.observed_at) or (opts and opts.now)), + } +end + +local function reduce_host(snapshot, uplink_id, member, opts) + local iface = member_interface(member, uplink_id) + local st = status_by_interface(snapshot, iface) + local state, usable = semantic_state_from_host(st) + local endpoint = interface_endpoint(snapshot, iface) + local ifname = st and st.ifname or member.device or member.ifname or endpoint.ifname or endpoint.device or endpoint.name + return { + id = tostring(uplink_id), + role = member.role or uplink_id, + interface = iface, + ifname = ifname, + state = state, + usable = usable, + observed = st ~= nil, + metric = member_metric(member), + path_address = path_address(snapshot, iface, ifname, st), + uptime_s = st and (tonumber(st.uptime_s) or tonumber(st.uptime)) or nil, + age_s = st and (tonumber(st.age_s) or tonumber(st.age)) or nil, + source = status_source(snapshot, st), + status_source = status_source(snapshot, st), + link = link_view(snapshot, iface, member, 'host-multiwan'), + probes = st and copy(st.probes) or nil, + observed_at = st and (st.observed_at or ((observed_multiwan(snapshot) or {}).observed_at)) or (opts and opts.now), + } +end + +function M.reduce(snapshot, opts) + opts = opts or {} + local out = { + schema = 'devicecode.net.backhaul/1', + state = 'unknown', + observed_at = opts.now, + uplinks = {}, + } + local members = snapshot and snapshot.wan and snapshot.wan.configured_members or {} + local total, online, known = 0, 0, 0 + for uplink_id, member in pairs(members or {}) do + if is_table(member) and member.enabled ~= false and member.disabled ~= true then + local rec + if gsm_source(member) then rec = reduce_gsm(snapshot, uplink_id, member, opts) + else rec = reduce_host(snapshot, uplink_id, member, opts) end + out.uplinks[tostring(uplink_id)] = rec + total = total + 1 + if rec.state ~= 'unknown' then known = known + 1 end + if rec.usable == true and rec.state == 'online' then online = online + 1 end + end + end + if total == 0 then out.state = 'not_configured' + elseif known == 0 then out.state = 'unknown' + elseif online == total then out.state = 'ok' + elseif online > 0 then out.state = 'degraded' + else out.state = 'down' end + out.total = total + out.online = online + return out +end + +M._test = { + observed_multiwan = observed_multiwan, + status_by_interface = status_by_interface, + select_path_address = select_path_address, +} + +return M diff --git a/src/services/net/backpressure.lua b/src/services/net/backpressure.lua new file mode 100644 index 00000000..40857335 --- /dev/null +++ b/src/services/net/backpressure.lua @@ -0,0 +1,33 @@ +-- services/net/backpressure.lua +-- Explicit NET queue/backpressure policy. + +local M = {} + +M.policy = { + config = { + queue_len = 4, + full = 'reject_newest', + }, + + completions = { + queue_len = 32, + full = 'reject_newest', + }, + + observations = { + queue_len = 64, + full = 'drop_oldest', -- latest observed state wins; terminal apply completions never use this queue. + }, + + gsm_uplinks = { + queue_len = 8, + full = 'reject_newest', + }, + + requests = { + queue_len = 16, + full = 'reject_newest', + }, +} + +return M diff --git a/src/services/net/config.lua b/src/services/net/config.lua new file mode 100644 index 00000000..b308329f --- /dev/null +++ b/src/services/net/config.lua @@ -0,0 +1,229 @@ +-- services/net/config.lua +-- Strict cfg/net normalisation boundary for NET. +-- +-- This module intentionally accepts only devicecode.config/net/1. It does not +-- migrate older top-level net/network shapes. The output is product-level +-- intent, not an OpenWrt or HAL backend plan. + +local schema = require 'services.net.schema' + +local segments_dom = require 'services.net.domain.segments' +local interfaces_dom = require 'services.net.domain.interfaces' +local addressing_dom = require 'services.net.domain.addressing' +local firewall_dom = require 'services.net.domain.firewall' +local routing_dom = require 'services.net.domain.routing' +local multiwan_dom = require 'services.net.domain.multiwan' +local vpn_dom = require 'services.net.domain.vpn' +local dns_dom = require 'services.net.domain.dns' +local dhcp_dom = require 'services.net.domain.dhcp' +local diagnostics_dom = require 'services.net.domain.diagnostics' +local config_validate = require 'services.net.config_validate' + +local M = {} + +local SCHEMA = schema.CONFIG_SCHEMA +local INTENT_SCHEMA = schema.INTENT_SCHEMA + +local TOP_LEVEL = { + 'schema', 'version', 'product', 'description', + 'segments', 'interfaces', + 'addressing', 'dns', 'dhcp', + 'vlan_policy', + 'firewall', 'routing', 'wan', 'vpn', 'diagnostics', + 'runtime', 'policies', 'metadata', 'extensions', +} + +local function payload_record(value) + if schema.is_plain_table(value) and value.data ~= nil then + return value.data, value.rev + end + return value, nil +end + +function M.extract_payload(value) + return payload_record(value) +end + +local RUNTIME_FIELDS = { 'apply', 'observe', 'diagnostics', 'backpressure', 'metadata', 'extensions' } + +local function normalise_runtime(v) + local t, err = schema.optional_plain_table(v, { 'net', 'runtime' }) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, RUNTIME_FIELDS, { 'net', 'runtime' }) + if not ok then return nil, ferr end + local out = { + apply = schema.copy(t.apply or {}), + observe = schema.copy(t.observe or {}), + diagnostics = schema.copy(t.diagnostics or {}), + backpressure = schema.copy(t.backpressure or {}), + } + local metadata, merr = schema.copy_optional_table(t.metadata, { 'net', 'runtime', 'metadata' }) + if merr then return nil, merr end + local extensions, eerr = schema.copy_optional_table(t.extensions, { 'net', 'runtime', 'extensions' }) + if eerr then return nil, eerr end + out.metadata = metadata + out.extensions = extensions + return out, nil +end + + +local function normalise_vlan_policy(raw) + local src = raw.vlan_policy + if src == nil and type(raw.policies) == 'table' then src = raw.policies.vlan end + local t, err = schema.copy_table_or_empty(src, { 'net', 'vlan_policy' }) + if not t then return nil, err end + + local reserved = {} + if t.reserved ~= nil then + if not schema.is_plain_table(t.reserved) then return nil, schema.err({ 'net', 'vlan_policy', 'reserved' }, 'must be a map') end + for name, vid in pairs(t.reserved) do + if type(name) ~= 'string' or name == '' then return nil, schema.err({ 'net', 'vlan_policy', 'reserved' }, 'reserved names must be non-empty strings') end + local n, ierr = schema.optional_integer(vid, { 'net', 'vlan_policy', 'reserved', name }) + if ierr then return nil, ierr end + if n < 1 or n > 4094 then return nil, schema.err({ 'net', 'vlan_policy', 'reserved', name }, 'must be in the range 1..4094') end + reserved[name] = n + end + end + + local ranges = {} + if t.ranges ~= nil then + if not schema.is_plain_table(t.ranges) then return nil, schema.err({ 'net', 'vlan_policy', 'ranges' }, 'must be a map') end + for name, r in pairs(t.ranges) do + if not schema.is_plain_table(r) then return nil, schema.err({ 'net', 'vlan_policy', 'ranges', name }, 'must be a table') end + local from, ferr = schema.optional_integer(r.from, { 'net', 'vlan_policy', 'ranges', name, 'from' }) + if ferr then return nil, ferr end + local to, terr = schema.optional_integer(r.to, { 'net', 'vlan_policy', 'ranges', name, 'to' }) + if terr then return nil, terr end + if from == nil or to == nil or from < 1 or to > 4094 or from > to then + return nil, schema.err({ 'net', 'vlan_policy', 'ranges', name }, 'must have valid from/to VLAN ids') + end + ranges[name] = { from = from, to = to } + end + end + + return { reserved = reserved, ranges = ranges, raw = schema.copy(t) }, nil +end + +local function validate_segment_vlans(segments, vlan_policy) + local used = {} + local reserved_by_id = {} + for name, vid in pairs((vlan_policy or {}).reserved or {}) do reserved_by_id[vid] = name end + for id, seg in pairs(segments or {}) do + local vlan = seg.vlan + if vlan then + if vlan.reserved ~= nil then + local vid = (vlan_policy.reserved or {})[vlan.reserved] + if vid == nil then return nil, ('net.segments.%s.vlan.reserved references unknown reserved VLAN %s'):format(id, tostring(vlan.reserved)) end + vlan.id = vid + end + if vlan.id ~= nil then + if used[vlan.id] and used[vlan.id] ~= id then return nil, ('VLAN id %d is used by both %s and %s'):format(vlan.id, used[vlan.id], id) end + used[vlan.id] = id + local reserved_name = reserved_by_id[vlan.id] + if reserved_name ~= nil and vlan.reserved == nil and not seg.protected then + return nil, ('net.segments.%s uses reserved VLAN %d without declaring vlan.reserved'):format(id, vlan.id) + end + end + end + if seg.protected and seg.enabled == false then return nil, ('net.segments.%s is protected and cannot be disabled'):format(id) end + end + return true, nil +end + +local function ensure_shape(intent) + intent.stats = { + segments = schema.count_map(intent.segments), + interfaces = schema.count_map(intent.interfaces), + wan_members = schema.count_map(intent.wan and intent.wan.members), + vpn_tunnels = schema.count_map(intent.vpn and intent.vpn.tunnels), + } + return intent +end + +function M.normalise(value, opts) + opts = opts or {} + local raw, payload_rev = payload_record(value) + local t, err = schema.require_plain_table(raw, { 'cfg', 'net' }) + if not t then return nil, err end + + if t.schema ~= SCHEMA then + return nil, ('cfg/net schema must be %s'):format(SCHEMA) + end + local ok, ferr = schema.check_allowed_fields(t, TOP_LEVEL, { 'cfg', 'net' }) + if not ok then return nil, ferr end + + local rev = opts.rev or payload_rev or 0 + if type(rev) == 'number' then rev = math.floor(rev) end + local version = t.version or schema.DEFAULT_VERSION + if type(version) ~= 'number' or version % 1 ~= 0 or version < 1 then + return nil, 'cfg/net.version must be a positive integer when supplied' + end + + local segments, serr = segments_dom.normalise(t.segments) + if not segments then return nil, serr end + local vlan_policy, vperr = normalise_vlan_policy(t) + if not vlan_policy then return nil, vperr end + local vok, verr = validate_segment_vlans(segments, vlan_policy) + if vok ~= true then return nil, verr end + local interfaces, ierr = interfaces_dom.normalise(t.interfaces) + if not interfaces then return nil, ierr end + local addressing, aerr = addressing_dom.normalise(t.addressing) + if not addressing then return nil, aerr end + local dns, dnserr = dns_dom.normalise(t.dns) + if not dns then return nil, dnserr end + local dhcp, dhcperr = dhcp_dom.normalise(t.dhcp) + if not dhcp then return nil, dhcperr end + local firewall, fwerr = firewall_dom.normalise(t.firewall) + if not firewall then return nil, fwerr end + local routing, rerr = routing_dom.normalise(t.routing) + if not routing then return nil, rerr end + local wan, werr = multiwan_dom.normalise(t.wan) + if not wan then return nil, werr end + local vpn, vperr = vpn_dom.normalise(t.vpn) + if not vpn then return nil, vperr end + local diagnostics, derr = diagnostics_dom.normalise(t.diagnostics) + if not diagnostics then return nil, derr end + local runtime, rterr = normalise_runtime(t.runtime) + if not runtime then return nil, rterr end + local policies, perr = schema.copy_table_or_empty(t.policies, { 'net', 'policies' }) + if not policies then return nil, perr end + policies.vlan = policies.vlan or schema.copy(vlan_policy.raw or {}) + local metadata, merr = schema.copy_optional_table(t.metadata, { 'net', 'metadata' }) + if merr then return nil, merr end + local extensions, eerr = schema.copy_optional_table(t.extensions, { 'net', 'extensions' }) + if eerr then return nil, eerr end + + local intent = ensure_shape({ + schema = INTENT_SCHEMA, + config_schema = SCHEMA, + version = version, + rev = rev, + generation = opts.generation or 0, + product = t.product, + description = t.description, + segments = segments, + vlan_policy = vlan_policy, + interfaces = interfaces, + addressing = addressing, + dns = dns, + dhcp = dhcp, + firewall = firewall, + routing = routing, + wan = wan, + vpn = vpn, + diagnostics = diagnostics, + runtime = runtime, + policies = policies, + metadata = metadata, + extensions = extensions, + }) + + local valid, verr = config_validate.validate(intent) + if valid ~= true then return nil, verr end + return intent, nil +end + +M.SCHEMA = SCHEMA +M.INTENT_SCHEMA = INTENT_SCHEMA + +return M diff --git a/src/services/net/config_validate.lua b/src/services/net/config_validate.lua new file mode 100644 index 00000000..af6ac04c --- /dev/null +++ b/src/services/net/config_validate.lua @@ -0,0 +1,124 @@ +-- services/net/config_validate.lua +-- Cross-domain semantic validation for normalised NET intent. + +local schema = require 'services.net.schema' + +local M = {} + +local function has(t, id) return type(t) == 'table' and t[id] ~= nil end +local function err(path, msg) return schema.err(path, msg) end + +local function check_ref(map, id, path, kind) + if id == nil then return true, nil end + if type(id) ~= 'string' or id == '' then return nil, err(path, kind .. ' reference must be a non-empty string') end + if not has(map, id) then return nil, err(path, 'references unknown ' .. kind .. ' ' .. tostring(id)) end + return true, nil +end + +local function known_l3_ref(intent, id) + return has(intent.interfaces, id) or has(intent.segments, id) +end + +local function check_l3_ref(intent, id, path) + if type(id) ~= 'string' or id == '' then + return nil, err(path, 'interface reference must be a non-empty string') + end + if not known_l3_ref(intent, id) then + return nil, err(path, 'references unknown interface or segment ' .. tostring(id)) + end + return true, nil +end + +local function validate_interfaces(intent) + for id, iface in pairs(intent.interfaces or {}) do + local ok, e = check_ref(intent.segments, iface.segment, { 'net', 'interfaces', id, 'segment' }, 'segment') + if not ok then return nil, e end + for i, seg_id in ipairs(iface.segments or {}) do + ok, e = check_ref(intent.segments, seg_id, { 'net', 'interfaces', id, 'segments', i }, 'segment') + if not ok then return nil, e end + end + if iface.parent ~= nil and has(intent.interfaces, iface.parent) == false then + return nil, err({ 'net', 'interfaces', id, 'parent' }, 'references unknown interface ' .. tostring(iface.parent)) + end + end + return true, nil +end + +local function validate_segments(intent) + local host_sources = intent.dns and intent.dns.host_files and intent.dns.host_files.sources or {} + local zones = intent.firewall and intent.firewall.zones or {} + for id, seg in pairs(intent.segments or {}) do + local zone = seg.firewall and seg.firewall.zone or nil + if zone ~= nil and type(zones) == 'table' and next(zones) ~= nil and not zones[zone] then + return nil, err({ 'net', 'segments', id, 'firewall', 'zone' }, 'references unknown firewall zone ' .. tostring(zone)) + end + for i, source in ipairs((seg.dns and seg.dns.host_files) or {}) do + if type(host_sources) == 'table' and next(host_sources) ~= nil and not host_sources[source] then + return nil, err({ 'net', 'segments', id, 'dns', 'host_files', i }, 'references unknown DNS host file source ' .. tostring(source)) + end + end + end + return true, nil +end + + +local function validate_routing(intent) + if type(intent.interfaces) ~= 'table' or next(intent.interfaces) == nil then return true, nil end + for id, route in pairs((intent.routing and intent.routing.routes) or {}) do + local ok, e = check_l3_ref(intent, route.interface, { 'net', 'routing', 'routes', id, 'interface' }) + if not ok then return nil, e end + end + return true, nil +end + +local function validate_wan(intent) + -- If no interface catalogue is declared, member references are external provider names. + if type(intent.interfaces) ~= 'table' or next(intent.interfaces) == nil then return true, nil end + for id, member in pairs((intent.wan and intent.wan.members) or {}) do + local iface = member.interface + local src = member.source + if not (type(src) == 'table' and src.kind == 'gsm-uplink') then + local ok, e = check_l3_ref(intent, iface, { 'net', 'wan', 'members', id, 'interface' }) + if not ok then return nil, e end + end + end + return true, nil +end + +local function validate_firewall(intent) + local zones = intent.firewall and intent.firewall.zones or {} + if type(zones) ~= 'table' or next(zones) == nil then return true, nil end + for id, pol in pairs((intent.firewall and intent.firewall.policies) or {}) do + if pol.from ~= nil and not zones[pol.from] then + return nil, err({ 'net', 'firewall', 'policies', id, 'from' }, 'references unknown firewall zone ' .. tostring(pol.from)) + end + if pol.to ~= nil and not zones[pol.to] then + return nil, err({ 'net', 'firewall', 'policies', id, 'to' }, 'references unknown firewall zone ' .. tostring(pol.to)) + end + end + return true, nil +end + +local function validate_dhcp(intent) + for id, r in pairs((intent.dhcp and intent.dhcp.reservations) or {}) do + if r.segment ~= nil and not has(intent.segments, r.segment) then + return nil, err({ 'net', 'dhcp', 'reservations', id, 'segment' }, 'references unknown segment ' .. tostring(r.segment)) + end + if r.interface ~= nil and not has(intent.interfaces, r.interface) then + return nil, err({ 'net', 'dhcp', 'reservations', id, 'interface' }, 'references unknown interface ' .. tostring(r.interface)) + end + end + return true, nil +end + +function M.validate(intent) + local ok, e = validate_interfaces(intent); if not ok then return nil, e end + ok, e = validate_segments(intent); if not ok then return nil, e end + ok, e = validate_wan(intent); if not ok then return nil, e end + ok, e = validate_routing(intent); if not ok then return nil, e end + ok, e = validate_firewall(intent); if not ok then return nil, e end + ok, e = validate_dhcp(intent); if not ok then return nil, e end + return true, nil +end + +return M diff --git a/src/services/net/domain/addressing.lua b/src/services/net/domain/addressing.lua new file mode 100644 index 00000000..f09ecff7 --- /dev/null +++ b/src/services/net/domain/addressing.lua @@ -0,0 +1,28 @@ +-- services/net/domain/addressing.lua +-- Global addressing, DHCP and DNS policy normalisation. + +local schema = require 'services.net.schema' + +local M = {} + +local ALLOWED = { + 'ipv4', 'ipv6', 'dhcp', 'dns', 'pools', 'reservations', 'metadata', 'extensions', +} + +function M.normalise(v) + local t, err = schema.optional_plain_table(v, { 'net', 'addressing' }) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, ALLOWED, { 'net', 'addressing' }) + if not ok then return nil, ferr end + local out = { + ipv4 = schema.copy(t.ipv4 or {}), + ipv6 = schema.copy(t.ipv6 or {}), + dhcp = schema.copy(t.dhcp or {}), + dns = schema.copy(t.dns or {}), + pools = schema.copy(t.pools or {}), + reservations = schema.copy(t.reservations or {}), + } + return schema.with_optional_extensions(out, t, { 'net', 'addressing' }) +end + +return M diff --git a/src/services/net/domain/dhcp.lua b/src/services/net/domain/dhcp.lua new file mode 100644 index 00000000..fd458a81 --- /dev/null +++ b/src/services/net/domain/dhcp.lua @@ -0,0 +1,27 @@ +-- services/net/domain/dhcp.lua +-- Product-level DHCP policy intent. + +local schema = require 'services.net.schema' + +local M = {} + +local ALLOWED = { + 'enabled', 'defaults', 'reservations', 'options', 'relays', 'metadata', 'extensions', +} + +function M.normalise(v) + local t, err = schema.optional_plain_table(v, { 'net', 'dhcp' }) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, ALLOWED, { 'net', 'dhcp' }) + if not ok then return nil, ferr end + local out = { + enabled = t.enabled ~= false, + defaults = schema.copy(t.defaults or {}), + reservations = schema.copy(t.reservations or {}), + options = schema.copy(t.options or {}), + relays = schema.copy(t.relays or {}), + } + return schema.with_optional_extensions(out, t, { 'net', 'dhcp' }) +end + +return M diff --git a/src/services/net/domain/diagnostics.lua b/src/services/net/domain/diagnostics.lua new file mode 100644 index 00000000..89e5800f --- /dev/null +++ b/src/services/net/domain/diagnostics.lua @@ -0,0 +1,26 @@ +-- services/net/domain/diagnostics.lua +-- Product-level network diagnostics policy. + +local schema = require 'services.net.schema' + +local M = {} + +local ALLOWED = { + 'probes', 'reflectors', 'schedules', 'limits', 'metadata', 'extensions', +} + +function M.normalise(v) + local t, err = schema.optional_plain_table(v, { 'net', 'diagnostics' }) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, ALLOWED, { 'net', 'diagnostics' }) + if not ok then return nil, ferr end + local out = { + probes = schema.copy(t.probes or {}), + reflectors = schema.copy(t.reflectors or {}), + schedules = schema.copy(t.schedules or {}), + limits = schema.copy(t.limits or {}), + } + return schema.with_optional_extensions(out, t, { 'net', 'diagnostics' }) +end + +return M diff --git a/src/services/net/domain/dns.lua b/src/services/net/domain/dns.lua new file mode 100644 index 00000000..9cba6be2 --- /dev/null +++ b/src/services/net/domain/dns.lua @@ -0,0 +1,33 @@ +-- services/net/domain/dns.lua +-- Product-level DNS policy intent. + +local schema = require 'services.net.schema' + +local M = {} + +local ALLOWED = { + 'enabled', 'upstreams', 'search', 'zones', 'records', 'forwarders', + 'cache', 'security', 'domain', 'host_files', 'metadata', 'extensions', +} + +function M.normalise(v) + local t, err = schema.optional_plain_table(v, { 'net', 'dns' }) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, ALLOWED, { 'net', 'dns' }) + if not ok then return nil, ferr end + local out = { + enabled = t.enabled ~= false, + upstreams = schema.copy(t.upstreams or {}), + search = schema.copy(t.search or {}), + zones = schema.copy(t.zones or {}), + records = schema.copy(t.records or {}), + forwarders = schema.copy(t.forwarders or {}), + cache = schema.copy(t.cache or {}), + security = schema.copy(t.security or {}), + domain = t.domain, + host_files = schema.copy(t.host_files or {}), + } + return schema.with_optional_extensions(out, t, { 'net', 'dns' }) +end + +return M diff --git a/src/services/net/domain/firewall.lua b/src/services/net/domain/firewall.lua new file mode 100644 index 00000000..8c8d2e3b --- /dev/null +++ b/src/services/net/domain/firewall.lua @@ -0,0 +1,30 @@ +-- services/net/domain/firewall.lua +-- Product-level firewall and isolation policy normalisation. + +local schema = require 'services.net.schema' + +local M = {} + +local ALLOWED = { + 'defaults', 'zones', 'policies', 'rules', 'nat', 'port_forwards', + 'isolation', 'metadata', 'extensions', +} + +function M.normalise(v) + local t, err = schema.optional_plain_table(v, { 'net', 'firewall' }) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, ALLOWED, { 'net', 'firewall' }) + if not ok then return nil, ferr end + local out = { + defaults = schema.copy(t.defaults or {}), + zones = schema.copy(t.zones or {}), + policies = schema.copy(t.policies or {}), + rules = schema.copy(t.rules or {}), + nat = schema.copy(t.nat or {}), + port_forwards = schema.copy(t.port_forwards or {}), + isolation = schema.copy(t.isolation or {}), + } + return schema.with_optional_extensions(out, t, { 'net', 'firewall' }) +end + +return M diff --git a/src/services/net/domain/interfaces.lua b/src/services/net/domain/interfaces.lua new file mode 100644 index 00000000..b4c3fc59 --- /dev/null +++ b/src/services/net/domain/interfaces.lua @@ -0,0 +1,60 @@ +-- services/net/domain/interfaces.lua +-- Product-level logical interface normalisation. No OS interface names are required. + +local schema = require 'services.net.schema' + +local M = {} + +local ALLOWED = { + 'id', 'name', 'description', 'kind', 'role', 'enabled', 'segment', + 'segments', 'endpoint', 'parent', 'members', 'mtu', 'mac', 'addressing', + 'policy', 'tags', 'metadata', 'extensions', +} + +function M.normalise_record(id, rec, path) + local t, err = schema.require_plain_table(rec, path) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, ALLOWED, path) + if not ok then return nil, ferr end + if t.id ~= nil and t.id ~= id then return nil, schema.err({ schema.path(path), 'id' }, 'must match the map key') end + + local segment = nil + if t.segment ~= nil then + segment, err = schema.id(t.segment, { schema.path(path), 'segment' }) + if err then return nil, err end + end + local segments, serr = schema.id_list(t.segments, { schema.path(path), 'segments' }) + if serr then return nil, serr end + local members, merr = schema.id_list(t.members, { schema.path(path), 'members' }) + if merr then return nil, merr end + local tags, terr = schema.id_list(t.tags, { schema.path(path), 'tags' }) + if terr then return nil, terr end + local mtu, mtu_err = schema.optional_integer(t.mtu, { schema.path(path), 'mtu' }) + if mtu_err then return nil, mtu_err end + + local out = { + id = id, + name = t.name or id, + description = t.description, + kind = t.kind or 'logical', + role = t.role, + enabled = t.enabled ~= false, + segment = segment, + segments = segments, + endpoint = schema.copy(t.endpoint or {}), + parent = t.parent, + members = members, + mtu = mtu, + mac = t.mac, + addressing = schema.copy(t.addressing or {}), + policy = schema.copy(t.policy or {}), + tags = tags, + } + return schema.with_optional_extensions(out, t, path) +end + +function M.normalise(input) + return schema.map(input, { 'net', 'interfaces' }, M.normalise_record) +end + +return M diff --git a/src/services/net/domain/multiwan.lua b/src/services/net/domain/multiwan.lua new file mode 100644 index 00000000..1f13a4d5 --- /dev/null +++ b/src/services/net/domain/multiwan.lua @@ -0,0 +1,200 @@ +-- services/net/domain/multiwan.lua +-- Product-level WAN and multi-WAN policy intent. + +local schema = require 'services.net.schema' + +local M = {} + +local ALLOWED = { + 'enabled', 'members', 'health', 'runtime', 'failover', + 'load_balancing', 'rules', 'metadata', 'extensions', +} + +local MEMBER_ALLOWED = { + 'id', 'interface', 'source', 'metric', 'mwan_metric', 'weight', 'dynamic_weight', 'family', + 'track_ip', 'health', 'reliability', 'count', 'timeout', 'interval', 'up', 'down', + 'enabled', 'disabled', 'speedtest_url', 'speedtest_duration_s', 'shaping', 'metadata', 'extensions', +} + + +local MEMBER_SHAPING_ALLOWED = { + 'enabled', 'download', 'upload', 'router_traffic', 'control', 'fq_codel', 'metadata', 'extensions', +} +local MEMBER_SHAPING_DIRECTION_ALLOWED = { 'enabled', 'limit', 'metadata', 'extensions' } +local MEMBER_SHAPING_CONTROL_ALLOWED = { 'rate', 'ceil', 'metadata', 'extensions' } + +local function normalise_shaping_direction(v, path) + if v == nil then return nil, nil end + local t, err = schema.require_plain_table(v, path) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, MEMBER_SHAPING_DIRECTION_ALLOWED, path) + if not ok then return nil, ferr end + if type(t.limit) ~= 'string' or t.limit == '' then + return nil, schema.err({ schema.path(path), 'limit' }, 'must be a non-empty rate string') + end + return schema.with_optional_extensions({ enabled = t.enabled ~= false, limit = t.limit }, t, path) +end + +local function normalise_control(v, path) + if v == nil then return {}, nil end + local t, err = schema.require_plain_table(v, path) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, MEMBER_SHAPING_CONTROL_ALLOWED, path) + if not ok then return nil, ferr end + local out = { rate = t.rate, ceil = t.ceil } + return schema.with_optional_extensions(out, t, path) +end + +local function normalise_member_shaping(v, path) + if v == nil then return nil, nil end + local t, err = schema.require_plain_table(v, path) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, MEMBER_SHAPING_ALLOWED, path) + if not ok then return nil, ferr end + if t.router_traffic ~= nil and t.router_traffic ~= 'exempt' then + return nil, schema.err({ schema.path(path), 'router_traffic' }, 'must be exempt') + end + local download, derr = normalise_shaping_direction(t.download, { schema.path(path), 'download' }) + if derr then return nil, derr end + local upload, uerr = normalise_shaping_direction(t.upload, { schema.path(path), 'upload' }) + if uerr then return nil, uerr end + local control, cerr = normalise_control(t.control, { schema.path(path), 'control' }) + if cerr then return nil, cerr end + local out = { + enabled = t.enabled ~= false, + router_traffic = t.router_traffic or 'exempt', + download = download, + upload = upload, + control = control, + fq_codel = schema.copy(t.fq_codel or {}), + } + return schema.with_optional_extensions(out, t, path) +end + +local RULE_ALLOWED = { + 'family', 'proto', 'src_ip', 'dest_ip', 'src_port', 'dest_port', + 'policy', 'sticky', 'timeout', 'enabled', 'disabled', 'metadata', 'extensions', +} + + +local function sorted_keys(t) + local ks = {} + for k in pairs(t or {}) do ks[#ks + 1] = k end + table.sort(ks, function(a, b) return tostring(a) < tostring(b) end) + return ks +end + +local function has_source(member) + return type(member) == 'table' and member.source ~= nil +end + +local function route_metric_keys(t) + local ks = sorted_keys(t) + table.sort(ks, function(a, b) + local source_a, source_b = has_source(t[a]), has_source(t[b]) + if source_a ~= source_b then return not source_a end + return tostring(a) < tostring(b) + end) + return ks +end + +local function clean_rule(id, r, policy_name) + r = schema.copy(r or {}) + local ok, ferr = schema.check_allowed_fields(r, RULE_ALLOWED, { 'net', 'wan', 'rules', id }) + if not ok then return nil, ferr end + if r.enabled == false or r.disabled == true then return nil, nil, true end + if type(id) ~= 'string' or id == '' then return nil, schema.err({ 'net', 'wan', 'rules' }, 'rule ids must be non-empty strings') end + if type(r.policy) ~= 'string' or r.policy == '' then return nil, schema.err({ 'net', 'wan', 'rules', id, 'policy' }, 'must be a non-empty string') end + if r.policy ~= policy_name then return nil, schema.err({ 'net', 'wan', 'rules', id, 'policy' }, 'must match wan.load_balancing.policy') end + local out = { + family = r.family or 'ipv4', + policy = r.policy, + proto = r.proto, + src_ip = r.src_ip, + dest_ip = r.dest_ip, + src_port = r.src_port, + dest_port = r.dest_port, + sticky = r.sticky == true, + timeout = r.timeout, + } + local extended, xerr = schema.with_optional_extensions(out, r, { 'net', 'wan', 'rules', id }) + if not extended then return nil, xerr end + return extended, nil +end + +local function normalise_rules(t) + local src = t.rules or {} + local lb = schema.is_plain_table(t.load_balancing) and t.load_balancing or {} + local policy_name = lb.policy or 'balanced' + if src ~= nil and not schema.is_plain_table(src) then return nil, schema.err({ 'net', 'wan', 'rules' }, 'must be a map') end + local out = {} + for _, id in ipairs(sorted_keys(src)) do + local rec, err, skipped = clean_rule(id, src[id], policy_name) + if err then return nil, err end + if not skipped and rec then out[id] = rec end + end + return out, nil +end + +local function clean_member(id, m, index) + m = schema.copy(m or {}) + local ok, ferr = schema.check_allowed_fields(m, MEMBER_ALLOWED, { 'net', 'wan', 'members', id }) + if not ok then return nil, ferr end + m.id = m.id or id + m.interface = m.interface or id + -- Product-level NET policy uses metric. mwan_metric is accepted only as + -- a legacy config alias and is not carried in NET's normalised model. + m.metric = math.max(1, math.floor(tonumber(m.metric or m.mwan_metric) or 1)) + m.mwan_metric = nil + m.weight = math.max(1, math.floor(tonumber(m.weight) or 1)) + m.route_metric = 10 + index + local shaping, sherr = normalise_member_shaping(m.shaping, { 'net', 'wan', 'members', id, 'shaping' }) + if sherr then return nil, sherr end + m.shaping = shaping + + local src = m.source + if src ~= nil then + if not schema.is_plain_table(src) then return nil, schema.err({ 'net', 'wan', 'members', id, 'source' }, 'must be a table') end + if src.kind ~= 'gsm-uplink' then return nil, schema.err({ 'net', 'wan', 'members', id, 'source', 'kind' }, 'must be gsm-uplink') end + if type(src.id) ~= 'string' or src.id == '' then return nil, schema.err({ 'net', 'wan', 'members', id, 'source', 'id' }, 'must be a non-empty string') end + m.source = { kind = 'gsm-uplink', id = src.id } + end + return m, nil +end + +local function normalise_members(t) + local src = t.members or {} + if src ~= nil and not schema.is_plain_table(src) then return nil, schema.err({ 'net', 'wan', 'members' }, 'must be a map') end + local out = {} + local keys = route_metric_keys(src) + for i, id in ipairs(keys) do + if type(id) ~= 'string' or id == '' then return nil, schema.err({ 'net', 'wan', 'members' }, 'member ids must be non-empty strings') end + local rec, err = clean_member(id, src[id], i) + if not rec then return nil, err end + out[id] = rec + end + return out, nil +end + +function M.normalise(v) + local t, err = schema.optional_plain_table(v, { 'net', 'wan' }) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, ALLOWED, { 'net', 'wan' }) + if not ok then return nil, ferr end + local members, merr = normalise_members(t) + if not members then return nil, merr end + local rules, rerr = normalise_rules(t) + if not rules then return nil, rerr end + local out = { + enabled = t.enabled ~= false, + members = members, + rules = rules, + health = schema.copy(t.health or {}), + runtime = schema.copy(t.runtime or {}), + failover = schema.copy(t.failover or {}), + load_balancing = schema.copy(t.load_balancing or {}), + } + return schema.with_optional_extensions(out, t, { 'net', 'wan' }) +end + +return M diff --git a/src/services/net/domain/routing.lua b/src/services/net/domain/routing.lua new file mode 100644 index 00000000..f2cd22a7 --- /dev/null +++ b/src/services/net/domain/routing.lua @@ -0,0 +1,99 @@ +-- services/net/domain/routing.lua +-- Product-level routing and policy-routing intent. + +local schema = require 'services.net.schema' + +local M = {} + +local HOST_NETMASK = '255.255.255.255' + +local ALLOWED = { + 'tables', 'routes', 'rules', 'defaults', 'metadata', 'extensions', +} + +local ROUTE_ALLOWED = { + 'kind', 'target', 'netmask', 'interface', 'gateway', 'metric', 'table', + 'description', 'metadata', 'extensions', +} + +local function is_ipv4_addr(s) + if type(s) ~= 'string' then return false end + local a, b, c, d = s:match('^(%d+)%.(%d+)%.(%d+)%.(%d+)$') + if not a then return false end + for _, part in ipairs({ a, b, c, d }) do + local n = tonumber(part) + if n == nil or n < 0 or n > 255 then return false end + end + return true +end + +local function route_path(path, field) + return { schema.path(path), field } +end + +local function nonempty_string(v, path) + local value, err = schema.optional_string(v, path) + if err then return nil, err end + if value == nil or value == '' then return nil, schema.err(path, 'must be a non-empty string') end + return value, nil +end + +local function normalise_route(id, v, path) + local t, err = schema.require_plain_table(v, path) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, ROUTE_ALLOWED, path) + if not ok then return nil, ferr end + local kind = t.kind + if kind ~= 'host' and kind ~= 'subnet' then + return nil, schema.err(route_path(path, 'kind'), "must be 'host' or 'subnet'") + end + local target, terr = nonempty_string(t.target, route_path(path, 'target')) + if terr then return nil, terr end + local iface, ierr = nonempty_string(t.interface, route_path(path, 'interface')) + if ierr then return nil, ierr end + local netmask, nerr = schema.optional_string(t.netmask, route_path(path, 'netmask')) + if nerr then return nil, nerr end + if kind == 'host' then + if not is_ipv4_addr(target) then return nil, schema.err(route_path(path, 'target'), 'host route target must be a single IPv4 address') end + netmask = HOST_NETMASK + elseif netmask == nil or netmask == '' then + return nil, schema.err(route_path(path, 'netmask'), 'subnet route requires netmask') + end + local gateway, gerr = schema.optional_string(t.gateway, route_path(path, 'gateway')) + if gerr then return nil, gerr end + local metric, merr = schema.optional_integer(t.metric, route_path(path, 'metric')) + if merr then return nil, merr end + local table_name, taberr = schema.optional_string(t.table, route_path(path, 'table')) + if taberr then return nil, taberr end + local desc, derr = schema.optional_string(t.description, route_path(path, 'description')) + if derr then return nil, derr end + local out = { + kind = kind, + target = target, + netmask = netmask, + interface = iface, + gateway = gateway, + metric = metric, + table = table_name, + description = desc, + } + return schema.with_optional_extensions(out, t, path) +end + +function M.normalise(v) + local t, err = schema.optional_plain_table(v, { 'net', 'routing' }) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, ALLOWED, { 'net', 'routing' }) + if not ok then return nil, ferr end + local routes, rerr = schema.map(t.routes, { 'net', 'routing', 'routes' }, normalise_route) + if not routes then return nil, rerr end + local out = { + tables = schema.copy(t.tables or {}), + routes = routes, + rules = schema.copy(t.rules or {}), + defaults = schema.copy(t.defaults or {}), + } + return schema.with_optional_extensions(out, t, { 'net', 'routing' }) +end + +return M diff --git a/src/services/net/domain/segments.lua b/src/services/net/domain/segments.lua new file mode 100644 index 00000000..776222e2 --- /dev/null +++ b/src/services/net/domain/segments.lua @@ -0,0 +1,177 @@ +-- services/net/domain/segments.lua +-- Product-level network segment normalisation. No HAL or backend knowledge. + +local schema = require 'services.net.schema' + +local M = {} + +local ALLOWED = { + 'id', 'name', 'description', 'kind', 'enabled', 'protected', 'user_editable', 'purpose', 'vlan', 'addressing', + 'dhcp', 'dns', 'firewall', 'routing', 'shaping', 'vpn', 'policy', 'l2', + 'tags', 'metadata', 'extensions', +} + + +local SHAPING_ALLOWED = { + 'enabled', 'download', 'upload', 'host_default', 'host_overrides', + 'fq_codel', 'metadata', 'extensions', +} + +local DIRECTION_ALLOWED = { 'enabled', 'limit', 'metadata', 'extensions' } +local HOST_ALLOWED = { + 'mode', 'all_hosts', 'download', 'upload', 'fq_codel', 'metadata', 'extensions', +} +local HOST_DIRECTION_ALLOWED = { 'enabled', 'sustained_rate', 'peak_rate', 'burst_budget', 'metadata', 'extensions' } + +local function normalise_direction(t, path, require_limit) + if t == nil then return nil, nil end + local v, err = schema.require_plain_table(t, path) + if not v then return nil, err end + local ok, ferr = schema.check_allowed_fields(v, DIRECTION_ALLOWED, path) + if not ok then return nil, ferr end + if require_limit and (type(v.limit) ~= 'string' or v.limit == '') then + return nil, schema.err({ schema.path(path), 'limit' }, 'must be a non-empty rate string') + end + return schema.with_optional_extensions({ enabled = v.enabled ~= false, limit = v.limit }, v, path) +end + +local function normalise_host_direction(t, path, required) + if t == nil then + if required then return nil, schema.err(path, 'is required') end + return nil, nil + end + local v, err = schema.require_plain_table(t, path) + if not v then return nil, err end + local ok, ferr = schema.check_allowed_fields(v, HOST_DIRECTION_ALLOWED, path) + if not ok then return nil, ferr end + for _, field in ipairs({ 'sustained_rate', 'peak_rate', 'burst_budget' }) do + if type(v[field]) ~= 'string' or v[field] == '' then + return nil, schema.err({ schema.path(path), field }, 'must be a non-empty rate string') + end + end + return schema.with_optional_extensions({ + enabled = v.enabled ~= false, + sustained_rate = v.sustained_rate, + peak_rate = v.peak_rate, + burst_budget = v.burst_budget, + }, v, path) +end + +local function normalise_host_default(t, path) + if t == nil then return nil, nil end + local v, err = schema.require_plain_table(t, path) + if not v then return nil, err end + local ok, ferr = schema.check_allowed_fields(v, HOST_ALLOWED, path) + if not ok then return nil, ferr end + if v.mode ~= 'budgeted_peak' then + return nil, schema.err({ schema.path(path), 'mode' }, 'must be budgeted_peak') + end + local download, derr = normalise_host_direction(v.download, { schema.path(path), 'download' }, true) + if not download then return nil, derr end + local upload, uerr = normalise_host_direction(v.upload, { schema.path(path), 'upload' }, true) + if not upload then return nil, uerr end + local out = { + mode = v.mode, + all_hosts = v.all_hosts ~= false, + download = download, + upload = upload, + fq_codel = schema.copy(v.fq_codel or {}), + } + return schema.with_optional_extensions(out, v, path) +end + +local function normalise_shaping(v, path) + if v == nil then return {}, nil end + local t, err = schema.require_plain_table(v, path) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, SHAPING_ALLOWED, path) + if not ok then return nil, ferr end + local download, derr = normalise_direction(t.download, { schema.path(path), 'download' }, false) + if derr then return nil, derr end + local upload, uerr = normalise_direction(t.upload, { schema.path(path), 'upload' }, false) + if uerr then return nil, uerr end + local host_default, herr = normalise_host_default(t.host_default, { schema.path(path), 'host_default' }) + if herr then return nil, herr end + local out = { + enabled = t.enabled ~= false, + download = download, + upload = upload, + host_default = host_default, + host_overrides = schema.copy(t.host_overrides or {}), + fq_codel = schema.copy(t.fq_codel or {}), + } + return schema.with_optional_extensions(out, t, path) +end + +local function normalise_vlan(v, path) + if v == nil then return nil, nil end + if type(v) == 'number' then + if v % 1 ~= 0 or v < 1 or v > 4094 then return nil, schema.err(path, 'must be a VLAN id in the range 1..4094') end + return { id = v }, nil + end + local t, err = schema.require_plain_table(v, path) + if not t then return nil, err end + local out = schema.copy(t) + if out.reserved ~= nil then + if type(out.reserved) ~= 'string' or out.reserved == '' then + return nil, schema.err({ schema.path(path), 'reserved' }, 'must be a non-empty reserved VLAN name') + end + end + if out.auto ~= nil then + if type(out.auto) ~= 'string' or out.auto == '' then + return nil, schema.err({ schema.path(path), 'auto' }, 'must be a non-empty VLAN range name') + end + end + if out.id ~= nil then + local id, ierr = schema.optional_integer(out.id, { schema.path(path), 'id' }) + if ierr then return nil, ierr end + if id < 1 or id > 4094 then return nil, schema.err({ schema.path(path), 'id' }, 'must be in the range 1..4094') end + out.id = id + end + return out, nil +end + +function M.normalise_record(id, rec, path) + local t, err = schema.require_plain_table(rec, path) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, ALLOWED, path) + if not ok then return nil, ferr end + if t.id ~= nil and t.id ~= id then return nil, schema.err({ schema.path(path), 'id' }, 'must match the map key') end + + local vlan, verr = normalise_vlan(t.vlan, { schema.path(path), 'vlan' }) + if verr then return nil, verr end + local tags, terr = schema.id_list(t.tags, { schema.path(path), 'tags' }) + if terr then return nil, terr end + + local shaping, sherr = normalise_shaping(t.shaping, { schema.path(path), 'shaping' }) + if not shaping then return nil, sherr end + + local out = { + id = id, + name = t.name or id, + description = t.description, + kind = t.kind or 'lan', + enabled = t.enabled ~= false, + protected = t.protected == true, + user_editable = t.user_editable ~= false, + purpose = t.purpose, + vlan = vlan, + addressing = schema.copy(t.addressing or {}), + l2 = schema.copy(t.l2 or {}), + dhcp = schema.copy(t.dhcp or {}), + dns = schema.copy(t.dns or {}), + firewall = schema.copy(t.firewall or {}), + routing = schema.copy(t.routing or {}), + shaping = shaping, + vpn = schema.copy(t.vpn or {}), + policy = schema.copy(t.policy or {}), + tags = tags, + } + return schema.with_optional_extensions(out, t, path) +end + +function M.normalise(input) + return schema.map(input, { 'net', 'segments' }, M.normalise_record) +end + +return M diff --git a/src/services/net/domain/vpn.lua b/src/services/net/domain/vpn.lua new file mode 100644 index 00000000..cd793da7 --- /dev/null +++ b/src/services/net/domain/vpn.lua @@ -0,0 +1,29 @@ +-- services/net/domain/vpn.lua +-- Product-level VPN and overlay transport intent. + +local schema = require 'services.net.schema' + +local M = {} + +local ALLOWED = { + 'enabled', 'tunnels', 'peers', 'policies', 'routes', 'overlays', + 'metadata', 'extensions', +} + +function M.normalise(v) + local t, err = schema.optional_plain_table(v, { 'net', 'vpn' }) + if not t then return nil, err end + local ok, ferr = schema.check_allowed_fields(t, ALLOWED, { 'net', 'vpn' }) + if not ok then return nil, ferr end + local out = { + enabled = t.enabled == true, + tunnels = schema.copy(t.tunnels or {}), + peers = schema.copy(t.peers or {}), + policies = schema.copy(t.policies or {}), + routes = schema.copy(t.routes or {}), + overlays = schema.copy(t.overlays or {}), + } + return schema.with_optional_extensions(out, t, { 'net', 'vpn' }) +end + +return M diff --git a/src/services/net/drift.lua b/src/services/net/drift.lua new file mode 100644 index 00000000..769905db --- /dev/null +++ b/src/services/net/drift.lua @@ -0,0 +1,114 @@ +-- services/net/drift.lua +-- Pure desired-vs-observed drift calculation for NET. + +local tablex = require 'shared.table' + +local M = {} + +local function copy(v) return tablex.deep_copy(v) end + +local function add(items, domain, kind, id, fields) + local rec = { + domain = domain, + kind = kind, + id = id, + severity = (fields and fields.severity) or 'warn', + } + if fields then + for k, v in pairs(fields) do + if k ~= 'severity' then rec[k] = v end + end + end + items[#items + 1] = rec +end + +local function observed_by_id(observed, key) + local t = observed and observed[key] or nil + return type(t) == 'table' and t or {} +end + +local function desired_segment_for_interface(iface) + if type(iface) ~= 'table' then return nil end + return iface.segment or (type(iface.segments) == 'table' and iface.segments[1]) or nil +end + +local function check_interfaces(items, snapshot, observed) + local desired = snapshot.interfaces or {} + local obs = observed_by_id(observed, 'interfaces') + for id, iface in pairs(desired) do + if iface.enabled ~= false and obs[id] == nil then + add(items, 'interfaces', 'missing_interface', id, { severity = 'error', desired = copy(iface) }) + end + end + for id, rec in pairs(obs) do + local iface = desired[id] + if iface == nil then + add(items, 'interfaces', 'unexpected_interface', id, { observed = copy(rec) }) + elseif iface.enabled ~= false and rec.enabled == false then + add(items, 'interfaces', 'interface_disabled', id, { severity = 'error', desired = copy(iface), observed = copy(rec) }) + end + end +end + +local function check_segments(items, snapshot, observed) + local desired = snapshot.segments or {} + local obs = observed_by_id(observed, 'segments') + for id, seg in pairs(desired) do + if seg.enabled ~= false and obs[id] == nil then + add(items, 'segments', 'missing_segment', id, { severity = 'error', desired = copy(seg) }) + end + end + for id, rec in pairs(obs) do + if desired[id] == nil then + add(items, 'segments', 'unexpected_segment', id, { observed = copy(rec) }) + end + end +end + +local function check_firewall(items, snapshot, observed) + local zones = snapshot.firewall and snapshot.firewall.zones or {} + local obs_zones = observed and observed.firewall and observed.firewall.zones or nil + if type(obs_zones) ~= 'table' then return end + for id, zone in pairs(zones or {}) do + if obs_zones[id] == nil then + add(items, 'firewall', 'missing_zone', id, { severity = 'error', desired = copy(zone) }) + end + end + for id, zone in pairs(obs_zones) do + if zones[id] == nil then add(items, 'firewall', 'unexpected_zone', id, { observed = copy(zone) }) end + end +end + +local function check_wan(items, snapshot, observed) + local desired = snapshot.wan and snapshot.wan.realised_members or {} + local obs = observed and observed.wan and observed.wan.members or nil + if type(obs) ~= 'table' then return end + for id, member in pairs(desired or {}) do + if member.enabled ~= false and obs[id] == nil then + add(items, 'wan', 'missing_member', id, { severity = 'error', desired = copy(member) }) + end + end + for id, member in pairs(obs) do + if desired[id] == nil then add(items, 'wan', 'unexpected_member', id, { observed = copy(member) }) end + end +end + +function M.calculate(snapshot, opts) + opts = opts or {} + snapshot = snapshot or {} + local observed = snapshot.observed and snapshot.observed.snapshot or snapshot.observed or {} + local items = {} + check_interfaces(items, snapshot, observed) + check_segments(items, snapshot, observed) + check_firewall(items, snapshot, observed) + check_wan(items, snapshot, observed) + return { + converged = #items == 0, + items = items, + updated_at = opts.now and opts.now() or nil, + } +end + +M._test = { desired_segment_for_interface = desired_segment_for_interface } + +return M diff --git a/src/services/net/events.lua b/src/services/net/events.lua new file mode 100644 index 00000000..d7b74fb5 --- /dev/null +++ b/src/services/net/events.lua @@ -0,0 +1,142 @@ +-- services/net/events.lua +-- Semantic event selection for the NET service coordinator. + +local priority_event = require 'devicecode.support.priority_event' +local queue = require 'devicecode.support.queue' + +local M = {} + +local NOT_READY = {} + +local function recv_now(rx, map) + local item = queue.try_now(rx:recv_op(), NOT_READY) + if item == NOT_READY then return nil end + return map(item) +end + +local function recv_op(rx, map) + return rx:recv_op():wrap(function (item) + return map(item) + end) +end + +function M.map_completion(ev) + if ev == nil then return { kind = 'completion_queue_closed' } end + return ev +end + +function M.map_config_event(ev) + if ev == nil then return { kind = 'config_watch_closed' } end + if type(ev) == 'table' and ev.kind == 'config_closed' then + return { kind = 'config_watch_closed', err = ev.err } + end + if type(ev) == 'table' and ev.kind == 'config_changed' then + return { + kind = 'config_changed', + payload = ev.record or ev.raw, + rev = ev.rev, + watch_generation = ev.generation, + origin = ev.msg and ev.msg.origin or ev.origin, + } + end + return { kind = 'config_event_unknown', event = ev } +end + +function M.map_observed_event(msg) + if msg == nil then return { kind = 'observation_closed' } end + local payload = msg.payload or msg + return { + kind = 'observed_state', + event = payload, + origin = msg.origin, + topic = msg.topic, + } +end + +function M.map_capability_status(name, msg) + if msg == nil then return { kind = 'capability_status_closed', capability = name } end + return { + kind = 'capability_status', + capability = name, + payload = msg.payload, + origin = msg.origin, + topic = msg.topic, + } +end + +local function try_config_now(state) + if not state.config_watch then return nil end + local ev = state.config_watch:try_recv_now() + if ev == nil then return nil end + return M.map_config_event(ev) +end + +local function try_observed_now(state) + if not state.observed_sub then return nil end + local ev = queue.try_now(state.observed_sub:recv_op(), NOT_READY) + if ev == NOT_READY then return nil end + return M.map_observed_event(ev) +end + +local function try_gsm_uplink_now(state) + if not state.gsm_uplink_watch then return nil end + return state.gsm_uplink_watch:try_recv_now() +end + +local function add_capability_sources(state, sources) + if state.cap_deps and type(state.cap_deps.event_source) == 'function' then + sources[#sources + 1] = state.cap_deps:event_source({ name = 'capability_dependencies' }) + return + end + + local subs = state.capability_status_subs or {} + local names = {} + for name in pairs(subs) do names[#names + 1] = name end + table.sort(names) + for i = 1, #names do + local name = names[i] + local sub = subs[name] + sources[#sources + 1] = { + name = 'capability_' .. tostring(name), + try_now = function () return recv_now(sub, function (msg) return M.map_capability_status(name, msg) end) end, + recv_op = function () return recv_op(sub, function (msg) return M.map_capability_status(name, msg) end) end, + } + end +end + +function M.next_service_event_op(state) + local sources = { + { + name = 'completion', + try_now = function () return recv_now(state.done_rx, M.map_completion) end, + recv_op = function () return recv_op(state.done_rx, M.map_completion) end, + }, + { + name = 'config', + enabled = function () return state.config_watch ~= nil end, + try_now = function () return try_config_now(state) end, + recv_op = function () return state.config_watch:recv_op():wrap(M.map_config_event) end, + }, + { + name = 'gsm_uplinks', + enabled = function () return state.gsm_uplink_watch ~= nil end, + try_now = function () return try_gsm_uplink_now(state) end, + recv_op = function () return state.gsm_uplink_watch:recv_op() end, + }, + { + name = 'observed', + enabled = function () return state.observed_sub ~= nil end, + try_now = function () return try_observed_now(state) end, + recv_op = function () return state.observed_sub:recv_op():wrap(M.map_observed_event) end, + }, + } + add_capability_sources(state, sources) + + return priority_event.sources_op { + label = 'net.service.next_event', + pending = state.pending, + sources = sources, + } +end + +return M diff --git a/src/services/net/generation.lua b/src/services/net/generation.lua new file mode 100644 index 00000000..d73fafa7 --- /dev/null +++ b/src/services/net/generation.lua @@ -0,0 +1,80 @@ +-- services/net/generation.lua +-- Generation records for NET config/application lifetimes. + +local tablex = require 'shared.table' + +local M = {} + +local function copy(v) return tablex.deep_copy(v) end + +function M.new(id, intent, reason, opts) + opts = opts or {} + return { + generation = id, + intent = copy(intent), + reason = reason or 'config_changed', + state = 'accepted', + accepted_at = opts.now, + apply = nil, + runtime = {}, + } +end + +function M.start_apply(gen, apply_id, opts) + if not gen then return nil, 'generation required' end + gen.state = 'applying' + gen.apply = { + apply_id = apply_id, + state = 'running', + started_at = opts and opts.now or nil, + } + return gen +end + +function M.mark_applied(gen, result, opts) + if not gen then return nil, 'generation required' end + gen.state = 'applied' + gen.apply = gen.apply or {} + gen.apply.state = 'applied' + gen.apply.completed_at = opts and opts.now or nil + gen.apply.result = copy(result) + return gen +end + +function M.mark_failed(gen, reason, result, opts) + if not gen then return nil, 'generation required' end + gen.state = 'failed' + gen.reason = reason or gen.reason + gen.apply = gen.apply or {} + gen.apply.state = 'failed' + gen.apply.completed_at = opts and opts.now or nil + gen.apply.reason = reason + gen.apply.result = copy(result) + return gen +end + +function M.snapshot(gen) + if not gen then return nil end + return { + generation = gen.generation, + reason = gen.reason, + state = gen.state, + intent_rev = gen.intent and gen.intent.rev or nil, + accepted_at = gen.accepted_at, + apply = copy(gen.apply), + runtime = copy(gen.runtime), + } +end + +function M.cancel(gen, reason) + if not gen then return true, nil end + gen.state = 'cancelled' + gen.reason = reason or gen.reason + if gen.apply then gen.apply.state = 'cancelled'; gen.apply.reason = gen.reason end + if gen.apply_handle and type(gen.apply_handle.cancel) == 'function' then + return gen.apply_handle:cancel(reason or 'generation_cancelled') + end + return true, nil +end + +return M diff --git a/src/services/net/gsm_uplink_watch.lua b/src/services/net/gsm_uplink_watch.lua new file mode 100644 index 00000000..325242be --- /dev/null +++ b/src/services/net/gsm_uplink_watch.lua @@ -0,0 +1,99 @@ +-- services/net/gsm_uplink_watch.lua +-- Retained GSM uplink-state adapter for the NET coordinator. + +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local queue = require 'devicecode.support.queue' +local tablex = require 'shared.table' +local topics = require 'services.net.topics' + +local M = {} +local Watch = {} +Watch.__index = Watch + +local NOT_READY = {} + +local function copy(v) return tablex.deep_copy(v) end + +local function role_from_topic(topic) + if type(topic) ~= 'table' then return nil end + if topic[1] == 'state' and topic[2] == 'gsm' and topic[3] == 'uplink' then + local role = topic[4] + if type(role) == 'string' and role ~= '' then return role end + end + return nil +end + +local function map_event(ev) + if ev == nil then return { kind = 'gsm_uplink_watch_closed' } end + if type(ev) == 'table' and ev.op == 'replay_done' then return { kind = 'gsm_uplink_replay_done', origin = ev.origin } end + + local role = type(ev) == 'table' and role_from_topic(ev.topic) or nil + if not role then return { kind = 'gsm_uplink_unknown', event = ev } end + + if ev.op == 'unretain' then + return { + kind = 'gsm_uplink_changed', + role = role, + op = ev.op, + topic = ev.topic, + origin = ev.origin, + payload = { + schema = 'devicecode.gsm.uplink/1', + id = role, + role = role, + state = 'unavailable', + connected = false, + available = false, + reason = 'unretained', + }, + } + end + + local payload = copy(ev.payload or {}) + payload.schema = payload.schema or 'devicecode.gsm.uplink/1' + payload.id = payload.id or role + payload.role = payload.role or role + return { + kind = 'gsm_uplink_changed', + role = role, + op = ev.op, + topic = ev.topic, + origin = ev.origin, + payload = payload, + } +end + +function Watch:recv_op() + return self._watch:recv_op():wrap(map_event) +end + +function Watch:try_recv_now() + local ev = queue.try_now(self._watch:recv_op(), NOT_READY) + if ev == NOT_READY then return nil end + return map_event(ev) +end + +function Watch:close() + if self._closed then return true, nil end + self._closed = true + return bus_cleanup.unwatch_retained(self._conn, self._watch) +end + +function Watch:terminate(_reason) + return self:close() +end + +function M.open(conn, opts) + opts = opts or {} + local watch, err = bus_cleanup.watch_retained(conn, topics.gsm_uplink_pattern(), { + replay = true, + queue_len = opts.queue_len or 8, + full = opts.full or 'reject_newest', + }) + if not watch then return nil, err end + return setmetatable({ _conn = conn, _watch = watch, _closed = false }, Watch), nil +end + +M._test = { map_event = map_event, role_from_topic = role_from_topic } + +return M diff --git a/src/services/net/hal_client.lua b/src/services/net/hal_client.lua new file mode 100644 index 00000000..a1310aca --- /dev/null +++ b/src/services/net/hal_client.lua @@ -0,0 +1,181 @@ +-- services/net/hal_client.lua +-- NET-facing semantic HAL client. +-- +-- This module deliberately exposes product-level operations only. NET callers +-- should not see host-specific command or configuration details here. + +local op = require 'fibers.op' +local tablex = require 'shared.table' +local cap_sdk = require 'services.hal.sdk.cap' +local dep_failure = require 'devicecode.support.dependency_failure' + +local M = {} +local Client = {} +Client.__index = Client + +local function copy(v) + if type(v) == 'table' then return tablex.deep_copy(v) end + return v +end + +local function reason_text(reason, fallback) + if type(reason) == 'table' then + return tostring(reason.err or reason.reason or reason.message or reason.code or fallback) + end + if reason ~= nil then return tostring(reason) end + return tostring(fallback) +end + +local function failure_result(reason, fallback, code) + local out = { + ok = false, + err = reason_text(reason, fallback or 'network HAL rejected request'), + reason = copy(reason), + } + if code ~= nil then out.code = code end + return out +end + +local function reply_to_result(reply, err, dependency_key) + if not reply then + local dep = dep_failure.from_no_route(dependency_key, err, { + err = 'no_route', + source = 'network_hal_call', + }) + if dep then return dep end + return failure_result({ err = 'network HAL call failed', detail = err }, 'network HAL call failed') + end + + if reply.ok ~= true then + return failure_result(reply.reason, 'network HAL rejected request', reply.code) + end + + if type(reply.reason) == 'table' then + local out = copy(reply.reason) + if out.ok == nil then out.ok = true end + return out + end + + return { + ok = true, + result = reply.reason, + } +end + +local function default_ref(conn, class, id) + if not conn then return nil end + return cap_sdk.new_curated_cap_ref(conn, class, id or 'main') +end + +function M.new(conn, opts) + opts = opts or {} + local resolve_defaults = opts.resolve_defaults ~= false + return setmetatable({ + conn = conn, + network_config = opts.network_config_cap or (resolve_defaults and default_ref(conn, 'network-config', opts.network_config_id)), + network_state = opts.network_state_cap or (resolve_defaults and default_ref(conn, 'network-state', opts.network_state_id)), + network_diagnostics = opts.network_diagnostics_cap or (resolve_defaults and default_ref(conn, 'network-diagnostics', opts.network_diagnostics_id)), + -- Dry-run must be explicit. A missing network-config capability should not + -- look like a successful host apply on production devices. + dry_run = opts.dry_run == true, + }, Client) +end + +function Client:available() + return self.network_config ~= nil +end + +function Client:observation_available() + return self.conn ~= nil and self.network_state ~= nil +end + +function Client:apply_intent_op(intent, opts) + opts = opts or {} + if self.network_config and type(self.network_config.call_control_op) == 'function' then + return self.network_config:call_control_op('apply', { + intent = intent, + opts = opts, + }, opts):wrap(function (reply, err) return reply_to_result(reply, err, 'network_config') end) + end + + if self.dry_run == true then + return op.always({ + ok = true, + applied = false, + changed = false, + backend = 'none', + dry_run = true, + note = 'network-config HAL capability not configured; explicit dry-run apply', + }) + end + + return op.always({ + ok = false, + err = 'network-config HAL capability not configured', + reason = { + code = 'missing_network_config_hal', + detail = 'network-config HAL capability not configured', + }, + }) +end + + +function Client:apply_live_weights_op(req, opts) + opts = opts or {} + if self.network_config and type(self.network_config.call_control_op) == 'function' then + return self.network_config:call_control_op('apply_live_weights', req or {}, opts):wrap(function (reply, err) return reply_to_result(reply, err, 'network_config') end) + end + return op.always({ ok = false, err = 'network-config HAL capability not configured', reason = { code = 'missing_network_config_hal' } }) +end + +function Client:apply_shaping_op(req, opts) + opts = opts or {} + if self.network_config and type(self.network_config.call_control_op) == 'function' then + return self.network_config:call_control_op('apply_shaping', req or {}, opts):wrap(function (reply, err) return reply_to_result(reply, err, 'network_config') end) + end + return op.always({ ok = false, err = 'network-config HAL capability not configured', reason = { code = 'missing_network_config_hal' } }) +end + +function Client:speedtest_op(req, opts) + opts = opts or {} + if self.network_diagnostics and type(self.network_diagnostics.call_control_op) == 'function' then + return self.network_diagnostics:call_control_op('speedtest', req or {}, opts):wrap(function (reply, err) return reply_to_result(reply, err, 'network_diagnostics') end) + end + return op.always({ ok = false, err = 'network-diagnostics HAL capability not configured', reason = { code = 'missing_network_diagnostics_hal' } }) +end + +function Client:read_counters_op(req, opts) + opts = opts or {} + if self.network_diagnostics and type(self.network_diagnostics.call_control_op) == 'function' then + return self.network_diagnostics:call_control_op('read_counters', req or {}, opts):wrap(function (reply, err) return reply_to_result(reply, err, 'network_diagnostics') end) + end + return op.always({ ok = false, err = 'network-diagnostics HAL capability not configured', reason = { code = 'missing_network_diagnostics_hal' } }) +end + +function Client:start_observation_op(opts) + opts = opts or {} + if self.network_state and type(self.network_state.call_control_op) == 'function' then + return self.network_state:call_control_op('watch', opts, opts):wrap(function (reply, err) return reply_to_result(reply, err, 'network_state') end) + end + return op.always({ + ok = false, + err = 'network-state HAL capability not configured', + reason = { + code = 'missing_network_state_hal', + detail = 'network-state HAL capability not configured', + }, + }) +end + +function Client:open_observed_subscription(opts) + opts = opts or {} + if not self.network_state or type(self.network_state.get_event_sub) ~= 'function' then + return nil, 'network-state HAL capability not configured' + end + return self.network_state:get_event_sub('observed', { + queue_len = opts.queue_len or 32, + full = opts.full or 'drop_oldest', + }) +end + +return M diff --git a/src/services/net/intent_realiser.lua b/src/services/net/intent_realiser.lua new file mode 100644 index 00000000..14e51128 --- /dev/null +++ b/src/services/net/intent_realiser.lua @@ -0,0 +1,116 @@ +-- services/net/intent_realiser.lua +-- Turns product-level NET intent plus observed source facts into apply intent. + +local tablex = require 'shared.table' + +local M = {} + +local function copy(v) return tablex.deep_copy(v) end +local function sorted_keys(t) + local ks = {} + for k in pairs(t or {}) do ks[#ks + 1] = k end + table.sort(ks, function(a, b) return tostring(a) < tostring(b) end) + return ks +end + +local function gsm_ifname(sources, role) + local rec = sources and sources.gsm_uplinks and sources.gsm_uplinks[role] + if type(rec) ~= 'table' then return nil end + local linux = type(rec.linux) == 'table' and rec.linux or {} + local ifname = linux.ifname + if type(ifname) == 'string' and ifname ~= '' then return ifname end + return nil +end + +local function merge_ipv4(existing, metric) + local out = copy(existing or {}) + out.mode = out.mode or out.proto or 'dhcp' + out.proto = nil + out.peerdns = out.peerdns ~= nil and out.peerdns or false + out.metric = metric + return out +end + +local function set_route_metric_on_interface(intent, iface_id, metric) + local iface = intent.interfaces and intent.interfaces[iface_id] + if type(iface) ~= 'table' then return false end + iface.addressing = type(iface.addressing) == 'table' and iface.addressing or {} + iface.addressing.ipv4 = merge_ipv4(iface.addressing.ipv4, metric) + return true +end + +local function set_route_metric_on_segment(intent, seg_id, metric) + local seg = intent.segments and intent.segments[seg_id] + if type(seg) ~= 'table' then return false end + seg.addressing = type(seg.addressing) == 'table' and seg.addressing or {} + seg.addressing.ipv4 = merge_ipv4(seg.addressing.ipv4, metric) + return true +end + +local function realisable_static_member(intent, iface_id) + return (intent.interfaces and intent.interfaces[iface_id] ~= nil) + or (intent.segments and intent.segments[iface_id] ~= nil) +end + +function M.realise(base_intent, sources, opts) + opts = opts or {} + local intent = copy(base_intent or {}) + intent.interfaces = intent.interfaces or {} + intent.wan = intent.wan or {} + intent.wan.members = intent.wan.members or {} + + local realised_members = {} + for index, member_id in ipairs(sorted_keys(intent.wan.members)) do + local member = intent.wan.members[member_id] + if type(member) == 'table' and member.enabled ~= false and member.disabled ~= true then + local iface_id = member.interface or member_id + member.interface = iface_id + local route_metric = tonumber(member.route_metric) or (10 + index) + member.route_metric = route_metric + member.metric = tonumber(member.metric) or 1 + -- HAL/OpenWrt still consumes this backend-shaped alias when realising mwan3. + member.mwan_metric = member.metric + local src = member.source + if type(src) == 'table' and src.kind == 'gsm-uplink' then + local ifname = gsm_ifname(sources or {}, src.id) + if ifname then + intent.interfaces[iface_id] = { + kind = 'direct', + role = 'wan', + enabled = true, + endpoint = { ifname = ifname }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false, metric = route_metric } }, + firewall = { zone = 'wan' }, + dhcp = { enabled = false }, + source = { kind = 'gsm-uplink', id = src.id }, + } + realised_members[member_id] = member + end + elseif realisable_static_member(intent, iface_id) then + if not set_route_metric_on_interface(intent, iface_id, route_metric) then + set_route_metric_on_segment(intent, iface_id, route_metric) + end + realised_members[member_id] = member + elseif opts.keep_unrealised == true then + realised_members[member_id] = member + end + end + end + intent.wan.members = realised_members + return intent +end + +function M.realised_fingerprint(intent, sources) + local realised = M.realise(intent, sources or {}) + local parts = {} + for _, id in ipairs(sorted_keys(realised.wan and realised.wan.members)) do + local m = realised.wan.members[id] + local iface = m and m.interface or id + local i = realised.interfaces and realised.interfaces[iface] or nil + local ep = type(i) == 'table' and type(i.endpoint) == 'table' and i.endpoint or {} + parts[#parts + 1] = table.concat({ tostring(id), tostring(iface), tostring(ep.ifname), tostring(m and m.route_metric), tostring(m and m.metric) }, ':') + end + return table.concat(parts, '|') +end + +return M diff --git a/src/services/net/metrics.lua b/src/services/net/metrics.lua new file mode 100644 index 00000000..df7234d6 --- /dev/null +++ b/src/services/net/metrics.lua @@ -0,0 +1,142 @@ +-- services/net/metrics.lua +-- NET observability metrics helpers. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' + +local perform = fibers.perform + +local M = {} + +local function obs_log(svc, level, payload) + if svc and type(svc.obs_log) == 'function' then svc:obs_log(level, payload) end +end + +local COUNTER_STATS = { + 'rx_bytes', + 'rx_packets', + 'rx_dropped', + 'rx_errors', + 'tx_bytes', + 'tx_packets', + 'tx_dropped', + 'tx_errors', +} + +local STATIC_COUNTER_ALIASES = { + { alias = 'adm', interface = 'adm' }, + { alias = 'jan', interface = 'jan' }, + { alias = 'wan', interface = 'wan' }, +} + +local function sorted_keys(t) + local keys = {} + for k in pairs(t or {}) do keys[#keys + 1] = k end + table.sort(keys, function(a, b) return tostring(a) < tostring(b) end) + return keys +end + +local function has_counter_interface(snapshot, id) + return type(id) == 'string' and id ~= '' + and (type(snapshot.interfaces) == 'table' and snapshot.interfaces[id] ~= nil + or type(snapshot.segments) == 'table' and snapshot.segments[id] ~= nil) +end + +local function add_counter_alias(list, seen_alias, seen_iface, alias, iface) + if type(alias) ~= 'string' or alias == '' or type(iface) ~= 'string' or iface == '' then return end + if seen_alias[alias] or seen_iface[iface] then return end + seen_alias[alias] = true + seen_iface[iface] = true + list[#list + 1] = { alias = alias, interface = iface } +end + +local function counter_aliases(snapshot) + snapshot = snapshot or {} + local out, seen_alias, seen_iface = {}, {}, {} + for i = 1, #STATIC_COUNTER_ALIASES do + local rec = STATIC_COUNTER_ALIASES[i] + if has_counter_interface(snapshot, rec.interface) then add_counter_alias(out, seen_alias, seen_iface, rec.alias, rec.interface) end + end + + local members = snapshot.wan and snapshot.wan.configured_members or {} + for _, member_id in ipairs(sorted_keys(members)) do + local member = members[member_id] + local iface = type(member) == 'table' and (member.interface or member_id) or member_id + add_counter_alias(out, seen_alias, seen_iface, tostring(member_id), iface) + end + return out +end + +local function counter_request_from_aliases(aliases) + local interfaces = {} + for i = 1, #(aliases or {}) do interfaces[#interfaces + 1] = aliases[i].interface end + return { interfaces = interfaces, stats = COUNTER_STATS } +end + +local function publish_counter_metrics(svc, counters, aliases) + if not svc or type(svc.obs_metric) ~= 'function' or type(counters) ~= 'table' then return true, nil end + local alias_by_interface = {} + for i = 1, #(aliases or {}) do alias_by_interface[aliases[i].interface] = aliases[i].alias end + for _, iface in ipairs(sorted_keys(counters)) do + local rec = counters[iface] + local alias = alias_by_interface[iface] + local stats = type(rec) == 'table' and rec.statistics or nil + if alias and type(stats) == 'table' then + for i = 1, #COUNTER_STATS do + local stat = COUNTER_STATS[i] + local value = tonumber(stats[stat]) + if value ~= nil then + svc:obs_metric(stat, { + value = value, + namespace = { 'net', alias, stat }, + }) + end + end + end + end + return true, nil +end + + +function M.poll_counter_metrics_once(state) + if not state.conn or not state.hal or type(state.hal.read_counters_op) ~= 'function' then return true, nil end + local snap = state.model and state.model:snapshot() or {} + if snap.ready ~= true then return true, nil end + local aliases = counter_aliases(snap) + if #aliases == 0 then return true, nil end + local ok, result = pcall(function () + return perform(state.hal:read_counters_op(counter_request_from_aliases(aliases), { + timeout = state.counter_poll_timeout_s, + })) + end) + if not ok then + obs_log(state.svc, 'warn', { what = 'net_counter_poll_failed', err = tostring(result) }) + return true, nil + end + if not result or result.ok ~= true then + obs_log(state.svc, 'debug', { what = 'net_counter_poll_unavailable', err = result and result.err or 'no_result' }) + return true, nil + end + publish_counter_metrics(state.svc, result.counters, aliases) + return true, nil +end + +function M.counter_metrics_loop(state) + local interval = tonumber(state.counter_poll_interval_s) or 60 + if interval <= 0 then interval = 60 end + while true do + M.poll_counter_metrics_once(state) + local snap = state.model and state.model:snapshot() or {} + local delay = snap.ready == true and interval or math.min(interval, 1.0) + perform(sleep.sleep_op(delay)) + end +end + +M._test = { + counter_aliases = counter_aliases, + counter_request_from_aliases = counter_request_from_aliases, + publish_counter_metrics = publish_counter_metrics, + counter_stats = COUNTER_STATS, +} + +return M diff --git a/src/services/net/model.lua b/src/services/net/model.lua new file mode 100644 index 00000000..1a45633b --- /dev/null +++ b/src/services/net/model.lua @@ -0,0 +1,101 @@ +-- services/net/model.lua +-- Observable NET service model. + +local support_model = require 'devicecode.support.model' +local tablex = require 'shared.table' + +local M = {} + +local function copy(v) return tablex.deep_copy(v) end + +function M.initial(service_id) + return { + service = 'net', + service_id = service_id, + state = 'starting', + ready = false, + reason = nil, + + generation = 0, + config = { + rev = nil, + schema = nil, + config_schema = nil, + version = nil, + }, + + apply = { + state = 'idle', + generation = nil, + apply_id = nil, + last_applied_rev = nil, + last_error = nil, + last_result = nil, + }, + + intent = { + active = nil, + last_rejected = nil, + generation = nil, + }, + + pending = {}, + + dependencies = {}, + + observed = { + last_event = nil, + last_event_at = nil, + last_subject = nil, + snapshot = nil, + interfaces = {}, + segments = {}, + }, + + drift = { + converged = nil, + items = {}, + updated_at = nil, + }, + + segments = {}, + vlan_policy = {}, + policies = {}, + interfaces = {}, + addressing = {}, + dns = {}, + dhcp = {}, + firewall = {}, + routing = {}, + wan = { configured_members = {}, realised_members = {} }, + backhaul = { schema = 'devicecode.net.backhaul/1', state = 'unknown', uplinks = {} }, + wan_runtime = { uplinks = {}, speedtests = {}, live_weights = {}, last_weight_apply = nil }, + sources = { gsm_uplinks = {} }, + vpn = {}, + diagnostics = {}, + + stats = { + config_updates = 0, + apply_started = 0, + apply_completed = 0, + stale_completions = 0, + observations = 0, + speedtests_started = 0, + speedtests_completed = 0, + live_weight_applies = 0, + gsm_uplink_updates = 0, + }, + } +end + +function M.new(service_id, opts) + opts = opts or {} + return support_model.new(M.initial(service_id), { + label = opts.label or 'net.model', + copy = copy, + }) +end + +function M.deep_copy(v) return copy(v) end + +return M diff --git a/src/services/net/observer_manager.lua b/src/services/net/observer_manager.lua new file mode 100644 index 00000000..59516c4f --- /dev/null +++ b/src/services/net/observer_manager.lua @@ -0,0 +1,67 @@ +-- services/net/observer_manager.lua +-- Owns the NET network-state observation subscription and watch request. + +local fibers = require 'fibers' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' + +local M = {} +local Observer = {} +Observer.__index = Observer + +function M.start(spec) + if type(spec) ~= 'table' then return nil, 'observer spec required' end + local hal = spec.hal + if not hal or type(hal.open_observed_subscription) ~= 'function' then return nil, 'network-state HAL capability not configured' end + + local sub, sub_err = hal:open_observed_subscription({ + queue_len = spec.queue_len or 64, + full = spec.full or 'drop_oldest', + }) + if not sub then return nil, sub_err or 'network observed subscription failed' end + + local handle, err + if type(hal.start_observation_op) == 'function' then + handle, err = scoped_work.start { + lifetime_scope = spec.lifetime_scope, + reaper_scope = spec.reaper_scope or spec.lifetime_scope, + report_scope = spec.report_scope or spec.lifetime_scope, + identity = { + kind = 'net_observation_started', + service_id = spec.service_id or 'net', + generation = spec.generation or 0, + }, + run = function () + local result = fibers.perform(hal:start_observation_op(spec.options or {})) + return { ok = result and result.ok == true, result = result } + end, + report = function (ev) + return queue.try_admit_required(spec.done_tx, ev, 'net_observation_start_report_failed') + end, + } + if not handle then + if sub.close then sub:close(err or 'observation_start_failed') end + return nil, err or 'network observation start failed' + end + end + + return setmetatable({ sub = sub, handle = handle }, Observer), nil +end + +function Observer:subscription() + return self.sub +end + +function Observer:terminate(reason) + if self.handle and self.handle.cancel then self.handle:cancel(reason or 'observer terminated') end + self.handle = nil + if self.sub then + local sub = self.sub + self.sub = nil + if sub.close then return sub:close(reason or 'observer terminated') end + if sub.unsubscribe then return sub:unsubscribe() end + end + return true, nil +end + +return M diff --git a/src/services/net/projection.lua b/src/services/net/projection.lua new file mode 100644 index 00000000..3572866c --- /dev/null +++ b/src/services/net/projection.lua @@ -0,0 +1,72 @@ +-- services/net/projection.lua +-- Public state projection for NET. + +local tablex = require 'shared.table' +local topics = require 'services.net.topics' + +local M = {} + +local function copy(v) return tablex.deep_copy(v) end + +local function count_map(t) + local n = 0 + for _ in pairs(t or {}) do n = n + 1 end + return n +end + +function M.summary_topic() return topics.summary() end +function M.apply_topic() return topics.apply() end +function M.segment_topic(id) return topics.segment(id) end +function M.segments_topic() return topics.segments() end +function M.vlan_policy_topic() return topics.vlan_policy() end +function M.interface_topic(id) return topics.interface(id) end +function M.domain_topic(name) return topics.domain(name) end +function M.sources_topic() return topics.sources() end + +function M.summary(snapshot) + snapshot = snapshot or {} + return { + service = 'net', + state = snapshot.state, + ready = snapshot.ready == true, + reason = snapshot.reason, + generation = snapshot.generation, + config = copy(snapshot.config), + apply = copy(snapshot.apply), + pending = copy(snapshot.pending), + hal = copy(snapshot.hal), + counts = { + segments = count_map(snapshot.segments), + interfaces = count_map(snapshot.interfaces), + wan_members = count_map(snapshot.wan and snapshot.wan.configured_members), + gsm_uplinks = count_map(snapshot.sources and snapshot.sources.gsm_uplinks), + vpn_tunnels = count_map(snapshot.vpn and snapshot.vpn.tunnels), + }, + drift = copy(snapshot.drift), + observed = { + last_event_at = snapshot.observed and snapshot.observed.last_event_at or nil, + last_subject = snapshot.observed and snapshot.observed.last_subject or nil, + }, + dependencies = copy(snapshot.dependencies), + stats = copy(snapshot.stats), + } +end + +function M.apply(snapshot) return copy((snapshot or {}).apply or {}) end +function M.segments(snapshot) + snapshot = snapshot or {} + return { + service = 'net', + kind = 'net.segments', + generation = snapshot.generation, + rev = snapshot.config and snapshot.config.rev or nil, + segments = copy(snapshot.segments or {}), + vlan_policy = copy(snapshot.vlan_policy or ((snapshot.policies or {}).vlan) or {}), + } +end +function M.vlan_policy(snapshot) return copy((snapshot or {}).vlan_policy or (((snapshot or {}).policies or {}).vlan) or {}) end +function M.segment(seg) return copy(seg or {}) end +function M.interface(iface) return copy(iface or {}) end +function M.domain(snapshot, name) return copy((snapshot or {})[name] or {}) end + +return M diff --git a/src/services/net/publisher.lua b/src/services/net/publisher.lua new file mode 100644 index 00000000..4ab9cddc --- /dev/null +++ b/src/services/net/publisher.lua @@ -0,0 +1,168 @@ +-- services/net/publisher.lua +-- Immediate retained-state publication for NET, with explicit dirty state. + +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local projection = require 'services.net.projection' + +local M = {} + +local DOMAIN_TOPICS = { + 'addressing', 'dns', 'dhcp', 'firewall', 'routing', + 'wan', 'backhaul', 'wan_runtime', 'sources', 'vpn', 'diagnostics', 'observed', 'drift', +} + + +local function publish_map(conn, published, next_map, dirty_map, topic_fn, payload_fn) + local seen = {} + next_map = next_map or {} + for id, rec in pairs(next_map) do + seen[id] = true + if dirty_map == true or (type(dirty_map) == 'table' and dirty_map[id]) or not published[id] then + local ok, err = bus_cleanup.retain(conn, topic_fn(id), payload_fn(rec)) + if ok ~= true then return nil, err end + published[id] = true + end + end + for id in pairs(published) do + if not seen[id] then + local ok, err = bus_cleanup.unretain(conn, topic_fn(id)) + if ok ~= true then return nil, err end + published[id] = nil + end + end + return true, nil +end + +function M.new_state() + return { + segments = {}, + interfaces = {}, + domains = {}, + summary = false, + apply = false, + segments_catalogue = false, + vlan_policy = false, + } +end + +function M.new_dirty_state() + return { + summary = false, + apply = false, + segments = {}, + interfaces = {}, + domains = {}, + segments_catalogue = false, + vlan_policy = false, + } +end + +function M.mark_all(dirty) + dirty.summary = true + dirty.apply = true + dirty.segments = true + dirty.interfaces = true + dirty.domains = true + dirty.segments_catalogue = true + dirty.vlan_policy = true + return dirty +end + +function M.mark_summary(dirty) dirty.summary = true; return dirty end +function M.mark_apply(dirty) dirty.apply = true; dirty.summary = true; return dirty end +function M.mark_segment(dirty, id) if dirty.segments ~= true then dirty.segments[id] = true end; dirty.segments_catalogue = true; dirty.summary = true; return dirty end +function M.mark_interface(dirty, id) if dirty.interfaces ~= true then dirty.interfaces[id] = true end; dirty.summary = true; return dirty end +function M.mark_domain(dirty, name) if dirty.domains ~= true then dirty.domains[name] = true end; dirty.summary = true; return dirty end + +function M.clear_dirty(dirty) + dirty.summary = false + dirty.apply = false + dirty.segments = {} + dirty.interfaces = {} + dirty.domains = {} + dirty.segments_catalogue = false + dirty.vlan_policy = false + return dirty +end + +function M.publish_dirty_now(conn, snapshot, dirty, published) + published = published or M.new_state() + dirty = dirty or M.mark_all(M.new_dirty_state()) + + local ok, err + if dirty.summary or not published.summary then + ok, err = bus_cleanup.retain(conn, projection.summary_topic(), projection.summary(snapshot)) + if ok ~= true then return nil, err end + published.summary = true + end + + if dirty.apply or not published.apply then + ok, err = bus_cleanup.retain(conn, projection.apply_topic(), projection.apply(snapshot)) + if ok ~= true then return nil, err end + published.apply = true + end + + ok, err = publish_map(conn, published.segments, snapshot.segments, dirty.segments, projection.segment_topic, projection.segment) + if ok ~= true then return nil, err end + + if dirty.segments_catalogue or not published.segments_catalogue then + ok, err = bus_cleanup.retain(conn, projection.segments_topic(), projection.segments(snapshot)) + if ok ~= true then return nil, err end + published.segments_catalogue = true + end + + if dirty.vlan_policy or not published.vlan_policy then + ok, err = bus_cleanup.retain(conn, projection.vlan_policy_topic(), projection.vlan_policy(snapshot)) + if ok ~= true then return nil, err end + published.vlan_policy = true + end + + ok, err = publish_map(conn, published.interfaces, snapshot.interfaces, dirty.interfaces, projection.interface_topic, projection.interface) + if ok ~= true then return nil, err end + + local dirty_domains = dirty.domains + for i = 1, #DOMAIN_TOPICS do + local name = DOMAIN_TOPICS[i] + if dirty_domains == true or (type(dirty_domains) == 'table' and dirty_domains[name]) or not published.domains[name] then + ok, err = bus_cleanup.retain(conn, projection.domain_topic(name), projection.domain(snapshot, name)) + if ok ~= true then return nil, err end + published.domains[name] = true + end + end + + M.clear_dirty(dirty) + return true, nil +end + +function M.publish_all_now(conn, snapshot, published) + local dirty = M.mark_all(M.new_dirty_state()) + return M.publish_dirty_now(conn, snapshot, dirty, published) +end + +function M.cleanup_now(conn, published) + if not published then return true, nil end + + for id in pairs(published.segments or {}) do + bus_cleanup.unretain(conn, projection.segment_topic(id)) + published.segments[id] = nil + end + for id in pairs(published.interfaces or {}) do + bus_cleanup.unretain(conn, projection.interface_topic(id)) + published.interfaces[id] = nil + end + for name in pairs(published.domains or {}) do + bus_cleanup.unretain(conn, projection.domain_topic(name)) + published.domains[name] = nil + end + if published.summary then bus_cleanup.unretain(conn, projection.summary_topic()) end + if published.apply then bus_cleanup.unretain(conn, projection.apply_topic()) end + if published.segments_catalogue then bus_cleanup.unretain(conn, projection.segments_topic()) end + if published.vlan_policy then bus_cleanup.unretain(conn, projection.vlan_policy_topic()) end + published.summary = false + published.apply = false + published.segments_catalogue = false + published.vlan_policy = false + return true, nil +end + +return M diff --git a/src/services/net/schema.lua b/src/services/net/schema.lua new file mode 100644 index 00000000..d5f9ddf3 --- /dev/null +++ b/src/services/net/schema.lua @@ -0,0 +1,172 @@ +-- services/net/schema.lua +-- Strict product-level cfg/net schema helpers. +-- +-- This module is pure. It defines the NET configuration contract and small +-- normalisation helpers used by the domain modules. Compatibility with older +-- shapes must not be added here; migration belongs outside the NET service. + +local tablex = require 'shared.table' + +local M = {} + +M.CONFIG_SCHEMA = 'devicecode.config/net/1' +M.INTENT_SCHEMA = 'devicecode.net.intent/1' +M.DEFAULT_VERSION = 1 + +local ID_PATTERN = '^[%w][%w%._%-]*$' + +function M.copy(v) + return tablex.deep_copy(v) +end + +function M.is_plain_table(v) + return type(v) == 'table' and getmetatable(v) == nil +end + +function M.path(path) + if type(path) == 'table' then + local out = {} + for i = 1, #path do out[i] = tostring(path[i]) end + return table.concat(out, '.') + end + return tostring(path or 'value') +end + +function M.err(path, message) + return ('%s: %s'):format(M.path(path), tostring(message)) +end + +function M.require_plain_table(v, path) + if not M.is_plain_table(v) then + return nil, M.err(path, 'must be a plain table') + end + return v, nil +end + +function M.optional_plain_table(v, path) + if v == nil then return {}, nil end + return M.require_plain_table(v, path) +end + +function M.id(v, path) + if type(v) ~= 'string' or v == '' then + return nil, M.err(path, 'must be a non-empty string') + end + if not v:match(ID_PATTERN) then + return nil, M.err(path, 'must contain only letters, digits, underscore, hyphen or dot, and must start with a word character') + end + return v, nil +end + +function M.optional_string(v, path) + if v == nil then return nil, nil end + if type(v) ~= 'string' then return nil, M.err(path, 'must be a string') end + return v, nil +end + +function M.optional_boolean(v, path) + if v == nil then return nil, nil end + if type(v) ~= 'boolean' then return nil, M.err(path, 'must be a boolean') end + return v, nil +end + +function M.optional_number(v, path) + if v == nil then return nil, nil end + if type(v) ~= 'number' then return nil, M.err(path, 'must be a number') end + return v, nil +end + +function M.optional_integer(v, path) + if v == nil then return nil, nil end + if type(v) ~= 'number' or v % 1 ~= 0 then return nil, M.err(path, 'must be an integer') end + return v, nil +end + +function M.string_list(v, path) + if v == nil then return {}, nil end + if type(v) ~= 'table' or not tablex.is_array(v) then + return nil, M.err(path, 'must be an array of strings') + end + local out = {} + for i = 1, #v do + if type(v[i]) ~= 'string' or v[i] == '' then + return nil, M.err({ M.path(path), i }, 'must be a non-empty string') + end + out[i] = v[i] + end + return out, nil +end + +function M.id_list(v, path) + local list, err = M.string_list(v, path) + if not list then return nil, err end + for i = 1, #list do + local _, ierr = M.id(list[i], { M.path(path), i }) + if ierr then return nil, ierr end + end + return list, nil +end + +function M.map(v, path, item_fn) + if v == nil then return {}, nil end + if not M.is_plain_table(v) then + return nil, M.err(path, 'must be a map keyed by id') + end + local keys = tablex.sorted_keys(v) + -- In Lua, an empty table can satisfy both array-like and map-like + -- predicates. For configuration boundaries, {} is a valid empty map; + -- reject only non-empty array-shaped values. + if #keys > 0 and tablex.is_array(v) then + return nil, M.err(path, 'must be a map keyed by id') + end + local out = {} + for i = 1, #keys do + local id = keys[i] + local sid, ierr = M.id(tostring(id), { M.path(path), id }) + if not sid then return nil, ierr end + local rec, rerr = item_fn(sid, v[id], { M.path(path), sid }) + if not rec then return nil, rerr end + out[sid] = rec + end + return out, nil +end + +function M.copy_table_or_empty(v, path) + if v == nil then return {}, nil end + if not M.is_plain_table(v) then return nil, M.err(path, 'must be a plain table') end + return M.copy(v), nil +end + +function M.copy_optional_table(v, path) + if v == nil then return nil, nil end + if not M.is_plain_table(v) then return nil, M.err(path, 'must be a plain table') end + return M.copy(v), nil +end + +function M.check_allowed_fields(t, allowed, path) + if not M.is_plain_table(t) then return nil, M.err(path, 'must be a plain table') end + local ok = {} + for i = 1, #allowed do ok[allowed[i]] = true end + for k in pairs(t) do + if not ok[k] then return nil, M.err({ M.path(path), k }, 'field is not part of devicecode.config/net/1') end + end + return true, nil +end + +function M.count_map(t) + local n = 0 + for _ in pairs(t or {}) do n = n + 1 end + return n +end + +function M.with_optional_extensions(out, rec, path) + local metadata, merr = M.copy_optional_table(rec.metadata, { M.path(path), 'metadata' }) + if merr then return nil, merr end + local extensions, eerr = M.copy_optional_table(rec.extensions, { M.path(path), 'extensions' }) + if eerr then return nil, eerr end + out.metadata = metadata + out.extensions = extensions + return out, nil +end + +return M diff --git a/src/services/net/service.lua b/src/services/net/service.lua new file mode 100644 index 00000000..c2f1e07e --- /dev/null +++ b/src/services/net/service.lua @@ -0,0 +1,1254 @@ +-- services/net/service.lua +-- NET service coordinator. +-- +-- NET owns product-level network intent and publication. HAL owns host/network +-- mechanics. The coordinator has one suspending control point; blocking work is +-- admitted through scoped components. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' + +local service_base = require 'devicecode.service_base' +local config_watch = require 'devicecode.support.config_watch' +local queue = require 'devicecode.support.queue' + +local model_mod = require 'services.net.model' +local config_mod = require 'services.net.config' +local events = require 'services.net.events' +local publisher = require 'services.net.publisher' +local generation_mod = require 'services.net.generation' +local apply_runtime = require 'services.net.apply_runtime' +local stale = require 'services.net.stale' +local hal_client_mod = require 'services.net.hal_client' +local cap_deps_mod = require 'devicecode.support.capability_dependencies' +local observer_manager = require 'services.net.observer_manager' +local wan_manager = require 'services.net.wan_manager' +local metrics = require 'services.net.metrics' +local drift = require 'services.net.drift' +local backpressure = require 'services.net.backpressure' +local gsm_uplink_watch = require 'services.net.gsm_uplink_watch' +local intent_realiser = require 'services.net.intent_realiser' +local backhaul_model = require 'services.net.backhaul_model' + +local perform = fibers.perform + +local M = {} + +local function now() return fibers.now() end + +local function elapsed_ms(t0) + if not t0 then return nil end + return math.floor(((now() - t0) * 1000) + 0.5) +end + +local function new_service_id(name) + return tostring(name or 'net') +end + +local function obs_log(svc, level, payload) + if svc and type(svc.obs_log) == 'function' then svc:obs_log(level, payload) end +end + +local function obs_event(svc, kind, payload) + if svc and type(svc.obs_event) == 'function' then + payload = payload or {} + payload.kind = payload.kind or kind + svc:obs_event(kind, payload) + end +end + +local function set_status(svc, state, fields) + if svc and type(svc.status) == 'function' then svc:status(state, fields) end +end + +local function mark_all_dirty(state) + publisher.mark_all(state.dirty) +end + +local function mark_domain_dirty(state, name) + publisher.mark_domain(state.dirty, name) +end + +local function mark_interface_dirty(state, id) + if id ~= nil then publisher.mark_interface(state.dirty, tostring(id)) end +end + +local function mark_apply_dirty(state) + publisher.mark_apply(state.dirty) +end + +local function mark_summary_dirty(state) + publisher.mark_summary(state.dirty) +end + +local project_dependencies + +local function publish_snapshot(state) + if not state.conn then + publisher.clear_dirty(state.dirty) + return true, nil + end + return publisher.publish_dirty_now(state.conn, state.model:snapshot(), state.dirty, state.published) +end + +local function speedtests_blocked_by_apply(state) + if state.active_apply ~= nil or state.pending_intent ~= nil then return true end + local snap = state.model and state.model:snapshot() or nil + local apply_state = snap and snap.apply and snap.apply.state or nil + return apply_state == 'running' or apply_state == 'waiting_for_hal' +end + +local function reconcile_speedtests_if_ready(state, reason) + if speedtests_blocked_by_apply(state) then + local snap = state.model and state.model:snapshot() or nil + obs_log(state.svc, 'debug', { + what = 'speedtests_deferred', + reason = reason, + apply_state = snap and snap.apply and snap.apply.state or nil, + }) + return true, nil + end + return wan_manager.reconcile_speedtests(state, reason) +end + + +local function sorted_keys(t) + local keys = {} + for k in pairs(t or {}) do keys[#keys + 1] = tostring(k) end + table.sort(keys) + return keys +end + +local function latest_successful_speedtest(snapshot) + local tests = snapshot and snapshot.wan_runtime and snapshot.wan_runtime.speedtests or {} + local best = nil + for id, rec in pairs(tests or {}) do + if type(rec) == 'table' and rec.ok == true and rec.peak_mbps ~= nil then + if not best or (tonumber(rec.completed_at) or 0) > (tonumber(best.completed_at) or 0) then + best = { id = id, rec = rec, completed_at = tonumber(rec.completed_at) or 0 } + end + end + end + return best +end + +local function log_net_summary(state, reason) + if not state.svc or not state.model then return end + local snap = state.model:snapshot() + local parts = {} + local wan_iface = snap.interfaces and snap.interfaces.wan or nil + local endpoint = type(wan_iface) == 'table' and type(wan_iface.endpoint) == 'table' and wan_iface.endpoint or {} + local latest = latest_successful_speedtest(snap) + local wan_state = (wan_iface and wan_iface.state) or nil + if not wan_state and latest and tostring(latest.id) == 'wan' and latest.rec and latest.rec.ok == true then wan_state = 'online' end + if not wan_state and endpoint.device then wan_state = 'configured' end + if wan_state then parts[#parts + 1] = 'wan=' .. tostring(wan_state) end + if endpoint.device or (latest and latest.rec and latest.rec.device) then parts[#parts + 1] = 'device=' .. tostring(endpoint.device or latest.rec.device) end + if endpoint.address or endpoint.ipaddr then parts[#parts + 1] = 'addr=' .. tostring(endpoint.address or endpoint.ipaddr) end + if latest then parts[#parts + 1] = string.format('speedtest=%s:%.1fMbps', tostring(latest.id), tonumber(latest.rec.peak_mbps) or 0) end + local last_apply = snap.wan_runtime and snap.wan_runtime.last_weight_apply or nil + if last_apply and type(last_apply.detail) == 'table' then + local detail = last_apply.detail + local eff, skipped = {}, {} + for _, m in ipairs(detail.members or {}) do eff[#eff + 1] = tostring(m.semantic_interface or m.interface or m.id or '?') end + for _, m in ipairs(detail.skipped_members or {}) do skipped[#skipped + 1] = tostring(m.semantic_interface or m.interface or m.id or '?') end + if #eff > 0 then parts[#parts + 1] = 'effective=' .. table.concat(eff, ',') end + if #skipped > 0 then parts[#parts + 1] = 'skipped=' .. table.concat(skipped, ',') end + end + local backhaul = snap.backhaul and snap.backhaul.uplinks or {} + local backhaul_parts = {} + for _, uplink_id in ipairs(sorted_keys(backhaul)) do + local rec = backhaul[uplink_id] + if type(rec) == 'table' then + backhaul_parts[#backhaul_parts + 1] = tostring(uplink_id) .. '=' .. tostring(rec.state or 'unknown') + end + end + if #backhaul_parts > 0 then parts[#parts + 1] = 'backhaul=' .. table.concat(backhaul_parts, ',') end + local summary = 'net summary ' .. table.concat(parts, ' ') + local tnow = now() + if state.operator_net_summary_key == summary and (tnow - (state.operator_net_summary_at or 0)) < 600 then return end + state.operator_net_summary_key = summary + state.operator_net_summary_at = tnow + obs_log(state.svc, 'info', { what = 'net_summary', summary = summary, reason = reason }) +end + + +local function backhaul_uplinks(backhaul) + local uplinks = type(backhaul) == 'table' and backhaul.uplinks or nil + return type(uplinks) == 'table' and uplinks or {} +end + +local function log_mwan_backhaul_transitions(state, before, after, trigger) + if not state.svc then return end + local prev = backhaul_uplinks(before) + local next_uplinks = backhaul_uplinks(after) + for _, uplink_id in ipairs(sorted_keys(next_uplinks)) do + local rec = next_uplinks[uplink_id] + if type(rec) == 'table' then + local source = type(rec.status_source) == 'table' and rec.status_source or rec.source + local tool = type(source) == 'table' and source.tool or nil + local prev_rec = prev[uplink_id] + local prev_state = type(prev_rec) == 'table' and prev_rec.state or nil + local state_now = rec.state + if (state_now == 'online' or state_now == 'offline') and state_now ~= prev_state and (tool == nil or tool == 'mwan3') then + obs_log(state.svc, 'info', { + what = state_now == 'online' and 'mwan_member_online' or 'mwan_member_offline', + summary = string.format('mwan member %s %s interface=%s ifname=%s', tostring(uplink_id), tostring(state_now), tostring(rec.interface or '?'), tostring(rec.ifname or '?')), + uplink_id = tostring(uplink_id), + interface = rec.interface, + ifname = rec.ifname, + state = state_now, + usable = rec.usable == true, + previous_state = prev_state, + subject = trigger and trigger.subject or nil, + source = trigger and trigger.source or nil, + }) + end + end + end +end + +local function set_model_state(state, service_state, reason) + state.model:update(function (s) + s.state = service_state + s.ready = service_state == 'running' + s.reason = reason + return project_dependencies(state, s) + end) + mark_summary_dirty(state) + return publish_snapshot(state) +end + +local function dependency_status(state, key) + return state.cap_deps:status(key) +end + +local function dependency_effectively_available(state, key) + return state.cap_deps:available(key) +end + +project_dependencies = function(state, s) + s.dependencies = state.cap_deps:snapshot() + return s +end + +local function set_pending_apply_projection(s, intent, reason) + s.pending = s.pending or {} + if intent == nil then + s.pending.network_apply = nil + return s + end + s.pending.network_apply = { + kind = 'network_apply', + generation = intent.generation, + rev = intent.rev, + reason = reason or 'network_config_unavailable', + } + return s +end + +local function normalise_config_event(state, ev) + local generation = state.next_generation + local intent, err = config_mod.normalise(ev.payload, { rev = ev.rev, generation = generation }) + if not intent then return nil, err end + return intent, nil +end + +local function cancel_active_runtime(state, reason) + wan_manager.cancel(state, reason or 'generation_replaced') + return true, nil +end + +local function cancel_active_generation(state, reason) + cancel_active_runtime(state, reason or 'generation_replaced') + local active = state.current_generation + if not active then return true, nil end + generation_mod.cancel(active, reason or 'generation_replaced') + state.current_generation = nil + state.active_apply = nil + state.pending_intent = nil + state.pending_apply_reason = nil + return true, nil +end + + +local function backhaul_for_interface(backhaul, uplink_id, iface_id) + local uplinks = backhaul and backhaul.uplinks or nil + if type(uplinks) ~= 'table' then return nil end + local direct = uplinks[tostring(uplink_id)] + if type(direct) == 'table' then return direct end + for id, rec in pairs(uplinks) do + if type(rec) == 'table' and (rec.interface == iface_id or rec.id == iface_id or id == iface_id) then + return rec + end + end + return nil +end + +local function apply_backhaul_to_interfaces(s) + local members = s and s.wan and s.wan.configured_members or {} + local interfaces = s and s.interfaces or nil + if type(interfaces) ~= 'table' then return s end + for uplink_id, member in pairs(members or {}) do + if type(member) == 'table' and member.enabled ~= false and member.disabled ~= true then + local iface_id = member.interface or uplink_id + local iface = interfaces[iface_id] + local bh = backhaul_for_interface(s.backhaul, uplink_id, iface_id) + if type(iface) == 'table' and type(bh) == 'table' then + -- mwan3/backhaul is the semantic authority for WAN reachability. + -- Interface config and speedtests enrich this record; they must not + -- make an mwan3-online uplink appear offline to consumers. + iface.state = bh.state + iface.status = bh.state + iface.online = bh.state == 'online' and bh.usable == true + iface.usable = bh.usable == true + iface.backhaul = { + state = bh.state, + usable = bh.usable == true, + ifname = bh.ifname, + path_address = model_mod.deep_copy(bh.path_address), + source = model_mod.deep_copy(bh.source), + observed_at = bh.observed_at, + } + end + end + end + return s +end + +local function mark_wan_member_interfaces_dirty(state) + local snap = state and state.model and state.model:snapshot() or nil + local members = snap and snap.wan and snap.wan.configured_members or {} + for uplink_id, member in pairs(members or {}) do + if type(member) == 'table' and member.enabled ~= false and member.disabled ~= true then + mark_interface_dirty(state, member.interface or uplink_id) + end + end +end + +local function carry_forward_wan_runtime(previous, intent, generation) + local out = { generation = generation, uplinks = {}, speedtests = {}, live_weights = { state = 'idle', generation = generation } } + if type(previous) ~= 'table' then return out end + local members = intent and intent.wan and intent.wan.members or {} + for uplink_id, member in pairs(members or {}) do + if type(member) == 'table' and member.enabled ~= false and member.disabled ~= true then + local rec = previous.speedtests and previous.speedtests[tostring(uplink_id)] or nil + if type(rec) == 'table' then + local kept = model_mod.deep_copy(rec) + if kept.state == 'running' then + kept.state = kept.last_success and 'ok' or 'skipped' + kept.ok = kept.last_success ~= nil + kept.last_skip_reason = kept.last_skip_reason or 'generation_replaced' + end + out.speedtests[tostring(uplink_id)] = kept + end + end + end + out.last_weight_apply = model_mod.deep_copy(previous.last_weight_apply) + return out +end + +local function copy_wan_catalogue_and_realisation(configured_wan, realised_wan) + local wan = model_mod.deep_copy(configured_wan or {}) + wan.configured_members = model_mod.deep_copy((configured_wan and configured_wan.members) or {}) + wan.realised_members = model_mod.deep_copy((realised_wan and realised_wan.members) or {}) + wan.members = nil + return wan +end + +local function copy_intent_to_model(s, apply_intent, configured_intent) + configured_intent = configured_intent or apply_intent + local generation = configured_intent.generation or apply_intent.generation + s.generation = generation + s.config = { + rev = configured_intent.rev, + schema = configured_intent.schema, + config_schema = configured_intent.config_schema, + version = configured_intent.version, + } + s.intent = s.intent or {} + s.intent.generation = generation + s.intent.realised_generation = apply_intent.generation + s.segments = apply_intent.segments or {} + s.vlan_policy = apply_intent.vlan_policy or {} + s.policies = apply_intent.policies or {} + s.interfaces = apply_intent.interfaces or {} + s.addressing = apply_intent.addressing or {} + s.dns = apply_intent.dns or {} + s.dhcp = apply_intent.dhcp or {} + s.firewall = apply_intent.firewall or {} + s.routing = apply_intent.routing or {} + -- Keep the product-level WAN member catalogue stable in the public NET + -- model. The OpenWrt/HAL apply intent may legitimately omit GSM-backed + -- members until a Linux ifname exists; that volatility must not remove + -- state/net/backhaul.uplinks or local-UI cards. The realised set is kept + -- separately for diagnostics and apply/runtime policy that needs it. + s.wan = copy_wan_catalogue_and_realisation(configured_intent.wan, apply_intent.wan) + s.sources = s.sources or { gsm_uplinks = {} } + s.backhaul = backhaul_model.reduce(s, { now = now() }) + apply_backhaul_to_interfaces(s) + s.wan_runtime = carry_forward_wan_runtime(s.wan_runtime, configured_intent, generation) + s.vpn = apply_intent.vpn or {} + s.diagnostics = apply_intent.diagnostics or {} + return s +end + + +local function realised_sources(state) + local snap = state.model and state.model:snapshot() or {} + return snap.sources or { gsm_uplinks = {} } +end + +local function realise_intent(state, intent) + return intent_realiser.realise(intent, realised_sources(state)) +end + +local function realised_fingerprint(state, intent) + return intent_realiser.realised_fingerprint(intent, realised_sources(state)) +end + +local function accept_pending_intent(state, intent, reason) + reason = reason or 'network_config_unavailable' + local apply_intent = realise_intent(state, intent) + local gen = generation_mod.new(apply_intent.generation, apply_intent, reason, { now = now() }) + gen.state = 'waiting_for_hal' + gen.reason = reason + gen.apply = { state = 'waiting_for_hal', reason = reason } + + state.current_generation = gen + state.active_apply = nil + state.pending_intent = intent + state.pending_apply_reason = reason + state.base_intent = intent + state.last_realised_source_fingerprint = realised_fingerprint(state, intent) + + state.model:update(function (s) + copy_intent_to_model(s, apply_intent, intent) + s.state = 'waiting_for_hal' + s.ready = false + s.reason = reason + s.intent = s.intent or {} + s.intent.active = generation_mod.snapshot(gen) + set_pending_apply_projection(s, intent, reason) + s.apply = { + state = 'waiting_for_hal', + generation = intent.generation, + apply_id = nil, + started_at = nil, + last_applied_rev = s.apply and s.apply.last_applied_rev or nil, + last_error = reason, + last_result = nil, + } + s.stats.config_updates = (s.stats.config_updates or 0) + 1 + return project_dependencies(state, s) + end) + mark_all_dirty(state) + return true, nil +end + +local function start_apply_for_intent(state, intent, reason) + local apply_intent = realise_intent(state, intent) + state.base_intent = intent + state.last_realised_source_fingerprint = realised_fingerprint(state, intent) + local generation = apply_intent.generation + local apply_id = state.next_apply_id + state.next_apply_id = apply_id + 1 + + local gen = state.current_generation + if not gen or gen.generation ~= generation then + gen = generation_mod.new(generation, apply_intent, reason, { now = now() }) + end + generation_mod.start_apply(gen, apply_id, { now = now() }) + state.current_generation = gen + state.active_apply = { generation = generation, apply_id = apply_id, rev = intent.rev } + state.pending_intent = nil + state.pending_apply_reason = nil + + state.model:update(function (s) + copy_intent_to_model(s, apply_intent, intent) + s.state = 'applying' + s.ready = false + s.reason = nil + s.intent = s.intent or {} + s.intent.active = generation_mod.snapshot(gen) + set_pending_apply_projection(s, nil) + s.apply = { + state = 'running', + generation = generation, + apply_id = apply_id, + started_at = now(), + last_applied_rev = s.apply and s.apply.last_applied_rev or nil, + last_error = nil, + last_result = nil, + } + s.stats.apply_started = (s.stats.apply_started or 0) + 1 + return project_dependencies(state, s) + end) + mark_all_dirty(state) + local ok_pub, pub_err = publish_snapshot(state) + if ok_pub ~= true then return nil, pub_err end + + obs_event(state.svc, 'apply_started', { generation = generation, apply_id = apply_id, rev = apply_intent.rev, reason = reason }) + obs_log(state.svc, 'info', { + what = 'apply_started', + summary = string.format('network apply %s started generation=%s reason=%s', tostring(apply_id), tostring(generation), tostring(reason or 'unknown')), + generation = generation, + apply_id = apply_id, + reason = reason, + }) + + local handle, err = apply_runtime.start_apply { + lifetime_scope = state.scope, + reaper_scope = state.scope, + report_scope = state.scope, + service_id = state.service_id, + generation = generation, + apply_id = apply_id, + intent = apply_intent, + hal = state.hal, + done_tx = state.done_tx, + } + if not handle then + generation_mod.mark_failed(gen, err or 'apply_start_failed', nil, { now = now() }) + state.active_apply = nil + state.model:update(function (s) + s.intent.active = generation_mod.snapshot(gen) + s.apply.state = 'failed_to_start' + s.apply.last_error = tostring(err or 'apply_start_failed') + return project_dependencies(state, s) + end) + mark_apply_dirty(state) + return nil, err + end + + gen.apply_handle = handle + return true, nil +end + +local function reconcile_apply_admission(state, reason) + if state.active_apply ~= nil then return true, nil end + local intent = state.pending_intent + if not intent then return true, nil end + if not dependency_effectively_available(state, 'network_config') then + local wait_reason = state.pending_apply_reason or 'network_config_unavailable' + set_status(state.svc, 'waiting_for_hal', { reason = wait_reason }) + return publish_snapshot(state) + end + return start_apply_for_intent(state, intent, reason or state.pending_apply_reason or 'network_config_available') +end + +local function handle_config_changed(state, ev) + local intent, err = normalise_config_event(state, ev) + if not intent then + obs_log(state.svc, 'warn', { what = 'config_rejected', err = tostring(err) }) + state.model:update(function (s) + s.state = 'degraded' + s.ready = false + s.reason = tostring(err) + s.intent = s.intent or {} + s.intent.last_rejected = { rev = ev.rev, err = tostring(err), rejected_at = now() } + return project_dependencies(state, s) + end) + mark_summary_dirty(state) + return publish_snapshot(state) + end + + state.next_generation = intent.generation + 1 + state.base_intent = intent + cancel_active_generation(state, 'config_replaced') + obs_event(state.svc, 'config_accepted', { + rev = intent.rev, + generation = intent.generation, + segments = intent.stats and intent.stats.segments or nil, + interfaces = intent.stats and intent.stats.interfaces or nil, + }) + + local ok, apply_err = accept_pending_intent(state, intent, 'network_config_unavailable') + if ok ~= true then return nil, apply_err end + + ok, apply_err = reconcile_apply_admission(state, 'config_changed') + if ok ~= true then + state.model:update(function (s) + s.state = 'degraded' + s.ready = false + s.reason = tostring(apply_err or 'apply_start_failed') + return project_dependencies(state, s) + end) + mark_summary_dirty(state) + publish_snapshot(state) + return nil, apply_err or 'net apply start failed' + end + return true, nil +end + +local function classify_network_config_apply_failure(state, result) + local failure = result and result.result or result + return state.cap_deps:classify_call_failure('network_config', failure) +end + +local function return_apply_to_pending(state, result, reason) + reason = reason or 'network_config_unavailable' + local gen = state.current_generation + local apply_intent = gen and gen.intent or nil + local intent = state.base_intent or apply_intent + state.active_apply = nil + if not intent then return set_model_state(state, 'waiting_for_hal', reason) end + + gen.state = 'waiting_for_hal' + gen.reason = reason + gen.apply = gen.apply or {} + gen.apply.state = 'waiting_for_hal' + gen.apply.reason = reason + gen.apply.result = result + + state.pending_intent = intent + state.pending_apply_reason = reason + + state.model:update(function (s) + s.state = 'waiting_for_hal' + s.ready = false + s.reason = reason + s.intent = s.intent or {} + s.intent.active = generation_mod.snapshot(gen) + set_pending_apply_projection(s, intent, reason) + s.apply = s.apply or {} + s.apply.state = 'waiting_for_hal' + s.apply.completed_at = now() + s.apply.last_result = result and result.result or result + s.apply.last_error = reason + s.stats.apply_completed = (s.stats.apply_completed or 0) + 1 + return project_dependencies(state, s) + end) + mark_apply_dirty(state) + set_status(state.svc, 'waiting_for_hal', { reason = reason }) + obs_event(state.svc, 'apply_deferred', { generation = gen.generation, reason = reason }) + return publish_snapshot(state) +end + +local function handle_apply_done(state, ev) + if not stale.apply_current(state, ev) then + stale.reject(state, ev) + mark_summary_dirty(state) + return publish_snapshot(state) + end + + local result = ev.result or {} + local apply_ok = ev.status == 'ok' and result.ok == true + local reason = nil + if not apply_ok then reason = ev.primary or (result.result and result.result.err) or 'apply_failed' end + + if not apply_ok then + local failure_class = classify_network_config_apply_failure(state, result) + if failure_class == 'route_missing' then + return return_apply_to_pending(state, result, 'network_config_unavailable') + end + end + + state.active_apply = nil + if state.current_generation then + if apply_ok then generation_mod.mark_applied(state.current_generation, result, { now = now() }) + else generation_mod.mark_failed(state.current_generation, reason, result, { now = now() }) end + end + + state.model:update(function (s) + s.state = apply_ok and 'running' or 'degraded' + s.ready = apply_ok + s.reason = reason + s.intent = s.intent or {} + s.intent.active = generation_mod.snapshot(state.current_generation) + set_pending_apply_projection(s, nil) + s.apply.state = apply_ok and 'applied' or 'failed' + s.apply.completed_at = now() + s.apply.last_result = result.result + s.apply.last_error = reason + if apply_ok then s.apply.last_applied_rev = result.intent_rev or (s.config and s.config.rev) end + s.stats.apply_completed = (s.stats.apply_completed or 0) + 1 + return project_dependencies(state, s) + end) + + mark_apply_dirty(state) + set_status(state.svc, apply_ok and 'running' or 'degraded', reason and { reason = reason } or nil) + local started_at = state.model:snapshot().apply and state.model:snapshot().apply.started_at or nil + local apply_elapsed_ms = elapsed_ms(started_at) + obs_event(state.svc, 'apply_completed', { generation = ev.generation, apply_id = ev.apply_id, ok = apply_ok, reason = reason, elapsed_ms = apply_elapsed_ms }) + obs_log(state.svc, apply_ok and 'info' or 'warn', { + what = apply_ok and 'apply_completed' or 'apply_failed', + summary = apply_ok + and string.format('network apply %s completed in %.1fs', tostring(ev.apply_id), (apply_elapsed_ms or 0) / 1000) + or string.format('network apply %s failed: %s', tostring(ev.apply_id), tostring(reason or 'unknown')), + generation = ev.generation, + apply_id = ev.apply_id, + ok = apply_ok, + reason = reason, + elapsed_ms = apply_elapsed_ms, + }) + + local ok_pub, pub_err = publish_snapshot(state) + if ok_pub ~= true then return nil, pub_err end + if apply_ok then + local ok, err = wan_manager.reconcile_speedtests(state, 'apply_done') + mark_domain_dirty(state, 'wan_runtime') + publish_snapshot(state) + return ok, err + end + return true, nil +end + +local function merge_observation(state, s, observed_event) + local observed = observed_event and observed_event.observed or nil + s.observed = s.observed or { interfaces = {}, segments = {} } + s.observed.last_event = observed_event + s.observed.last_event_at = now() + s.observed.last_subject = observed_event and observed_event.subject or nil + + if type(observed) == 'table' then + if type(observed.interfaces) == 'table' then s.observed.interfaces = model_mod.deep_copy(observed.interfaces) end + if type(observed.segments) == 'table' then s.observed.segments = model_mod.deep_copy(observed.segments) end + s.observed.snapshot = model_mod.deep_copy(observed) + elseif observed_event and observed_event.subject and observed_event.subject:match('^interface:') and type(observed_event.interface) == 'table' then + local id = observed_event.subject:match('^interface:(.+)$') + if id then s.observed.interfaces[id] = model_mod.deep_copy(observed_event.interface) end + end + + s.stats.observations = (s.stats.observations or 0) + 1 + s.drift = drift.calculate(s, { now = now }) + s.backhaul = backhaul_model.reduce(s, { now = now() }) + apply_backhaul_to_interfaces(s) + return project_dependencies(state, s) +end + +local function handle_observed_state(state, ev) + local observed_event = ev and ev.event or nil + if type(observed_event) ~= 'table' then return true, nil end + local before = state.model and state.model:snapshot().backhaul or nil + state.model:update(function (s) return merge_observation(state, s, observed_event) end) + local after = state.model and state.model:snapshot().backhaul or nil + log_mwan_backhaul_transitions(state, before, after, observed_event) + mark_domain_dirty(state, 'observed') + mark_domain_dirty(state, 'backhaul') + mark_wan_member_interfaces_dirty(state) + mark_domain_dirty(state, 'drift') + obs_event(state.svc, 'network_observed', { subject = observed_event.subject, source = observed_event.source, kind = observed_event.kind }) + local ok, err = reconcile_speedtests_if_ready(state, 'observed_state') + mark_domain_dirty(state, 'wan_runtime') + local pub_ok, pub_err = publish_snapshot(state) + if pub_ok ~= true then return nil, pub_err end + return ok, err +end + +local function handle_gsm_uplink_changed(state, ev) + if ev.kind == 'gsm_uplink_replay_done' then return true, nil end + if ev.kind == 'gsm_uplink_unknown' then + obs_log(state.svc, 'debug', { what = 'gsm_uplink_unknown', event = ev.event }) + return true, nil + end + local role = ev.role + if type(role) ~= 'string' or role == '' then return true, nil end + local payload = model_mod.deep_copy(ev.payload or {}) + payload.schema = payload.schema or 'devicecode.gsm.uplink/1' + payload.id = payload.id or role + payload.role = payload.role or role + payload.updated_at = now() + state.model:update(function (s) + s.sources = s.sources or { gsm_uplinks = {} } + s.sources.gsm_uplinks = s.sources.gsm_uplinks or {} + s.sources.gsm_uplinks[role] = payload + s.backhaul = backhaul_model.reduce(s, { now = now() }) + apply_backhaul_to_interfaces(s) + s.stats.gsm_uplink_updates = (s.stats.gsm_uplink_updates or 0) + 1 + return project_dependencies(state, s) + end) + mark_domain_dirty(state, 'sources') + mark_domain_dirty(state, 'backhaul') + mark_wan_member_interfaces_dirty(state) + mark_summary_dirty(state) + local ifname = payload.linux and payload.linux.ifname or payload.interface + obs_event(state.svc, 'gsm_uplink_changed', { + role = role, + state = payload.state, + connected = payload.connected == true, + ifname = ifname, + }) + local uplink_key = tostring(role) .. '|' .. tostring(payload.state) .. '|' .. tostring(payload.connected == true) .. '|' .. tostring(ifname or '') + state._operator_gsm_uplink = state._operator_gsm_uplink or {} + if state._operator_gsm_uplink[role] ~= uplink_key then + state._operator_gsm_uplink[role] = uplink_key + local expected_online = payload.expected_online == true or payload.expected_connected == true + local level = (payload.connected == true or not expected_online) and 'info' or 'warn' + obs_log(state.svc, level, { + what = 'gsm_uplink_changed', + summary = string.format('cellular uplink %s %s%s%s', tostring(role), tostring(payload.state or (payload.connected and 'connected' or 'disconnected')), ifname and (' ifname=' .. tostring(ifname)) or '', expected_online and ' expected=true' or ''), + role = role, + state = payload.state, + connected = payload.connected == true, + expected_online = expected_online or nil, + ifname = ifname, + }) + end + local structural_ok, structural_err = true, nil + if state.base_intent then + local fp = realised_fingerprint(state, state.base_intent) + if fp ~= state.last_realised_source_fingerprint then + local next_intent = model_mod.deep_copy(state.base_intent) + next_intent.generation = state.next_generation + state.next_generation = state.next_generation + 1 + state.base_intent = next_intent + local active_apply = state.active_apply + if active_apply and state.svc then + obs_log(state.svc, 'warn', { + what = 'apply_superseded', + summary = string.format('network apply %s superseded by generation=%s reason=gsm_uplink_binding_changed', tostring(active_apply.apply_id or '?'), tostring(next_intent.generation)), + apply_id = active_apply.apply_id, + generation = active_apply.generation, + next_generation = next_intent.generation, + reason = 'gsm_uplink_binding_changed', + }) + end + cancel_active_generation(state, 'gsm_uplink_binding_changed') + structural_ok, structural_err = accept_pending_intent(state, next_intent, 'gsm_uplink_binding_changed') + if structural_ok == true then structural_ok, structural_err = reconcile_apply_admission(state, 'gsm_uplink_binding_changed') end + end + end + local ok, err = reconcile_speedtests_if_ready(state, 'gsm_uplink_changed') + mark_domain_dirty(state, 'wan_runtime') + local pub_ok, pub_err = publish_snapshot(state) + if pub_ok ~= true then return nil, pub_err end + if structural_ok ~= true then return nil, structural_err end + return ok, err +end + +local function ensure_observer_started(state, reason) + if state.observe == false or state.observer ~= nil then return true, nil end + if not state.hal or type(state.hal.open_observed_subscription) ~= 'function' then return true, nil end + local observer, err = observer_manager.start { + lifetime_scope = state.scope, + reaper_scope = state.scope, + report_scope = state.scope, + service_id = state.service_id, + generation = state.current_generation and state.current_generation.generation or 0, + hal = state.hal, + done_tx = state.done_tx, + queue_len = state.observation_queue_len, + full = state.observation_full, + options = state.observation_options, + } + if not observer then + obs_log(state.svc, 'debug', { what = 'network_observation_not_started', reason = reason, err = tostring(err) }) + state.model:update(function (m) + m.observed.last_error = tostring(err) + return project_dependencies(state, m) + end) + mark_summary_dirty(state) + mark_domain_dirty(state, 'observed') + return true, nil + end + state.observer = observer + state.observed_sub = observer:subscription() + state.model:update(function (m) + return project_dependencies(state, m) + end) + mark_summary_dirty(state) + return true, nil +end + +local function stop_observer(state, reason) + if state.observer then state.observer:terminate(reason or 'network_state_unavailable') end + state.observer = nil + state.observed_sub = nil + return true, nil +end + +local function reconcile_observer_admission(state, reason) + if dependency_effectively_available(state, 'network_state') then + return ensure_observer_started(state, reason or 'network_state_available') + end + return stop_observer(state, reason or ('network_state_' .. tostring(dependency_status(state, 'network_state')))) +end + +local function handle_observation_started(state, ev) + local work = ev.result or {} + local result = work.result or work + local ok = ev.status == 'ok' and work.ok == true and (result.ok == nil or result.ok == true) + state.model:update(function (m) + if not ok then m.observed.last_error = result.err or ev.primary or 'observation_start_failed' end + return project_dependencies(state, m) + end) + if not ok then stop_observer(state, 'observation_start_failed') end + mark_summary_dirty(state) + mark_domain_dirty(state, 'observed') + return publish_snapshot(state) +end + +local function handle_dependency_status(state, ev) + local key = ev.key or ev.capability + local status = key and state.cap_deps:status(key) or ev.status + state.model:update(function (m) + return project_dependencies(state, m) + end) + mark_summary_dirty(state) + if key == 'network_state' then + local ok, err = reconcile_observer_admission(state, 'network_state_' .. tostring(status)) + if ok ~= true then return nil, err end + elseif key == 'network_config' then + if dependency_effectively_available(state, 'network_config') then + local ok, err = reconcile_apply_admission(state, 'network_config_available') + if ok ~= true then return nil, err end + end + end + return publish_snapshot(state) +end + + +local function schedule_speedtest_retry(state, uplink_id, generation, delay_s, reason) + if not state or not state.scope or not state.done_tx then return true, nil end + if type(uplink_id) ~= 'string' or uplink_id == '' then return true, nil end + delay_s = tonumber(delay_s) or 10 + if delay_s < 1 then delay_s = 1 end + state.speedtest_retries = state.speedtest_retries or {} + local existing = state.speedtest_retries[uplink_id] + local tnow = now() + if type(existing) == 'table' and (existing.due_at or 0) > tnow then return true, nil end + local retry_id = state.next_speedtest_retry_id or 1 + state.next_speedtest_retry_id = retry_id + 1 + state.speedtest_retries[uplink_id] = { + retry_id = retry_id, + uplink_id = uplink_id, + generation = generation, + due_at = tnow + delay_s, + reason = reason or 'speedtest_retry', + } + local ok, err = state.scope:spawn(function () + sleep.sleep(delay_s) + queue.assert_admit_required(state.done_tx, { + kind = 'speedtest_retry_due', + uplink_id = uplink_id, + generation = generation, + retry_id = retry_id, + reason = reason or 'speedtest_retry', + }, 'net_speedtest_retry_report_failed') + end) + if ok ~= true then + state.speedtest_retries[uplink_id] = nil + return nil, err or 'speedtest_retry_spawn_failed' + end + obs_log(state.svc, 'debug', { + what = 'speedtest_retry_scheduled', + uplink_id = uplink_id, + generation = generation, + retry_id = retry_id, + delay_s = delay_s, + reason = reason, + }) + return true, nil +end + +local function handle_speedtest_retry_due(state, ev) + local pending = state.speedtest_retries and state.speedtest_retries[ev.uplink_id] or nil + if type(pending) ~= 'table' or pending.retry_id ~= ev.retry_id then return true, nil end + state.speedtest_retries[ev.uplink_id] = nil + obs_log(state.svc, 'debug', { + what = 'speedtest_retry_due', + uplink_id = ev.uplink_id, + generation = ev.generation, + retry_id = ev.retry_id, + reason = ev.reason, + }) + return reconcile_speedtests_if_ready(state, 'speedtest_retry_due') +end + +local function handle_speedtest_done(state, ev) + local ok, err = wan_manager.handle_speedtest_done(state, ev) + if ok == true and err ~= 'stale' then + local snap = state.model:snapshot() + local rec = snap.wan_runtime and snap.wan_runtime.speedtests and snap.wan_runtime.speedtests[ev.uplink_id] + if rec then + local mbps = rec.ok == true and tonumber(rec.peak_mbps) or nil + obs_log(state.svc, rec.ok == true and 'info' or 'warn', { + what = 'speedtest_completed', + summary = rec.ok == true + and string.format('speedtest %s on %s/%s: %.1f Mbps', tostring(ev.uplink_id), tostring(rec.interface or '?'), tostring(rec.device or '?'), mbps or 0) + or string.format('speedtest %s failed: %s', tostring(ev.uplink_id), tostring(rec.err or 'unknown')), + uplink_id = ev.uplink_id, + interface = rec.interface, + device = rec.device, + ok = rec.ok == true, + err = rec.err, + peak_mbps = mbps, + retry_delay_s = rec.retry_delay_s, + retry_after = rec.retry_after, + failure_count = rec.failure_count, + }) + end + if rec and rec.ok == true and rec.peak_mbps ~= nil and state.svc and type(state.svc.obs_metric) == 'function' then + state.svc:obs_metric('speedtest', { + value = rec.peak_mbps, + namespace = { 'net', rec.interface or ev.uplink_id, 'speedtest' }, + }) + end + if rec and rec.ok ~= true and rec.retry_after then + local delay_s = math.max(1, (tonumber(rec.retry_after) or (now() + (tonumber(rec.retry_delay_s) or 10))) - now()) + local rok, rerr = schedule_speedtest_retry(state, tostring(ev.uplink_id or ''), ev.generation, delay_s, rec.err or 'speedtest_failed') + if rok ~= true then return nil, rerr end + end + end + mark_domain_dirty(state, 'wan_runtime') + local pub_ok, pub_err = publish_snapshot(state) + if pub_ok ~= true then return nil, pub_err end + if ok == true and err == 'stale' then return true, nil end + return ok, err +end + +local function handle_live_weights_done(state, ev) + local ok, err = wan_manager.handle_live_weights_done(state, ev) + local result = ev and ev.result and (ev.result.result or ev.result) or {} + if ok == true and err ~= 'stale' then + if result.ok == true then + state._live_weight_failure_log = nil + local detail = type(result.detail) == 'table' and result.detail or {} + local effective = type(detail.members) == 'table' and detail.members or {} + local skipped = type(detail.skipped_members) == 'table' and detail.skipped_members or {} + local effective_names, skipped_names = {}, {} + for i = 1, #effective do + effective_names[#effective_names + 1] = tostring(effective[i].semantic_interface or effective[i].interface or effective[i].id or '?') + end + for i = 1, #skipped do + skipped_names[#skipped_names + 1] = tostring(skipped[i].semantic_interface or skipped[i].interface or skipped[i].id or '?') + end + local suffix = '' + if #skipped_names > 0 then + suffix = ' effective=' .. (#effective_names > 0 and table.concat(effective_names, ',') or '?') + .. ' skipped=' .. table.concat(skipped_names, ',') .. ' reason=no_mwan3_mark' + end + obs_log(state.svc, 'info', { + what = 'live_weights_applied', + summary = string.format('live weights applied generation=%s%s', tostring(ev.generation or '?'), suffix), + generation = ev.generation, + weight_apply_id = ev.weight_apply_id, + skipped_members = skipped, + effective_members = effective, + }) + else + local signature = tostring(ev.generation or '?') .. ':' .. tostring(result.err or 'unknown') + local rec = state._live_weight_failure_log + local tnow = now() + if not rec or rec.signature ~= signature then + rec = { signature = signature, repeats = 0, first_at = tnow, last_logged_at = tnow } + state._live_weight_failure_log = rec + obs_log(state.svc, 'warn', { + what = 'live_weights_failed', + summary = string.format('live weights failed generation=%s: %s', tostring(ev.generation or '?'), tostring(result.err or 'unknown')), + generation = ev.generation, + weight_apply_id = ev.weight_apply_id, + err = result.err, + }) + else + rec.repeats = (rec.repeats or 0) + 1 + if tnow - (rec.last_logged_at or rec.first_at or tnow) >= 60 then + rec.last_logged_at = tnow + obs_log(state.svc, 'warn', { + what = 'live_weights_still_failing', + summary = string.format('live weights still failing generation=%s (%d repeat%s): %s', tostring(ev.generation or '?'), rec.repeats, rec.repeats == 1 and '' or 's', tostring(result.err or 'unknown')), + generation = ev.generation, + weight_apply_id = ev.weight_apply_id, + repeats = rec.repeats, + err = result.err, + }) + end + end + end + end + mark_domain_dirty(state, 'wan_runtime') + local pub_ok, pub_err = publish_snapshot(state) + if pub_ok ~= true then return nil, pub_err end + if ok == true and err ~= 'stale' then log_net_summary(state, 'live_weights_done') end + if ok == true and err == 'stale' then return true, nil end + return ok, err +end + +local function handle_event(state, ev) + if ev.kind == 'config_changed' then + return handle_config_changed(state, ev) + elseif ev.kind == 'net_apply_done' then + return handle_apply_done(state, ev) + elseif ev.kind == 'observed_state' then + return handle_observed_state(state, ev) + elseif ev.kind == 'gsm_uplink_changed' or ev.kind == 'gsm_uplink_replay_done' or ev.kind == 'gsm_uplink_unknown' then + return handle_gsm_uplink_changed(state, ev) + elseif ev.kind == 'gsm_uplink_watch_closed' then + state.gsm_uplink_watch = nil + return true, nil + elseif ev.kind == 'net_speedtest_done' then + return handle_speedtest_done(state, ev) + elseif ev.kind == 'speedtest_retry_due' then + return handle_speedtest_retry_due(state, ev) + elseif ev.kind == 'net_live_weights_done' then + return handle_live_weights_done(state, ev) + elseif ev.kind == 'net_observation_started' then + return handle_observation_started(state, ev) + elseif ev.kind == 'capability_status' or ev.kind == 'capability_dependency_changed' then + return handle_dependency_status(state, ev) + elseif ev.kind == 'capability_dependency_closed' then + return handle_dependency_status(state, ev) + elseif ev.kind == 'capability_status_closed' then + if state.capability_status_subs then state.capability_status_subs[ev.capability] = nil end + return true, nil + elseif ev.kind == 'observation_closed' then + state.observed_sub = nil + if state.observer then state.observer:terminate('observation_closed'); state.observer = nil end + state.model:update(function (m) return project_dependencies(state, m) end) + mark_summary_dirty(state) + return publish_snapshot(state) + elseif ev.kind == 'config_watch_closed' then + return nil, ev.err or 'config_watch_closed' + elseif ev.kind == 'completion_queue_closed' then + return nil, 'completion_queue_closed' + elseif ev.kind == 'config_event_unknown' then + obs_log(state.svc, 'warn', { what = 'config_event_unknown', event = ev.event }) + return true, nil + end + obs_log(state.svc, 'debug', { what = 'ignored_event', kind = tostring(ev.kind) }) + return true, nil +end + +function M.run(scope, params) + if type(scope) ~= 'table' then error('net.service.run: scope required', 2) end + params = params or {} + if type(params) ~= 'table' then error('net.service.run: params table required', 2) end + + local conn = params.conn + local svc = params.svc or (conn and service_base.new(conn, { name = params.name or 'net', env = params.env })) or nil + local service_id = new_service_id(params.service_id or params.name or 'net') + local model = model_mod.new(service_id, { label = 'net.service' }) + local done_tx, done_rx = mailbox.new(params.done_queue_len or backpressure.policy.completions.queue_len, { full = backpressure.policy.completions.full }) + local published = publisher.new_state() + local dirty = publisher.mark_all(publisher.new_dirty_state()) + local cfg_watch + local gsm_watch + + if conn then + local werr + cfg_watch, werr = config_watch.open(conn, 'net', { + queue_len = params.config_queue_len or backpressure.policy.config.queue_len, + full = backpressure.policy.config.full, + changed_kind = 'config_changed', + closed_kind = 'config_closed', + }) + if not cfg_watch then error(werr or 'net config watch failed', 2) end + + if params.gsm_uplink_watch ~= false then + local gerr + gsm_watch, gerr = gsm_uplink_watch.open(conn, { + queue_len = params.gsm_uplink_queue_len or ((backpressure.policy.gsm_uplinks and backpressure.policy.gsm_uplinks.queue_len) or 8), + full = params.gsm_uplink_full or ((backpressure.policy.gsm_uplinks and backpressure.policy.gsm_uplinks.full) or 'reject_newest'), + }) + if not gsm_watch then error(gerr or 'net gsm uplink watch failed', 2) end + end + end + + local cap_deps = assert(cap_deps_mod.open(conn, { + { key = 'network_config', class = 'network-config', ref = params.hal_client and params.hal_client.network_config_cap, required = true }, + { key = 'network_state', class = 'network-state', ref = params.hal_client and params.hal_client.network_state_cap, required = false }, + { key = 'network_diagnostics', class = 'network-diagnostics', ref = params.hal_client and params.hal_client.network_diagnostics_cap, required = false }, + }, params.hal_client or {})) + + local hal = params.hal or hal_client_mod.new(conn, { + network_config_cap = cap_deps:ref('network_config'), + network_state_cap = cap_deps:ref('network_state'), + network_diagnostics_cap = cap_deps:ref('network_diagnostics'), + }) + + local state = { + conn = conn, + svc = svc, + scope = scope, + service_id = service_id, + model = model, + published = published, + dirty = dirty, + config_watch = cfg_watch, + gsm_uplink_watch = gsm_watch, + observed_sub = nil, + observer = nil, + observe = params.observe ~= false, + observation_queue_len = params.observation_queue_len or ((backpressure.policy.observations and backpressure.policy.observations.queue_len) or 32), + observation_full = params.observation_full or ((backpressure.policy.observations and backpressure.policy.observations.full) or 'drop_oldest'), + observation_options = params.observation or {}, + done_tx = done_tx, + done_rx = done_rx, + pending = {}, + hal = hal, + cap_deps = cap_deps, + capability_status_subs = nil, + next_generation = 1, + next_apply_id = 1, + next_speedtest_id = 1, + next_speedtest_retry_id = 1, + next_weight_apply_id = 1, + active_speedtests = {}, + speedtest_retries = {}, + active_weight_apply = nil, + counter_poll_enabled = params.counter_metrics ~= false and params.counter_poll ~= false, + counter_poll_interval_s = params.counter_poll_interval_s or params.counter_metrics_interval_s or 60, + counter_poll_timeout_s = params.counter_poll_timeout_s or 5, + current_generation = nil, + active_apply = nil, + pending_intent = nil, + pending_apply_reason = nil, + now = now, + } + state.mark_dirty = function(kind, name) + if kind == 'domain' then mark_domain_dirty(state, name) + elseif kind == 'apply' then mark_apply_dirty(state) + elseif kind == 'summary' then mark_summary_dirty(state) + else mark_all_dirty(state) end + end + + if state.counter_poll_enabled and conn then + scope:spawn(function () metrics.counter_metrics_loop(state) end) + end + + scope:finally(function (_, status, primary) + local reason = primary or status or 'net service closed' + cancel_active_generation(state, reason) + stop_observer(state, reason) + if cfg_watch then cfg_watch:close(); cfg_watch = nil end + if gsm_watch then gsm_watch:close(); gsm_watch = nil end + cap_deps:terminate(reason) + done_tx:close(reason) + publisher.cleanup_now(conn, published) + model:terminate(reason) + end) + + if svc then + svc:spawn_heartbeat(params.heartbeat_s or 30.0, 'tick') + set_status(svc, 'starting') + obs_log(svc, 'debug', { what = 'service_start' }) + end + + set_model_state(state, 'waiting_for_config', 'no_config') + reconcile_observer_admission(state, 'service_start') + publish_snapshot(state) + + if params.config ~= nil then + local ok, err = handle_config_changed(state, { kind = 'config_changed', payload = params.config, rev = params.rev }) + if ok ~= true then error(err or 'initial net config failed', 2) end + end + + while true do + local ev = perform(events.next_service_event_op(state)) + local ok, err = handle_event(state, ev) + if ok ~= true then + set_status(svc, 'failed', { reason = tostring(err) }) + error(err or 'net service event failed', 0) + end + end +end + +function M.start(conn, opts) + opts = opts or {} + opts.conn = conn + return M.run(fibers.current_scope(), opts) +end + +return M diff --git a/src/services/net/shaping.lua b/src/services/net/shaping.lua deleted file mode 100644 index c2918ca3..00000000 --- a/src/services/net/shaping.lua +++ /dev/null @@ -1,314 +0,0 @@ -local log = require "services.log" -local exec = require 'fibers.exec' -local unpack = table.unpack or unpack -- luacheck: ignore -- Compatibility fallback -local bit = rawget(_G, "bit") or require "bit32" -local fqcmemorylimit = "1Mb" -local fqcflows = 256 -local ifbsuffix = "-i" - --- Ip functions -local function parse_ip(s) - local ip = {} - ip[1], ip[2], ip[3], ip[4] = string.match(s, "(%d+)%.(%d+)%.(%d+)%.(%d+)") - ip.subnetmask = string.match(s, ".*/(%d+)") - for i, j in pairs(ip) do - ip[i] = assert(tonumber(j)) - end - -- aargh no error checking - return ip, nil -end - -local function get_ipv4(iface) - local output, err = exec.command("ip", "addr", "show", "dev", iface):output() - - if err then - return nil, "failed to run command: ip addr show dev " .. iface - end - - local raw = string.match(output, "inet%s+(%S+)") - if not raw then - return nil, "interface has no ipv4 address" - end - - local ip, parse_err = parse_ip(raw) - if parse_err then - return nil, parse_err - end - - return ip -end - -local function ip_to_number(ip) - local n = 0 - - for i, v in ipairs(ip) do - n = n + bit.lshift(v, 8 * (4 - i)) - end - - return n -end - -local function number_to_ip(number) - local ip = {} - - for i = 1, 4 do - ip[i] = bit.band(bit.rshift(number, 8 * (4 - i)), bit.lshift(1, 8) - 1) - end - - return ip -end - -local function usable_ips(ip) - -- check the ip address and subnet mask - local number = ip_to_number(ip) - local mask = bit.lshift(1, 32 - ip.subnetmask) - 1 -- 0b10...0 - 1 --> 0b01...1 - local baseip = bit.band(number, bit.bnot(mask)) - local i = 0 - - return function() - i = i + 1 - if i < mask then return i, number_to_ip(baseip + i) end - end -end - -local function setup_ifb(ifbname) - -- Check if IFB already exists - local err = exec.command("ip", "link", "show", ifbname):run() - local exists = not err - - if exists then - log.info("IFB already exists:", ifbname) - else - err = exec.command("ip", "link", "add", "name", ifbname, "type", "ifb"):run() - - if err then - return "Failed to create IFB: " .. tostring(err) - end - end - - err = exec.command("ip", "link", "set", "dev", ifbname, "up"):run() - - if err then - return "Failed to bring IFB up: " .. tostring(err) - end - - return nil -end - -local function clear(iface) - local err = exec.command("tc", "qdisc", "del", "dev", iface, "root"):run() - - if err then - log.warn("Failed to delete root qdisc:", "tc qdisc del dev " .. iface .. " root") - end - - err = exec.command("tc", "qdisc", "del", "dev", iface, "ingress"):run() - - if err then - log.warn("Failed to delete ingress qdisc:", "tc qdisc del dev " .. iface .. " ingress") - end - - return nil -end - -local function ingress_mirror(iface, ifbiface) - local err = exec.command("tc", "qdisc", "add", "dev", iface, "handle", "ffff:", "ingress"):run() - - if err then - log.error("Failed to add ingress qdisc:", "tc qdisc add dev " .. iface .. " handle ffff: ingress") - return "Failed to add ingress qdisc" - end - - err = exec.command( - "tc", "filter", "add", "dev", iface, "parent", "ffff:", "protocol", "ip", "u32", - "match", "u32", "0", "0", - "action", "mirred", "egress", "redirect", "dev", ifbiface - ):run() - - if err then - log.error("Failed to add mirred redirect filter:", - "tc filter add dev " .. iface .. " ... redirect dev " .. ifbiface) - return "Failed to add mirred redirect filter" - end - - return nil -end - -local function host_shape(iface, ipnet, hostsdir, rate) - if ipnet.subnetmask < 24 then - return "error: subnet must be /24 or smaller" - end - - local octet - - if hostsdir == "dst" then - octet = "16" - elseif hostsdir == "src" then - octet = "12" - else - return "error: invalid dir" - end - - local function run(...) - local err = exec.command(...):run() - if err then error("tc command failed: " .. table.concat({ ... }, " ")) end - end - - -- 1. Root qdisc - run("tc", "qdisc", "add", "dev", iface, "root", "handle", "1:", "htb") - - -- 2. 4th octet hasher - run("tc", "filter", "add", "dev", iface, "parent", "1:0", "prio", "1", "protocol", "ip", "u32") - run("tc", "filter", "add", "dev", iface, "parent", "1:0", "prio", "1", "handle", "100:", "protocol", "ip", "u32", - "divisor", "256") - run("tc", "filter", "add", "dev", iface, "protocol", "ip", "parent", "1:0", "prio", "1", "u32", "ht", "800::", - "match", "ip", hostsdir, table.concat(ipnet, ".") .. "/24", - "hashkey", "mask", "0x000000ff", "at", octet, "link", "100:") - - -- 3. Per-host shaping - for i, ip in usable_ips(ipnet) do - local classid = string.format("1:%x", i) - local fq_handle = string.format("%x:0", i + 1) - local ipstr = table.concat(ip, ".") - local ip4 = ip[4] - - run("tc", "class", "add", "dev", iface, "parent", "1:", "classid", classid, - "htb", "rate", rate.rate, "ceil", rate.ceil, "burst", rate.burst) - - run("tc", "qdisc", "add", "dev", iface, "parent", classid, "handle", fq_handle, - "fq_codel", "memory_limit", fqcmemorylimit, "flows", tostring(fqcflows)) - - run("tc", "filter", "add", "dev", iface, "parent", "1:0", "protocol", "ip", "prio", "1", "u32", - "ht", "100:" .. string.format("%x", ip4) .. ":", "match", "ip", hostsdir, ipstr .. "/32", "flowid", classid) - end - - return nil -end - -local function shape_wan(net_cfg, iface) - local wan = iface - log.info("Shaping wan started", wan) - local wanifb = wan .. ifbsuffix - - local err_wanifb = setup_ifb(wanifb) - - if err_wanifb then - log.error(err_wanifb) - end - - clear(iface) - clear(wanifb) - ingress_mirror(iface, wanifb) - - local shaping = net_cfg.shaping or {} - - -- Downlink (ingress from WAN, redirected to IFB) - local ingress = shaping.ingress - - if ingress and ingress.qdisc == "cake" then - local args = { "tc", "qdisc", "replace", "dev", wanifb, "root", "cake", "dual-dsthost", "nat" } - - if ingress.bandwidth then - table.insert(args, "bandwidth") - table.insert(args, ingress.bandwidth) - end - - local err = exec.command(unpack(args)):run() - - if err then - log.warn("Failed to set downlink shaping:", table.concat(args, " ")) - end - end - - -- Uplink (egress directly from WAN) - local egress = shaping.egress - - if egress and egress.qdisc == "cake" then - local args = { "tc", "qdisc", "replace", "dev", wan, "root", "cake", "dual-srchost", "nat" } - - if egress.bandwidth then - table.insert(args, "bandwidth") - table.insert(args, egress.bandwidth) - end - - local err = exec.command(unpack(args)):run() - - if err then - log.warn("Failed to set uplink shaping:", table.concat(args, " ")) - end - end - - log.info("Shaping wan completed", wan) -end - -local function shape_lan(net_cfg) - local lan = "br-" .. net_cfg.id - log.info("Shaping lan started", lan) - local lanifb = lan .. ifbsuffix - - local err_lanifb = setup_ifb(lanifb) - - if err_lanifb then - log.error(err_lanifb) - end - - clear(lan) - clear(lanifb) - ingress_mirror(lan, lanifb) - - local ip, err_ip = get_ipv4(lan) - - if err_ip then - log.error("Failed to get LAN IP for shaping:", err_ip) - return - end - - local shaping = net_cfg.shaping or {} - - local function apply_direction(dir, dev, expected_hash) - local dir_cfg = shaping[dir] - - if not dir_cfg then return end - - local filter = dir_cfg.filters and dir_cfg.filters[1] - local class = dir_cfg.class_template and dir_cfg.class_template.classes[1] - - if not filter or not class then return end - - if filter.kind ~= "u32" or filter.hash_key ~= expected_hash then return end - - local hostsdir = expected_hash == "src_ip" and "src" or "dst" - host_shape(dev, ip, hostsdir, class.config) - end - - apply_direction("egress", lan, "dest_ip") - apply_direction("ingress", lanifb, "src_ip") - - log.info("Shaping lan completed", lan) -end - -local function apply(net_cfg) - local net_type = net_cfg.type - local interfaces = net_cfg.interfaces - - if not interfaces or #interfaces == 0 then - log.warn("No interfaces to shape for network:", net_cfg.id) - return - end - - for _, iface in ipairs(interfaces) do - if net_type == "local" then - log.info("Shaping LAN:", net_cfg.id, iface) - shape_lan(net_cfg) - elseif net_type == "backhaul" then - log.info("Shaping WAN:", net_cfg.id, iface) - shape_wan(net_cfg, iface) - else - log.warn("Unknown network type:", net_type) - end - end -end - -return { - apply = apply -} diff --git a/src/services/net/speedtest.lua b/src/services/net/speedtest.lua deleted file mode 100644 index 9c7e5220..00000000 --- a/src/services/net/speedtest.lua +++ /dev/null @@ -1,122 +0,0 @@ -local op = require 'fibers.op' -local sleep = require 'fibers.sleep' -local file = require 'fibers.stream.file' -local exec = require 'fibers.exec' -local sc = require 'fibers.utils.syscall' -local log = require "services.log" -local context = require "fibers.context" - - -local DOWNLOAD_URL = "https://proof.ovh.net/files/100Mb.dat" -local SAMPLE_INTERVAL = 0.06 -- 60ms between measurements -local NO_IMPROVEMENT_SAMPLES = 2 -- Number of consecutive samples with no improvement before stopping -local STARTUP_TIMEOUT = 2 -- Seconds to wait for connection to speedtest server - -local function run(ctx, owrt_interface, linux_interface) - local cmd = exec.command('mwan3', 'use', owrt_interface, 'wget', '-O', '/dev/null', DOWNLOAD_URL) - cmd:setpgid(true) - local stderr_pipe = assert(cmd:stderr_pipe()) - - local err = cmd:start() - if err then return nil, "Failed to start wget" end - - local rx_file = file.open("/sys/class/net/" .. linux_interface .. "/statistics/rx_bytes", "r") - if not rx_file then return nil, "Failed to open RX bytes" end - - local function get_rx_bytes() - rx_file:seek(0) -- Rewind to beginning - return tonumber(rx_file:read_all_chars()) - end - - local started = false - local ctx_timeout - - -- Detect the moment the download begins - while true do - ctx_timeout = context.with_timeout(ctx, STARTUP_TIMEOUT) - local line = op.choice( - stderr_pipe:read_line_op(), - -- stderr_pipe:read_some_chars_op(), - ctx_timeout:done_op() - ):perform() - if not line then break end - - if line:find("Writing to") then - log.debug("NET: speedtest started!") - started = true - break - end - end - stderr_pipe:close() - - if not started then - rx_file:close() - cmd:kill() - cmd:wait() - return nil, ctx:err() or ctx_timeout:err() or "Could not start file download" - end - - -- Capture initial bytes/time - local start_time = sc.monotime() - local start_bytes = get_rx_bytes() - if not start_bytes then - rx_file:close() - cmd:kill() - cmd:wait() - return nil, "Failed to read RX bytes" - end - - -- Set up a measurement loop - local prev_time = start_time - local prev_bytes = start_bytes - - local peak_speed = 0 - local consecutive_no_improvement = 0 - - while not ctx:err() do - sleep.sleep(SAMPLE_INTERVAL) - - local now = sc.monotime() - local current_bytes = get_rx_bytes() - if not current_bytes then - break - end - - local elapsed = now - prev_time - local diff = current_bytes - prev_bytes - local speed_mbps = (diff * 8) / (elapsed * 2 ^ 20) - - -- Update peak speed or increment the no-improvement counter - if speed_mbps > peak_speed then - peak_speed = speed_mbps - consecutive_no_improvement = 0 - else - consecutive_no_improvement = consecutive_no_improvement + 1 - end - - prev_time = now - prev_bytes = current_bytes - - -- If we've had too many consecutive samples without beating the peak, stop - if consecutive_no_improvement >= NO_IMPROVEMENT_SAMPLES then - break - end - end - - -- Stop the command - rx_file:close() - cmd:kill() - cmd:wait() - - if ctx:err() then return nil, ctx:err() end - - return { - peak = peak_speed, - data = (prev_bytes - start_bytes) / 2 ^ 20, - time = prev_time - start_time - }, nil -end - -return { - run = run -} diff --git a/src/services/net/stale.lua b/src/services/net/stale.lua new file mode 100644 index 00000000..d2cc0268 --- /dev/null +++ b/src/services/net/stale.lua @@ -0,0 +1,43 @@ +-- services/net/stale.lua +-- Uniform stale-completion checks for NET coordinator work. + +local M = {} + +local function bump(state) + if state and state.model then + state.model:update(function (s) + s.stats = s.stats or {} + s.stats.stale_completions = (s.stats.stale_completions or 0) + 1 + return s + end) + end +end + +function M.reject(state, _ev) + bump(state) + return true, nil +end + +function M.apply_current(state, ev) + return state.active_apply + and ev + and state.active_apply.generation == ev.generation + and state.active_apply.apply_id == ev.apply_id +end + +function M.speedtest_current(state, ev) + if not state or not ev then return false end + local rec = state.active_speedtests and state.active_speedtests[ev.uplink_id] + return rec + and rec.generation == ev.generation + and rec.speedtest_id == ev.speedtest_id +end + +function M.live_weights_current(state, ev) + return state.active_weight_apply + and ev + and state.active_weight_apply.generation == ev.generation + and state.active_weight_apply.weight_apply_id == ev.weight_apply_id +end + +return M diff --git a/src/services/net/topics.lua b/src/services/net/topics.lua new file mode 100644 index 00000000..e78d71fa --- /dev/null +++ b/src/services/net/topics.lua @@ -0,0 +1,29 @@ +-- services/net/topics.lua +-- Public topic vocabulary owned by the NET service. + +local M = {} + +local function token(v) + local s = tostring(v or '') + s = s:gsub('[^%w%._%-]', '_') + if s == '' then s = 'unknown' end + return s +end + +function M.config() return { 'cfg', 'net' } end +function M.summary() return { 'state', 'net', 'summary' } end +function M.apply() return { 'state', 'net', 'apply' } end +function M.segments() return { 'state', 'net', 'segments' } end +function M.vlan_policy() return { 'state', 'net', 'vlan-policy' } end +function M.segment(id) return { 'state', 'net', 'segment', token(id) } end +function M.interface(id) return { 'state', 'net', 'interface', token(id) } end +function M.domain(name) return { 'state', 'net', token(name) } end +function M.sources() return { 'state', 'net', 'sources' } end +function M.gsm_uplink_pattern() return { 'state', 'gsm', 'uplink', '+' } end +function M.gsm_uplink(role) return { 'state', 'gsm', 'uplink', token(role) } end +function M.event(name) return { 'event', 'net', token(name) } end +function M.rpc(method) return { 'net', 'rpc', token(method) } end + +M._test = { token = token } + +return M diff --git a/src/services/net/wan_manager.lua b/src/services/net/wan_manager.lua new file mode 100644 index 00000000..ef124927 --- /dev/null +++ b/src/services/net/wan_manager.lua @@ -0,0 +1,470 @@ +-- services/net/wan_manager.lua +-- Coordinator-facing WAN runtime manager. + +local wan_runtime = require 'services.net.wan_runtime' +local wan_policy = require 'services.net.wan_policy' +local stale = require 'services.net.stale' +local model_mod = require 'services.net.model' + +local M = {} + +local function now(state) + return state.now and state.now() or require('fibers').now() +end + +local function obs_log(state, level, payload) + local svc = state and state.svc + if svc and type(svc.obs_log) == 'function' then svc:obs_log(level, payload) end +end + +local function obs_event(state, kind, payload) + local svc = state and state.svc + if svc and type(svc.obs_event) == 'function' then + payload = payload or {} + payload.kind = payload.kind or kind + svc:obs_event(kind, payload) + end +end + +local function table_has_entries(t) + if type(t) ~= 'table' then return false end + return next(t) ~= nil +end + +local function backhaul_ready(snapshot) + local bh = snapshot and snapshot.backhaul or nil + return type(bh) == 'table' and table_has_entries(bh.uplinks) +end + +local function enrich_backhaul_status(payload, state, uplink) + local snap = state and state.model and state.model:snapshot() or nil + local status = wan_policy.uplink_observed_status(snap, uplink) + if type(status) == 'table' then + payload.backhaul_state = status.state + payload.backhaul_usable = status.usable + payload.backhaul_interface = status.interface + payload.backhaul_ifname = status.ifname + payload.backhaul_source = status.source + else + payload.backhaul_status = 'missing' + end + if not backhaul_ready(snap) then payload.backhaul = 'missing' end + return payload +end + +local function speedtest_skip_payload(reason, trigger, uplink, generation) + return { + reason = reason, + trigger = trigger, + generation = generation, + uplink_id = uplink and uplink.uplink_id or nil, + interface = uplink and uplink.request and uplink.request.interface or nil, + device = uplink and uplink.request and uplink.request.device or nil, + } +end + +local function enrich_measurement(payload, state, uplink) + local snap = state and state.model and state.model:snapshot() or nil + local measurement = snap and wan_policy.measurement(snap, uplink) or nil + if type(measurement) == 'table' then + payload.measurement_key = measurement.key + payload.address = measurement.address + payload.address_family = measurement.address_family + end + return payload +end + +local function report_speedtest_skip(state, reason, trigger, uplink, generation) + local payload = enrich_measurement(enrich_backhaul_status(speedtest_skip_payload(reason, trigger, uplink, generation), state, uplink), state, uplink) + payload.what = 'speedtest_skipped' + obs_log(state, 'debug', payload) + obs_event(state, 'speedtest_skipped', + enrich_measurement(enrich_backhaul_status(speedtest_skip_payload(reason, trigger, uplink, generation), state, uplink), state, uplink)) +end + +local function mark_speedtest_skipped(state, uplink, generation, reason) + state.model:update(function(s) + s.wan_runtime = s.wan_runtime or { uplinks = {}, speedtests = {}, live_weights = {} } + s.wan_runtime.speedtests = s.wan_runtime.speedtests or {} + local id = uplink.uplink_id + local rec = s.wan_runtime.speedtests[id] or { uplink_id = id } + local measurement = wan_policy.measurement(s, uplink) + if (reason == 'fresh_same_path' or reason == 'fresh_weak_path') and rec.last_success then + rec.state = 'ok' + rec.ok = true + if reason == 'fresh_weak_path' and type(measurement) == 'table' then + rec.last_success.measurement_key = measurement.key + rec.last_success.measurement = model_mod.deep_copy(measurement) + rec.last_success_key = measurement.key + end + else + rec.state = 'skipped' + rec.ok = false + end + rec.last_skip_generation = generation + rec.last_skip_reason = reason + rec.interface = uplink.request and uplink.request.interface or rec.interface + rec.device = uplink.request and uplink.request.device or rec.device + if type(measurement) == 'table' then + rec.current_measurement_key = measurement.key + rec.current_measurement = model_mod.deep_copy(measurement) + end + rec.updated_at = now(state) + s.wan_runtime.speedtests[id] = rec + return s + end) +end + +function M.cancel(state, reason) + for _, rec in pairs(state.active_speedtests or {}) do + if rec.handle and type(rec.handle.cancel) == 'function' then rec.handle:cancel(reason or 'wan_runtime_cancelled') end + end + state.active_speedtests = {} + if state.active_weight_apply and state.active_weight_apply.handle and type(state.active_weight_apply.handle.cancel) == 'function' then + state.active_weight_apply.handle:cancel(reason or 'wan_runtime_cancelled') + end + state.active_weight_apply = nil + return true, nil +end + +local function start_speedtest_for_uplink(state, uplink) + if not state.hal or type(state.hal.speedtest_op) ~= 'function' then return true, nil end + local snap = state.model:snapshot() + if not wan_policy.speedtest_enabled(snap) then return true, nil end + local generation = state.current_generation and state.current_generation.generation or snap.generation + if type(generation) ~= 'number' or generation <= 0 then return true, nil end + + local uplink_id = uplink.uplink_id + local prev = snap.wan_runtime and snap.wan_runtime.speedtests and snap.wan_runtime.speedtests[uplink_id] + if prev and prev.state == 'running' and prev.generation == generation then return true, nil end + local measurement, merr = wan_policy.measurement(snap, uplink) + if not measurement then return nil, merr or 'speedtest_measurement_key_unavailable' end + + local id = state.next_speedtest_id + state.next_speedtest_id = id + 1 + state.model:update(function(s) + s.wan_runtime = s.wan_runtime or { uplinks = {}, speedtests = {}, live_weights = {} } + s.wan_runtime.generation = generation + s.wan_runtime.uplinks[uplink_id] = { + uplink_id = uplink_id, + interface = uplink.request.interface, + device = uplink.request.device, + metric = uplink.request.metric, + updated_at = now(state), + } + local previous = s.wan_runtime.speedtests[uplink_id] + local same_path_failure = type(previous) == 'table' + and (previous.measurement_key == measurement.key or previous.current_measurement_key == measurement.key) + s.wan_runtime.speedtests[uplink_id] = { + state = 'running', + generation = generation, + id = id, + uplink_id = uplink_id, + interface = uplink.request.interface, + device = uplink.request.device, + metric = uplink.request.metric, + measurement_key = measurement.key, + measurement = model_mod.deep_copy(measurement), + failure_count = same_path_failure and (tonumber(previous.failure_count) or 0) or 0, + last_failure = same_path_failure and model_mod.deep_copy(previous.last_failure) or nil, + last_success = previous and model_mod.deep_copy(previous.last_success) or nil, + last_success_mbps = previous and previous.last_success_mbps or nil, + last_success_at = previous and previous.last_success_at or nil, + started_at = now(state), + } + s.stats.speedtests_started = (s.stats.speedtests_started or 0) + 1 + return s + end) + if state.mark_dirty then state.mark_dirty('domain', 'wan_runtime') end + + local handle, err = wan_runtime.start_speedtest { + lifetime_scope = state.scope, + reaper_scope = state.scope, + report_scope = state.scope, + service_id = state.service_id, + generation = generation, + speedtest_id = id, + uplink_id = uplink_id, + request = uplink.request, + hal = state.hal, + done_tx = state.done_tx, + } + if not handle then + state.model:update(function(s) + local rec = s.wan_runtime and s.wan_runtime.speedtests and s.wan_runtime.speedtests[uplink_id] + if rec then + rec.state = 'failed'; rec.ok = false; rec.err = tostring(err); rec.last_attempt = { state = 'failed', reason = 'failed_to_start', completed_at = now(state) } + end + return s + end) + return nil, err + end + state.active_speedtests[uplink_id] = { generation = generation, speedtest_id = id, handle = handle } + return true, nil +end + +local start_live_weight_apply + +local function apply_weights_if_ready(state, generation) + if state.active_weight_apply ~= nil then return true, nil end + for _, active in pairs(state.active_speedtests or {}) do + if active and active.generation == generation then return true, nil end + end + local snap = state.model:snapshot() + local weights = wan_policy.compute_weights(snap, generation, { now = now(state) }) + if not weights then return true, nil end + local previous = snap.wan_runtime and snap.wan_runtime.last_weight_apply and snap.wan_runtime.last_weight_apply.members + if wan_policy.weights_equal(previous, weights) then return true, nil end + return start_live_weight_apply(state, weights) +end + +function M.reconcile_speedtests(state, reason) + local snap = state.model:snapshot() + if not wan_policy.speedtest_enabled(snap) then + obs_log(state, 'debug', { + what = 'speedtests_skipped', + reason = 'speedtests_disabled', + trigger = reason + }) + return true, nil + end + local generation = state.current_generation and state.current_generation.generation or snap.generation + if type(generation) ~= 'number' or generation <= 0 then + obs_log(state, 'debug', { + what = 'speedtests_skipped', + reason = 'invalid_generation', + trigger = reason, + generation = generation + }) + return true, nil + end + + local uplinks = wan_policy.collect_uplinks(snap) + if #uplinks == 0 then + state.model:update(function(s) + s.wan_runtime = s.wan_runtime or { uplinks = {}, speedtests = {}, live_weights = {} } + s.wan_runtime.live_weights = { state = 'not_applicable', reason = 'no_wan_uplinks', generation = generation } + return s + end) + return true, nil + end + + state.model:update(function(s) + s.wan_runtime = s.wan_runtime or { uplinks = {}, speedtests = {}, live_weights = {} } + s.wan_runtime.generation = s.wan_runtime.generation or generation + s.wan_runtime.last_reconcile_reason = reason + s.wan_runtime.last_reconcile_at = now(state) + return s + end) + + local speedtest_pending = false + for i = 1, #uplinks do + local uplink = uplinks[i] + local observed_status = wan_policy.uplink_observed_status(snap, uplink) + local online = wan_policy.uplink_online(snap, uplink) + if not online then + local skip_reason = 'not_online' + if observed_status == nil then + skip_reason = 'waiting_for_observation' + else + local active = state.active_speedtests[uplink.uplink_id] + if active and active.handle and type(active.handle.cancel) == 'function' then + active.handle:cancel('uplink_offline') + end + state.active_speedtests[uplink.uplink_id] = nil + end + mark_speedtest_skipped(state, uplink, generation, skip_reason) + report_speedtest_skip(state, skip_reason, reason, uplink, generation) + else + local due, due_reason = wan_policy.speedtest_due(state.model:snapshot(), uplink, + { generation = generation, now = now(state) }) + if due then + speedtest_pending = true + local ok, err = start_speedtest_for_uplink(state, uplink) + if ok ~= true then return nil, err end + else + if due_reason == 'running' then speedtest_pending = true end + mark_speedtest_skipped(state, uplink, generation, due_reason or 'not_due') + report_speedtest_skip(state, due_reason or 'not_due', reason, uplink, generation) + end + end + end + if not speedtest_pending then return apply_weights_if_ready(state, generation) end + return true, nil +end + +M.start_speedtests = M.reconcile_speedtests + +function start_live_weight_apply(state, members) + if not members or #members == 0 or not state.hal or type(state.hal.apply_live_weights_op) ~= 'function' then + return + true, nil + end + local snap = state.model:snapshot() + local generation = state.current_generation and state.current_generation.generation or snap.generation + if type(generation) ~= 'number' or generation <= 0 then return true, nil end + + if state.active_weight_apply and state.active_weight_apply.handle and state.active_weight_apply.handle.cancel then + state.active_weight_apply.handle:cancel('live_weight_apply_superseded') + end + + local id = state.next_weight_apply_id + state.next_weight_apply_id = id + 1 + state.active_weight_apply = { generation = generation, weight_apply_id = id } + state.model:update(function(s) + s.wan_runtime = s.wan_runtime or { uplinks = {}, speedtests = {}, live_weights = {} } + s.wan_runtime.live_weights = { + state = 'running', + generation = generation, + id = id, + policy = wan_policy.live_weight_policy(snap), + members = model_mod.deep_copy(members), + started_at = now(state), + } + return s + end) + + local handle, err = wan_runtime.start_live_weights { + lifetime_scope = state.scope, + reaper_scope = state.scope, + report_scope = state.scope, + service_id = state.service_id, + generation = generation, + weight_apply_id = id, + members = members, + policy = wan_policy.live_weight_policy(snap), + persist = true, + hal = state.hal, + done_tx = state.done_tx, + } + if not handle then + state.active_weight_apply = nil + return nil, err + end + state.active_weight_apply.handle = handle + return true, nil +end + +function M.handle_speedtest_done(state, ev) + if not stale.speedtest_current(state, ev) then + stale.reject(state, ev) + return true, 'stale' + end + state.active_speedtests[ev.uplink_id] = nil + local work_result = ev.result or {} + local result = work_result.result or work_result + state.model:update(function(s) + s.wan_runtime = s.wan_runtime or { uplinks = {}, speedtests = {}, live_weights = {} } + local rec = s.wan_runtime.speedtests[ev.uplink_id] or { uplink_id = ev.uplink_id, id = ev.speedtest_id } + local completed_at = now(state) + rec.generation = ev.generation + rec.id = ev.speedtest_id + rec.ok = result.ok == true + rec.err = result.err + rec.interface = result.interface or (work_result.request and work_result.request.interface) or rec.interface + rec.device = result.device or (work_result.request and work_result.request.device) or rec.device + rec.metric = rec.metric or (work_result.request and work_result.request.metric) or 1 + rec.completed_at = completed_at + local mbps = tonumber(result.mbps or result.peak_mbps) + if result.ok == true and mbps ~= nil then + rec.state = 'ok' + rec.failure_count = 0 + rec.retry_after = nil + rec.retry_delay_s = nil + rec.last_failure = nil + rec.peak_mbps = mbps -- compatibility + rec.last_success_mbps = mbps -- compatibility + rec.last_success_at = completed_at + rec.last_success_key = rec.measurement_key + rec.last_success = { + mbps = mbps, + bytes = result.bytes, + data_mib = result.data_mib, + duration_s = result.duration_s, + completed_at = completed_at, + measurement_key = rec.measurement_key, + measurement = model_mod.deep_copy(rec.measurement), + } + + else + local reason = result.code or result.err or 'speedtest_failed' + rec.state = 'failed' + rec.ok = false + rec.err = reason + rec.failure_count = (tonumber(rec.failure_count) or 0) + 1 + rec.retry_delay_s = wan_policy.speedtest_retry_delay_s(s, rec) + rec.retry_after = completed_at + rec.retry_delay_s + rec.last_failure = { + state = 'failed', + reason = reason, + completed_at = completed_at, + failure_count = rec.failure_count, + retry_delay_s = rec.retry_delay_s, + retry_after = rec.retry_after, + } + rec.last_attempt = rec.last_failure -- compatibility + if rec.last_success_mbps ~= nil then rec.peak_mbps = rec.last_success_mbps end + end + rec.data_mib = result.data_mib + rec.duration_s = result.duration_s + s.wan_runtime.speedtests[ev.uplink_id] = rec + s.stats.speedtests_completed = (s.stats.speedtests_completed or 0) + 1 + return s + end) + local snap = state.model:snapshot() + local weights, werr = wan_policy.compute_weights(snap, ev.generation, { now = now(state) }) + if not weights then + state.model:update(function(s) + s.wan_runtime = s.wan_runtime or { uplinks = {}, speedtests = {}, live_weights = {} } + s.wan_runtime.live_weights = { + state = 'skipped', + generation = ev.generation, + reason = werr or 'no_weights', + updated_at = + now(state) + } + return s + end) + return true, nil + end + local previous = snap.wan_runtime and snap.wan_runtime.last_weight_apply and + snap.wan_runtime.last_weight_apply.members + if wan_policy.weights_equal(previous, weights) then return true, nil end + return start_live_weight_apply(state, weights) +end + +function M.handle_live_weights_done(state, ev) + if not stale.live_weights_current(state, ev) then + stale.reject(state, ev) + return true, 'stale' + end + state.active_weight_apply = nil + local work_result = ev.result or {} + local result = work_result.result or work_result + state.model:update(function(s) + s.wan_runtime = s.wan_runtime or { uplinks = {}, speedtests = {}, live_weights = {} } + s.wan_runtime.live_weights = { + state = result.ok == true and 'applied' or 'failed', + generation = ev.generation, + id = ev.weight_apply_id, + members = work_result.members or (work_result.request and work_result.request.members), + result = model_mod.deep_copy(result), + updated_at = now(state), + } + if result.ok == true then + s.wan_runtime.last_weight_apply = { + generation = ev.generation, + id = ev.weight_apply_id, + members = model_mod + .deep_copy(work_result.members or (work_result.request and work_result.request.members) or {}), + updated_at = + now(state) + } + end + s.stats.live_weight_applies = (s.stats.live_weight_applies or 0) + 1 + return s + end) + return true, nil +end + +return M diff --git a/src/services/net/wan_policy.lua b/src/services/net/wan_policy.lua new file mode 100644 index 00000000..90ed7f40 --- /dev/null +++ b/src/services/net/wan_policy.lua @@ -0,0 +1,413 @@ +-- services/net/wan_policy.lua +-- Pure WAN runtime policy for event-led speedtests and live weights. + +local tablex = require 'shared.table' + +local M = {} + +local DEFAULT_SPEEDTEST_INTERVAL_S = 6 * 60 * 60 +local DEFAULT_SPEEDTEST_RETRY_AFTER_S = 10 +local DEFAULT_SPEEDTEST_RETRY_MAX_S = 60 * 60 +local HARD_MAX_SPEEDTEST_DURATION_S = 1 + +local function copy(v) return tablex.deep_copy(v) end + +local function sorted_keys(t) + local ks = {} + for k in pairs(t or {}) do ks[#ks + 1] = k end + table.sort(ks, function(a, b) return tostring(a) < tostring(b) end) + return ks +end + +local function flag_enabled(v) + if v == true then return true end + if type(v) == 'table' then return v.enabled ~= false end + return false +end + +local function runtime_opts(snapshot) + local wan = snapshot and snapshot.wan or {} + local lb = type(wan.load_balancing) == 'table' and wan.load_balancing or {} + local rt = type(wan.runtime) == 'table' and wan.runtime or {} + local sp = type(lb.speedtests) == 'table' and lb.speedtests + or type(lb.speedtest) == 'table' and lb.speedtest + or type(rt.speedtests) == 'table' and rt.speedtests + or type(rt.speedtest) == 'table' and rt.speedtest + or {} + return sp +end + + +local function bounded_number(v, default, minv) + local n = tonumber(v) + if n == nil then n = default end + if minv ~= nil and n < minv then n = minv end + return n +end + +function M.speedtest_retry_delay_s(snapshot, rec) + local cfg = runtime_opts(snapshot) + local base = bounded_number(cfg.retry_after_s or cfg.retry_base_s or cfg.retry_s, + DEFAULT_SPEEDTEST_RETRY_AFTER_S, 1) + local cap = bounded_number(cfg.retry_max_s or cfg.max_retry_after_s or cfg.retry_cap_s, + DEFAULT_SPEEDTEST_RETRY_MAX_S, base) + if cap < base then cap = base end + local failures = math.floor(tonumber(rec and (rec.failure_count or rec.consecutive_failures)) or 1) + if failures < 1 then failures = 1 end + local delay = base + for _ = 2, failures do + delay = delay * 2 + if delay >= cap then return cap end + end + if delay > cap then delay = cap end + return delay +end + +function M.speedtest_enabled(snapshot) + local wan = snapshot and snapshot.wan or {} + local lb = type(wan.load_balancing) == 'table' and wan.load_balancing or {} + local rt = type(wan.runtime) == 'table' and wan.runtime or {} + return flag_enabled(lb.speedtest) + or flag_enabled(lb.speedtests) + or flag_enabled(rt.speedtest) + or flag_enabled(rt.speedtests) +end + +function M.live_weight_policy(snapshot) + local wan = snapshot and snapshot.wan or {} + local lb = type(wan.load_balancing) == 'table' and wan.load_balancing or {} + local rt = type(wan.runtime) == 'table' and wan.runtime or {} + return rt.weight_policy or rt.live_weight_policy or lb.weight_policy or lb.policy or 'balanced' +end + +local function member_interface(member, uplink_id) + member = member or {} + return member.interface or uplink_id +end + +local function gsm_source(member) + local src = member and member.source or nil + if type(src) ~= 'table' or src.kind ~= 'gsm-uplink' then return nil end + return src.id +end + +local function source_kind(member, status) + local src = member and member.source or nil + if type(src) == 'table' and type(src.kind) == 'string' and src.kind ~= '' then return src.kind end + local st_src = status and status.source or nil + if type(st_src) == 'table' and type(st_src.kind) == 'string' and st_src.kind ~= '' then return st_src.kind end + return 'host-multiwan' +end + +local function source_id(member, status) + local src = member and member.source or nil + if type(src) == 'table' and src.id ~= nil then return src.id end + local st_src = status and status.source or nil + if type(st_src) == 'table' and st_src.id ~= nil then return st_src.id end + return nil +end + +function M.gsm_uplink_state(snapshot, member) + local id = gsm_source(member) + if not id then return nil end + return snapshot and snapshot.sources and snapshot.sources.gsm_uplinks and snapshot.sources.gsm_uplinks[id] or nil +end + +function M.build_speedtest_request(snapshot, uplink_id, member) + member = member or {} + local iface_id = member_interface(member, uplink_id) + local iface = snapshot.interfaces and snapshot.interfaces[iface_id] or nil + local endpoint = type(iface) == 'table' and type(iface.endpoint) == 'table' and iface.endpoint or {} + local gsm = M.gsm_uplink_state(snapshot, member) + local gsm_linux = type(gsm) == 'table' and type(gsm.linux) == 'table' and gsm.linux or {} + local metric = tonumber(member.metric) or 1 + local duration = tonumber(member.speedtest_duration_s) or HARD_MAX_SPEEDTEST_DURATION_S + if duration <= 0 or duration > HARD_MAX_SPEEDTEST_DURATION_S then duration = HARD_MAX_SPEEDTEST_DURATION_S end + return { + interface = iface_id, + device = member.device or member.linux_interface or member.ifname or endpoint.ifname or endpoint.device or endpoint.name or (iface and iface.device) or gsm_linux.ifname, + url = member.speedtest_url, + max_duration_s = duration, + uplink_id = tostring(uplink_id), + metric = metric, + } +end + +function M.collect_uplinks(snapshot) + local out = {} + local members = snapshot and snapshot.wan and snapshot.wan.configured_members or {} + local keys = sorted_keys(members) + for i = 1, #keys do + local uplink_id = keys[i] + local member = members[uplink_id] + if type(member) == 'table' and member.enabled ~= false and member.disabled ~= true then + local req = M.build_speedtest_request(snapshot, uplink_id, member) + if type(req.interface) == 'string' and req.interface ~= '' then + out[#out + 1] = { uplink_id = tostring(uplink_id), member = copy(member), request = req } + end + end + end + return out +end + +local function status_by_interface(snapshot, iface) + local bh = snapshot and snapshot.backhaul or nil + local uplinks = type(bh) == 'table' and bh.uplinks or nil + if type(uplinks) ~= 'table' then return nil end + for id, rec in pairs(uplinks) do + if type(rec) == 'table' and (rec.interface == iface or rec.id == iface or id == iface) then + if rec.observed == false then return nil end + return rec + end + end + return nil +end + +local function status_online(st) + return type(st) == 'table' + and st.usable == true + and st.state == 'online' +end + +function M.uplink_observed_status(snapshot, uplink) + return status_by_interface(snapshot, uplink and uplink.request and uplink.request.interface) +end + +function M.uplink_online(snapshot, uplink) + return status_online(M.uplink_observed_status(snapshot, uplink)) +end + +local function normalise_address(addr) + if type(addr) ~= 'table' then return nil end + local address = addr.address or addr.ip or addr.addr + if type(address) ~= 'string' or address == '' then return nil end + if address == '0.0.0.0' or address == '::' then return nil end + return { + family = tostring(addr.family or addr.af or 'ipv4'), + address = address, + source = addr.source, + } +end + +function M.measurement(snapshot, uplink) + local status = M.uplink_observed_status(snapshot, uplink) + local addr = normalise_address(status and status.path_address) + local req = uplink and uplink.request or {} + local member = uplink and uplink.member or {} + local parts = { + schema = 'devicecode.net.speedtest-key/1', + uplink_id = tostring(uplink and uplink.uplink_id or ''), + interface = tostring(req.interface or ''), + device = tostring(req.device or ''), + source_kind = tostring(source_kind(member, status) or ''), + source_id = tostring(source_id(member, status) or ''), + address_family = addr and addr.family or 'unknown', + address = addr and addr.address or 'unknown', + address_known = addr ~= nil, + speedtest_url = tostring(req.url or ''), + } + local key = table.concat({ + 'v1', parts.uplink_id, parts.interface, parts.device, + parts.source_kind, parts.source_id, parts.address_family, + parts.address, parts.speedtest_url, + }, '|') + parts.key = key + parts.address_source = addr and addr.source or 'unobserved' + return parts, nil +end + +function M.measurement_key(snapshot, uplink) + local m, err = M.measurement(snapshot, uplink) + return m and m.key or nil, err, m +end + +local function now_from_opts(opts) + return opts and opts.now or nil +end + +local function speedtest_ttl(snapshot) + local cfg = runtime_opts(snapshot) + local ttl = tonumber(cfg.interval_s or cfg.refresh_s or cfg.ttl_s or cfg.max_age_s) + if ttl == nil then return DEFAULT_SPEEDTEST_INTERVAL_S end + return ttl +end + +local function speedtest_last_success(rec) + if type(rec) ~= 'table' then return nil, nil end + local last = type(rec.last_success) == 'table' and rec.last_success or nil + local mbps = last and tonumber(last.mbps) or tonumber(rec.last_success_mbps) or tonumber(rec.peak_mbps) + local completed = last and tonumber(last.completed_at) + or tonumber(rec.last_success_at) + or tonumber(rec.completed_at) + if mbps == nil or mbps <= 0 then return nil, completed end + return mbps, completed +end + +local function speedtest_last_key(rec) + if type(rec) ~= 'table' then return nil end + local last = type(rec.last_success) == 'table' and rec.last_success or nil + return last and (last.measurement_key or last.path_key) or rec.last_success_key or rec.measurement_key +end + +local function speedtest_last_address(rec) + local last = type(rec) == 'table' and type(rec.last_success) == 'table' and rec.last_success or nil + local m = last and type(last.measurement) == 'table' and last.measurement or nil + return m and m.address or nil +end + +local function address_known(address) + return type(address) == 'string' and address ~= '' and address ~= 'unknown' +end + +local function speedtest_last_address_known(rec) + return address_known(speedtest_last_address(rec)) +end + +local function weak_key_for_measurement(m) + if type(m) ~= 'table' then return nil end + return table.concat({ + 'v1', tostring(m.uplink_id or ''), tostring(m.interface or ''), tostring(m.device or ''), + tostring(m.source_kind or ''), tostring(m.source_id or ''), 'unknown', 'unknown', + tostring(m.speedtest_url or ''), + }, '|') +end + +local function speedtest_success_fresh(snapshot, rec, now) + local mbps, completed = speedtest_last_success(rec) + if mbps == nil then return false end + local ttl = speedtest_ttl(snapshot) + if not ttl or ttl <= 0 then return true end + return now ~= nil and completed ~= nil and completed > 0 and now - completed < ttl +end + +local function speedtest_success_valid(snapshot, rec, now, key, measurement) + if not speedtest_success_fresh(snapshot, rec, now) then return false end + if key == nil then return false end + if speedtest_last_key(rec) == key then return true end + -- If the original measurement was admitted before an address was observable, + -- allow that fresh result to be used while the current path is being + -- strengthened with a known address. The manager will re-key the cached + -- success on the skip path so future IP changes still matter. + if measurement and measurement.address_known == true and not speedtest_last_address_known(rec) + and speedtest_last_key(rec) == weak_key_for_measurement(measurement) then + return true + end + return false +end + +function M.speedtest_due(snapshot, uplink, opts) + opts = opts or {} + if not M.speedtest_enabled(snapshot) then return false, 'speedtests_disabled' end + if not M.uplink_online(snapshot, uplink) then return false, 'not_online' end + local key, _, measurement = M.measurement_key(snapshot, uplink) + local generation = opts.generation or snapshot.generation + local id = uplink.uplink_id + local runtime = snapshot.wan_runtime or {} + local rec = runtime.speedtests and runtime.speedtests[id] + if rec and rec.state == 'running' and rec.generation == generation then return false, 'running' end + local now = now_from_opts(opts) + if rec and rec.retry_after and now and now < rec.retry_after then + local retry_key = rec.measurement_key or rec.current_measurement_key or speedtest_last_key(rec) + if retry_key == key then return false, 'retry_later' end + end + if speedtest_success_valid(snapshot, rec, now, key, measurement) then + if measurement and measurement.address_known == true and not speedtest_last_address_known(rec) then + return false, 'fresh_weak_path' + end + return false, 'fresh_same_path' + end + if speedtest_success_fresh(snapshot, rec, now) and speedtest_last_key(rec) ~= nil then + local last_addr = speedtest_last_address(rec) + if measurement and measurement.address_known == true and address_known(last_addr) and last_addr ~= measurement.address then + return true, 'path_changed_ip' + end + return true, 'path_changed' + end + if speedtest_success_fresh(snapshot, rec, now) then return true, 'missing_previous_key' end + return true, 'due' +end + +local function member_fingerprint(m) + if type(m) ~= 'table' then return '' end + return table.concat({ tostring(m.id), tostring(m.interface), tostring(m.metric), tostring(m.weight) }, ':') +end + +function M.weights_equal(a, b) + a, b = a or {}, b or {} + if #a ~= #b then return false end + local aa, bb = {}, {} + for i = 1, #a do aa[i] = member_fingerprint(a[i]) end + for i = 1, #b do bb[i] = member_fingerprint(b[i]) end + table.sort(aa); table.sort(bb) + for i = 1, #aa do if aa[i] ~= bb[i] then return false end end + return true +end + +function M.compute_weights(snapshot, generation, opts) + opts = opts or {} + local runtime = snapshot.wan_runtime or {} + local tests = runtime.speedtests or {} + local measured, total = {}, 0 + local uplinks = M.collect_uplinks(snapshot) + local cfg = runtime_opts(snapshot) + local probe_weight = math.max(1, math.floor(tonumber(cfg.probe_weight) or 1)) + local weight_scale = math.max(1, math.floor(tonumber(cfg.weight_scale) or 100)) + + for _, uplink in ipairs(uplinks) do + local id = uplink.uplink_id + local rec = tests[id] + local mbps = nil + if rec then + local key, _, measurement = M.measurement_key(snapshot, uplink) + local success_mbps = speedtest_last_success(rec) + if success_mbps and speedtest_success_valid(snapshot, rec, now_from_opts(opts), key, measurement) then + mbps = success_mbps + end + end + if mbps and mbps > 0 then total = total + mbps end + measured[id] = mbps + end + local out = {} + if total <= 0 then + local online = {} + for _, uplink in ipairs(uplinks) do + if M.uplink_online(snapshot, uplink) then online[#online + 1] = uplink end + end + if #online == 0 then return nil, 'no_online_wan_uplinks' end + local fallback_weight = math.max(1, math.floor((weight_scale / #online) + 0.5)) + for _, uplink in ipairs(online) do + out[#out + 1] = { + id = uplink.uplink_id, + interface = uplink.request.interface, + metric = math.max(1, math.floor(tonumber(uplink.request.metric) or 1)), + weight = fallback_weight, + measured_mbps = nil, + probe = true, + reason = 'no_successful_speedtests', + } + end + return out, nil + end + + for _, uplink in ipairs(uplinks) do + local id = uplink.uplink_id + local mbps = measured[id] + local metric = math.max(1, math.floor(tonumber(uplink.request.metric) or 1)) + local weight, probe = probe_weight, true + if mbps and mbps > 0 then + weight = math.max(1, math.floor((mbps / total) * weight_scale + 0.5)) + probe = false + end + out[#out + 1] = { + id = id, + interface = uplink.request.interface, + metric = metric, + weight = weight, + measured_mbps = mbps, + probe = probe, + } + end + return out, nil +end + +return M diff --git a/src/services/net/wan_runtime.lua b/src/services/net/wan_runtime.lua new file mode 100644 index 00000000..0572538a --- /dev/null +++ b/src/services/net/wan_runtime.lua @@ -0,0 +1,123 @@ +-- services/net/wan_runtime.lua +-- Scoped WAN runtime work: speedtests and live multi-WAN weighting. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' + +local M = {} + +local HARD_MAX_SPEEDTEST_DURATION_S = 1 + +local function with_timeout(work_op, timeout_s, label) + timeout_s = tonumber(timeout_s) + if not timeout_s or timeout_s < 0 then return work_op end + return op.named_choice({ + work = work_op, + timeout = sleep.sleep_op(timeout_s):wrap(function () + return { ok = false, err = tostring(label or 'operation') .. '_timeout', timeout = true } + end), + }):wrap(function (which, result) + if which == 'timeout' then return result end + return result + end) +end + + +local function report_to(done_tx, label) + return function (ev) + return queue.try_admit_required(done_tx, ev, label) + end +end + +function M.start_speedtest(spec) + if type(spec) ~= 'table' then return nil, 'speedtest spec required' end + local lifetime_scope = spec.lifetime_scope + local hal = spec.hal + local done_tx = spec.done_tx + local request = spec.request + if not lifetime_scope then return nil, 'lifetime_scope required' end + if not hal then return nil, 'hal client required' end + if not done_tx then return nil, 'done_tx required' end + if type(request) ~= 'table' then return nil, 'speedtest request required' end + if type(spec.generation) ~= 'number' then return nil, 'generation required' end + if type(spec.speedtest_id) ~= 'number' then return nil, 'speedtest_id required' end + if type(spec.uplink_id) ~= 'string' or spec.uplink_id == '' then return nil, 'uplink_id required' end + + return scoped_work.start { + lifetime_scope = lifetime_scope, + reaper_scope = spec.reaper_scope or lifetime_scope, + report_scope = spec.report_scope or lifetime_scope, + identity = { + kind = 'net_speedtest_done', + service_id = spec.service_id or 'net', + generation = spec.generation, + speedtest_id = spec.speedtest_id, + uplink_id = spec.uplink_id, + }, + run = function () + local timeout_s = tonumber(request.max_duration_s or spec.timeout_s) or HARD_MAX_SPEEDTEST_DURATION_S + if timeout_s <= 0 or timeout_s > HARD_MAX_SPEEDTEST_DURATION_S then timeout_s = HARD_MAX_SPEEDTEST_DURATION_S end + local result = fibers.perform(with_timeout( + hal:speedtest_op(request, { timeout = false }), + timeout_s, + 'net_speedtest' + )) + return { + uplink_id = spec.uplink_id, + request = request, + result = result, + ok = result and result.ok == true, + } + end, + report = report_to(done_tx, 'net_speedtest_completion_report_failed'), + } +end + +function M.start_live_weights(spec) + if type(spec) ~= 'table' then return nil, 'live weight spec required' end + local lifetime_scope = spec.lifetime_scope + local hal = spec.hal + local done_tx = spec.done_tx + if not lifetime_scope then return nil, 'lifetime_scope required' end + if not hal then return nil, 'hal client required' end + if not done_tx then return nil, 'done_tx required' end + if type(spec.generation) ~= 'number' then return nil, 'generation required' end + if type(spec.weight_apply_id) ~= 'number' then return nil, 'weight_apply_id required' end + + local request = { + policy = spec.policy or 'balanced', + members = spec.members or {}, + persist = spec.persist ~= false, + } + + return scoped_work.start { + lifetime_scope = lifetime_scope, + reaper_scope = spec.reaper_scope or lifetime_scope, + report_scope = spec.report_scope or lifetime_scope, + identity = { + kind = 'net_live_weights_done', + service_id = spec.service_id or 'net', + generation = spec.generation, + weight_apply_id = spec.weight_apply_id, + }, + run = function () + local result = fibers.perform(with_timeout( + hal:apply_live_weights_op(request, { timeout = false }), + spec.timeout_s or 10, + 'net_live_weights' + )) + return { + request = request, + members = request.members, + result = result, + ok = result and result.ok == true, + } + end, + report = report_to(done_tx, 'net_live_weights_completion_report_failed'), + } +end + +return M diff --git a/src/services/support/capdeps.lua b/src/services/support/capdeps.lua new file mode 100644 index 00000000..38e14a00 --- /dev/null +++ b/src/services/support/capdeps.lua @@ -0,0 +1,101 @@ +-- services/support/capdeps.lua +-- Small helper for declared, narrowed cross-service capability dependencies. +-- +-- Service roots may hold the bus connection. Managers, drivers and backends +-- should receive dependency ports built here, not conn itself. + +local M = {} +local Resolver = {} +Resolver.__index = Resolver + +local function non_empty_string(v) + return type(v) == 'string' and v ~= '' +end + +local function shallow_copy(t) + local out = {} + for k, v in pairs(t or {}) do out[k] = v end + return out +end + +local function validate_methods(methods) + if type(methods) ~= 'table' or #methods == 0 then return nil, 'dependency methods must be a non-empty list' end + local out, seen = {}, {} + for i = 1, #methods do + local name = methods[i] + if not non_empty_string(name) then return nil, 'dependency method names must be non-empty strings' end + if seen[name] then return nil, 'duplicate dependency method: ' .. name end + seen[name] = true + out[#out + 1] = name + end + return out, nil +end + +function M.capability(spec) + if type(spec) ~= 'table' then return nil, 'capability dependency spec must be a table' end + if type(spec.sdk) ~= 'table' or type(spec.sdk.new_ref) ~= 'function' then return nil, 'capability dependency requires sdk.new_ref' end + local methods, merr = validate_methods(spec.methods) + if not methods then return nil, merr end + return { + kind = 'capability', + sdk = spec.sdk, + default_cap_id = spec.default_cap_id or 'main', + methods = methods, + }, nil +end + +local function narrow(ref, methods, meta) + local out = { + _capability_dependency = true, + dependency = meta, + } + for i = 1, #methods do + local name = methods[i] + if type(ref[name]) ~= 'function' then return nil, 'capability ref missing method ' .. name end + out[name] = function (_, ...) + return ref[name](ref, ...) + end + end + return out, nil +end + +function M.new(conn, declarations) + if conn == nil then return nil, 'dependency resolver requires conn' end + if type(declarations) ~= 'table' then return nil, 'dependency declarations must be a table' end + local declared = {} + for name, decl in pairs(declarations) do + if not non_empty_string(name) then return nil, 'dependency names must be non-empty strings' end + if type(decl) ~= 'table' or decl.kind ~= 'capability' then return nil, 'unsupported dependency declaration: ' .. tostring(name) end + declared[name] = shallow_copy(decl) + declared[name].methods = shallow_copy(decl.methods) + end + return setmetatable({ conn = conn, declarations = declared, cache = {} }, Resolver), nil +end + +function Resolver:get(name, cap_id) + local decl = self.declarations[name] + if not decl then return nil, 'undeclared dependency: ' .. tostring(name) end + local id = cap_id or decl.default_cap_id or 'main' + local key = tostring(name) .. '\0' .. tostring(id) + if self.cache[key] then return self.cache[key], nil end + local ref = decl.sdk.new_ref(self.conn, id) + local port, err = narrow(ref, decl.methods, { + name = name, + kind = decl.kind, + capability_id = id, + }) + if not port then return nil, err end + self.cache[key] = port + return port, nil +end + +function Resolver:factory(name) + return function (cap_id) + local port, err = self:get(name, cap_id) + if not port then return nil, err end + return port + end +end + +M.Resolver = Resolver +return M diff --git a/src/services/switch.lua b/src/services/switch.lua deleted file mode 100644 index 88e6c2ab..00000000 --- a/src/services/switch.lua +++ /dev/null @@ -1,105 +0,0 @@ -local fiber = require "fibers.fiber" -local sleep = require "fibers.sleep" -local context = require "fibers.context" -local op = require "fibers.op" -local log = require "services.log" - -local switch_service = { - name = "switch" -} -switch_service.__index = switch_service - -local function load_driver(switch_type) - local ok, driver = pcall(require, "services.switch." .. switch_type) - if not ok then - return nil, "Unsupported switch type: " .. tostring(switch_type) - end - return driver, nil -end - -local function handler(ctx, bus_connection) - local sub_switch = bus_connection:subscribe({ "config", "switch" }) - local stats_ctx = nil - local cancel_stats = nil - - while ctx:err() == nil do - op.choice( - sub_switch:next_msg_op():wrap(function(msg) - log.trace("Switch: Config received") - local cfg = msg.payload - local driver, err = load_driver(cfg.type) - - if err or not driver then - log.error("Switch: Error loading driver:", err) - return - end - - -- cancel any running stats loop - if cancel_stats then cancel_stats("new config") end - - -- create a fresh child context for stats - stats_ctx, cancel_stats = context.with_cancel(ctx) - - fiber.spawn(function() - while stats_ctx:err() == nil do - log.trace("Switch: Starting login") - local ok, err = driver.login(cfg.host, cfg.username, cfg.password) - - if not ok then - log.error("Switch: Login failed, retrying in 60s:", err) - sleep.sleep(60) - else - log.trace("Switch: Login successful") - break - end - end - - local fail_count = 0 - local max_fails = 5 - - while stats_ctx:err() == nil do - log.trace("Switch: Collecting metrics") - local stats, err = driver.get_stats(cfg.host) - - if err then - fail_count = fail_count + 1 - log.error("Switch: Stats error (fail " .. fail_count .. "):", err) - - if fail_count >= max_fails then - log.warn("Switch: Too many failures, retrying login") - local ok, lerr = driver.login(cfg.host, cfg.username, cfg.password) - if not ok then - log.error("Switch: Re-login failed:", lerr) - break - end - fail_count = 0 - end - - sleep.sleep(5) - else - log.trace("Switch: Publishing stats") - fail_count = 0 -- reset on success - bus_connection:publish_multiple({ "switch" }, stats, { retained = true }) - sleep.sleep(cfg.report_period or 10) - end - end - end) - end), - ctx:done_op() - ):perform() - end - - sub_switch:unsubscribe() -end - -function switch_service:start(ctx, bus_connection) - log.trace("Starting Switch Service") - - self.bus_connection = bus_connection - - fiber.spawn(function() - handler(ctx, bus_connection) - end) -end - -return switch_service diff --git a/src/services/switch/rtl8380m.lua b/src/services/switch/rtl8380m.lua deleted file mode 100644 index 1b86dc56..00000000 --- a/src/services/switch/rtl8380m.lua +++ /dev/null @@ -1,315 +0,0 @@ -local cjson = require "cjson.safe" -local socket = require "cqueues.socket" -local sleep = require "fibers.sleep" -local basexx = require "basexx" -local exec = require "fibers.exec" - -local PORT = 80 -local EXPONENT_HEX = "10001" -local DEFAULT_TIMEOUT = 10 - --- HTTP helpers -local function build_request(method, path, host, headers, body) - local req = string.format("%s %s HTTP/1.0\r\n", method, path) - req = req .. string.format("Host: %s\r\n", host) - req = req .. "Accept: */*\r\nConnection: close\r\n" - - if headers then - for k, v in pairs(headers) do - req = req .. string.format("%s: %s\r\n", k, v) - end - end - - req = req .. "\r\n" - if body then - req = req .. body - end - - return req -end - -local function parse_body(raw) - local i = raw:find("\r\n\r\n", 1, true) - if i then return raw:sub(i + 4) end - - i = raw:find("\n\n", 1, true) - if i then return raw:sub(i + 2) end - - local j = raw:find("{", 1, true) - if j then return raw:sub(j) end - - return raw -end - -local function safe_write_and_flush(s, data) - -- write may throw; catch it - local ok, wres, werr = pcall(function() return s:write(data) end) - if not ok then return nil, wres end -- wres is the thrown error - if not wres then return nil, werr end -- write returned nil, err - - local fok, ferr = pcall(function() return s:flush() end) - if not fok then return nil, ferr end - return true -end - -local function safe_read_all(s) - local chunks = {} - while true do - local ok, buf, err, part = pcall(function() return s:read(4096) end) - if not ok then return nil, buf end -- buf is thrown error - if buf and #buf > 0 then - chunks[#chunks + 1] = buf - elseif part and #part > 0 then - chunks[#chunks + 1] = part - end - if not buf then - if err and err ~= "eof" then - return nil, "read error: " .. tostring(err) - end - break - end - end - return table.concat(chunks) -end - -local function get_cgi_json(host, cmd, use_dummy) - local path = "/cgi/get.cgi?cmd=" .. - cmd .. (use_dummy and ("&dummy=" .. tostring(math.floor(os.time() * 1000))) or "") - local s, err = socket.connect(host, PORT) - if not s then return nil, "connect failed: " .. tostring(err) end - - s:settimeout(DEFAULT_TIMEOUT) - s:setmode("b", "b") - - local req = build_request("GET", path, host, {}) - local ok, werr = safe_write_and_flush(s, req) - if not ok then - s:close(); return nil, "write/flush failed: " .. tostring(werr) - end - - local raw, rerr = safe_read_all(s) - s:close() - if not raw then return nil, rerr end - - local body = parse_body(raw) - local js, derr = cjson.decode(body) - if not js then return nil, "decode error: " .. tostring(derr) end - return js -end - -local function post_cgi_json(host, path, payload, headers) - local s, err = socket.connect(host, PORT) - if not s then return nil, "connect failed: " .. tostring(err) end - - s:settimeout(DEFAULT_TIMEOUT) - s:setmode("b", "b") - - local req = build_request("POST", path, host, headers, payload) - local ok, werr = safe_write_and_flush(s, req) - if not ok then - s:close(); return nil, "write/flush failed: " .. tostring(werr) - end - - local raw, rerr = safe_read_all(s) - s:close() - if not raw then return nil, rerr end - - local body = parse_body(raw) - local js, derr = cjson.decode(body) - if not js then return nil, "decode error: " .. tostring(derr) end - return js -end - --- Encryption -local function urlencode_b64(s) - return (s:gsub("[+/=]", function(c) - return string.format("%%%02X", string.byte(c)) - end)) -end - -local function encrypt_password(modulus_hex, password) - -- Use /tmp to avoid cluttering working directory - local temp_id = tostring(os.time()) .. math.random(1000) - local asn1_file = "/tmp/pubkey_" .. temp_id .. ".asn1" - local der_file = "/tmp/pubkey_" .. temp_id .. ".der" - local pem_file = "/tmp/pubkey_" .. temp_id .. ".pem" - - -- 1. Build ASN.1 text (in Lua) - local asn1 = string.format([[ - asn1=SEQUENCE:pubkey - [pubkey] - modulus=INTEGER:0x%s - pubexp=INTEGER:0x%s - ]], modulus_hex, EXPONENT_HEX) - - -- Write to temp file - local f = io.open(asn1_file, "w") - if not f then return nil, "cannot write asn1 file" end - f:write(asn1) - f:close() - - -- 2. Generate DER - local err = exec.command("openssl", "asn1parse", "-genconf", asn1_file, "-out", der_file, "-noout"):run() - - if err then - exec.command("rm", asn1_file):run() - return nil, "asn1parse failed" - end - - -- 3. Convert to PEM - err = exec.command("openssl", "rsa", "-RSAPublicKey_in", "-inform", "DER", "-in", der_file, "-out", pem_file, - "-pubout"):run() - - if err then - exec.command("rm", asn1_file, der_file):run() - return nil, "rsa conversion failed" - end - - -- 4. Encrypt password with pkeyutl - local cmd3 = string.format( - "echo -n %q | openssl pkeyutl -encrypt -inkey %s -pubin -pkeyopt rsa_padding_mode:pkcs1 2>/dev/null", password, - pem_file) - local pipe = io.popen(cmd3, "r") - if not pipe then - exec.command("rm", asn1_file, der_file, pem_file):run() - return nil, "pkeyutl pipe failed" - end - - local ct = pipe:read("*all") - pipe:close() - - -- Clean up temp files - exec.command("rm", asn1_file, der_file, pem_file):run() - - if not ct or #ct == 0 then - return nil, "openssl pkeyutl failed" - end - - -- 5. Base64 + URL escape - local b64 = basexx.to_base64(ct) - local escaped = urlencode_b64(b64) - - return escaped -end - --- http calls -local function get_ports_info(host) - local js, err = get_cgi_json(host, "panel_info", true) - if not js or err then return nil, err end - return js.data.ports or nil -end - -local function get_sys_cpumem(host) - local js, err = get_cgi_json(host, "sys_cpumem", true) - if not js or err then return nil, nil, err end - return js.data.cpu, js.data.mem, nil -end - -local function get_sys_time(host) - local js, err = get_cgi_json(host, "sys_sysTime", true) - if not js or err then return nil, err end - return js.data.sysCurrTime or nil, nil -end - -local function get_ports_poe_info(host) - local js, err = get_cgi_json(host, "poe_poe", true) - if not js or err then return nil, nil, nil, err end - return js.data.ports, js.data.devPower, js.data.devTemp, nil -end - -local function get_modules(host) - local js, err = get_cgi_json(host, "home_login", false) - if not js or err then return nil, err end - return js.data and js.data.modulus or nil -end - -local function get_login_status(host) - local js, err = get_cgi_json(host, "home_loginStatus", false) - if not js or err then return nil, err end - return js.data and js.data.status or nil -end - -local function post_login(host, username, encoded_pwd) - local path = "/cgi/set.cgi?cmd=home_loginAuth" - local payload = string.format("_ds=1&username=%s&password=%s&_de=1", username, encoded_pwd) - local headers = { - ["Content-Type"] = "application/x-www-form-urlencoded; charset=UTF-8", - ["X-Requested-With"] = "XMLHttpRequest", - ["Content-Length"] = tostring(#payload), - } - return post_cgi_json(host, path, payload, headers) -end - --- authenticate_user = get modulus, encrypt password, then post login -local function authenticate_user(host, username, password) - local mod, err = get_modules(host) - if not mod then return nil, "failed to fetch modulus: " .. tostring(err) end - - local enc_pwd, perr = encrypt_password(mod, password) - if not enc_pwd then return nil, "encrypt error: " .. tostring(perr) end - - return post_login(host, username, enc_pwd) -end - --- TODO: handle unable to auth -local function login(host, username, password) - local _, err = authenticate_user(host, username, password) - - if err then return false, err end - - local tries = 0 - local max_tries = 10 - - while tries < max_tries do - local status, err = get_login_status(host) - - if err then return false, err end - if status == "ok" then return true end - if status == "fail" then return false, "login failed incorrect credentials" end - - tries = tries + 1 - sleep.sleep(1) - end - - return false, "login timeout" -end - -local function get_stats(host) - local stats = { - ["system"] = { - ["curr_time"] = 0, - ["mem"] = 0, - ["cpu"] = 0, - ["power"] = 0, - ["temp"] = 0 - }, - ["ports"] = {}, -- link/speed/etc - ["ports_poe"] = {} -- PoE status/limits - } - - local curr_time, err = get_sys_time(host) - if err then return nil, err end - stats.system.curr_time = curr_time - - local cpu, mem, err = get_sys_cpumem(host) - if err then return nil, err end - stats.system.cpu = cpu - stats.system.mem = mem - - local ports, err = get_ports_info(host) - if err then return nil, err end - stats.ports = ports - - local ports_poe, power, temp, err = get_ports_poe_info(host) - if err then return nil, err end - stats.ports_poe = ports_poe - stats.system.power = power - stats.system.temp = temp - - return stats, nil -end - -return { - login = login, - get_stats = get_stats, -} diff --git a/src/services/system.lua b/src/services/system.lua index faae4dfa..6a0d17f6 100644 --- a/src/services/system.lua +++ b/src/services/system.lua @@ -1,287 +1,715 @@ -local service = require "service" -local log = require "services.log" -local op = require "fibers.op" -local sleep = require "fibers.sleep" -local channel = require "fibers.channel" -local context = require "fibers.context" -local exec = require "fibers.exec" -local new_msg = require("bus").new_msg -local sysinfo = require "services.hal.sysinfo" -local usb3 = require "services.hal.usb3" -local alarms = require "services.system.alarms" -local sc = require "fibers.utils.syscall" - -local system_service = { - name = 'System' -} -system_service.__index = system_service +-- services/system.lua +-- +-- System Service +-- +-- Two long-running fibers: +-- • System Main — config, alarm management, shutdown orchestration +-- • System Sysinfo — periodic metrics reporting via HAL capabilities +-- +-- All hardware interaction is through HAL capability RPCs on the bus. +-- No direct file reads or exec calls are performed here. + +local fibers = require "fibers" +local op = require "fibers.op" +local sleep = require "fibers.sleep" +local channel = require "fibers.channel" + +local perform = fibers.perform + +local base = require 'devicecode.service_base' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local cap_sdk = require 'services.hal.sdk.cap' +local alarms = require "services.system.alarms" + +local SCHEMA_TARGET = "devicecode.config/system/1" + +local cap_unavailable_logged = {} + +-- ── topic helpers ──────────────────────────────── + +---@param name string +---@return Topic +local function t_cfg(name) return { 'cfg', name } end + +---@return Topic +local function t_state_time_synced() + return { 'state', 'time', 'synced' } +end -local function get_boot_time(_) - local uptime, err = sysinfo.get_uptime() - if err then - return nil, err +---@return Topic +local function t_state_system_shutdown() + return { 'state', 'system', 'shutdown' } +end + +---@return Topic +local function t_state_system_identity() + return { 'state', 'system', 'identity' } +end + +---@return Topic +local function t_state_system_stats() + return { 'state', 'system', 'stats' } +end + +-- ── config validation ────────────────────────────── + +---@param cfg any +---@return { report_period: number, usb3_enabled: boolean, alarms?: table }? +---@return string error +local function validate_config(cfg) + if type(cfg) ~= 'table' then + return nil, "config must be a table" + end + if cfg.schema ~= SCHEMA_TARGET then + return nil, "config.schema is not currently supported" + end + if type(cfg.report_period) ~= 'number' or cfg.report_period <= 0 then + return nil, "config.report_period must be a positive number" end - return math.floor(os.time() - uptime) + if type(cfg.usb3_enabled) ~= 'boolean' then + return nil, "config.usb3_enabled must be a boolean" + end + -- alarms is optional + if cfg.alarms ~= nil and type(cfg.alarms) ~= 'table' then + return nil, "config.alarms must be nil or a table" + end + return cfg, "" +end + +-- ── RPC helper ───────────────────────────────── + +local REQUEST_TIMEOUT = 10 +local CAP_RETRY_TIMEOUT = 0.1 + +---@param cap_ref CapabilityReference +---@param method string +---@param opts any +---@param timeout? number +---@return any value +---@return string error +local function cap_rpc(cap_ref, method, opts, timeout) + timeout = timeout or REQUEST_TIMEOUT + local reply, err = perform(cap_ref:call_control_op(method, opts, { timeout = timeout })) + if not reply then return nil, err or "rpc failed" end + if reply.ok ~= true then return nil, reply.reason or "rpc returned not ok" end + return reply.reason, "" +end + +---@param class CapabilityClass +---@param id CapabilityId +---@return string +local function cap_ref_key(class, id) + return tostring(class) .. ':' .. tostring(id) end -local function get_mem_stats(_) - local stats, err = sysinfo.get_ram_info() - if err then - return nil, err +---@param conn Connection +---@param svc ServiceBase +---@param class CapabilityClass +---@param id CapabilityId +---@return CapabilityReference? +---@return string error +local function wait_for_cap(conn, svc, class, id, timeout) + local cap_ref, cap_err = cap_sdk.new_cap_listener(conn, class, id):wait_for_cap({ + timeout = timeout or REQUEST_TIMEOUT, + }) + if not cap_ref then + local key = cap_ref_key(class, id) + local first = not cap_unavailable_logged[key] + cap_unavailable_logged[key] = true + svc:obs_log(first and 'warn' or 'debug', { + what = tostring(class) .. '_unavailable', + summary = string.format( + '%s capability %s unavailable: %s', + tostring(class), + tostring(id), + tostring(cap_err or 'unknown') + ), + class = class, + id = id, + err = cap_err, + }) + return nil, cap_err end - return { - total = stats.total, - used = stats.used, - free = stats.free, - util = (stats.used / stats.total) * 100 - } + return cap_ref, "" end -local METRICS = { - -- static metrics - { method = sysinfo.get_hw_revision, key = { 'hardware', 'revision' } }, - { method = sysinfo.get_fw_version, key = { 'firmware', 'version' } }, - -- Boot time should not be retrieved until time is synced - { method = get_boot_time, key = { 'boot_time' }, needs_time_sync = true }, - { method = sysinfo.get_board_revision, key = { 'hardware', 'board', 'revision' } }, - { method = sysinfo.get_serial, key = { 'hardware', 'serial' } }, - -- dynamic metrics - { method = sysinfo.get_cpu_model, key = { 'cpu', 'cpu_model' } }, - { method = sysinfo.get_cpu_utilisation_and_freq, key = { 'cpu' } }, - { method = get_mem_stats, key = { 'mem' } }, - { method = sysinfo.get_temperature, key = { 'temperature' } }, - { method = sysinfo.get_power_state, key = { 'power' } }, -} +---@param conn Connection +---@param svc ServiceBase +---@param specs { class: CapabilityClass, id: CapabilityId }[] +---@return table +local function discover_caps(conn, svc, specs) + local refs = {} + local seen = {} ----Configure USB hub and alarms ----@param config_msg table -function system_service:_handle_config(ctx, config_msg) - if config_msg.payload == nil then - log.error("System: Invalid configuration message") - return + for _, spec in ipairs(specs) do + local key = cap_ref_key(spec.class, spec.id) + if not seen[key] then + local cap_ref = wait_for_cap(conn, svc, spec.class, spec.id) + if cap_ref then + refs[key] = cap_ref + end + seen[key] = true + end end - local usb3_enabled = config_msg.payload.usb3_enabled - local report_period = config_msg.payload.report_period - if usb3_enabled == nil or report_period == nil then - log.error("System: Missing required configuration fields") - return + return refs +end + +local function discover_missing_caps(conn, svc, refs, specs, timeout) + refs = refs or {} + for _, spec in ipairs(specs or {}) do + local key = cap_ref_key(spec.class, spec.id) + if refs[key] == nil then + local cap_ref = wait_for_cap(conn, svc, spec.class, spec.id, timeout or CAP_RETRY_TIMEOUT) + if cap_ref then refs[key] = cap_ref end + end end + return refs +end + +---@param refs table +---@param class CapabilityClass +---@param id CapabilityId +---@return CapabilityReference? +local function get_cap_ref(refs, class, id) + return refs[cap_ref_key(class, id)] +end + +-- ── sysinfo fiber ──────────────────────────────── + +-- Temperature from zone0 (preserved metric name for historical continuity). +local THERMAL_ZONE0_ID = 'zone0' - self.report_period_channel:put(report_period) +local PLATFORM_IDENTITY_METRICS = { + { field = 'hw_revision', metric_key = 'hw_id' }, + { field = 'fw_version', metric_key = 'fw_id' }, + { field = 'serial', metric_key = 'serial' }, + { field = 'board_revision', metric_key = 'board_revision' }, +} - if not usb3_enabled then - -- this is just a copy-paste job, look at this again during mvp - -- and think about abstraction and renablement(?) etc - log.info(string.format("%s - %s: Disabling USB3", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - usb3.disable_usb3(ctx, self.model) +local SYSINFO_METRICS = { + { + class = 'cpu', + id = '1', + method = 'get', + field = 'utilisation', + metric_key = 'cpu_util', + mk_opts = cap_sdk.args.new.CpuGetOpts + }, + { + class = 'memory', + id = '1', + method = 'get', + field = 'util', + metric_key = 'mem_util', + mk_opts = cap_sdk.args.new.MemoryGetOpts + }, + { + class = 'thermal', + id = THERMAL_ZONE0_ID, + method = 'get', + field = '', + metric_key = 'temp', + mk_opts = function(_, max_age) return cap_sdk.args.new.ThermalGetOpts(max_age) end + } +} + +local function publish_system_metric(svc, metric_key, value) + if value == nil then return false end + svc:obs_metric(metric_key, { + namespace = { 'system', metric_key }, + value = value + }) + return true +end + + +local function publish_platform_identity_state(svc, identity) + if type(identity) ~= 'table' then return false end + svc:_retain(t_state_system_identity(), { + schema = 'devicecode.system.identity/1', + state = 'ok', + observed_at = fibers.now(), + at = svc:wall(), + hw_revision = identity.hw_revision, + fw_version = identity.fw_version, + serial = identity.serial, + board_revision = identity.board_revision, + }) + return true +end + +local function collect_sysinfo_state(svc, cap_refs, max_age) + local out = { + schema = 'devicecode.system.stats/1', + state = 'ok', + observed_at = fibers.now(), + at = svc:wall(), + missing = {}, + } + + local function missing(key, err) + out.missing[#out.missing + 1] = { key = key, err = tostring(err or 'unavailable') } + out.state = 'partial' end - local cfg_alarms = config_msg.payload.alarms - if cfg_alarms and type(cfg_alarms) == "table" then - self.alarm_manager:delete_all() - for _, alarm in ipairs(cfg_alarms) do - local add_err = self.alarm_manager:add(alarm) - if add_err then - log.error("Failed to add alarm: ", add_err) + for _, m in ipairs(SYSINFO_METRICS) do + local cap_ref = get_cap_ref(cap_refs, m.class, m.id) + if not cap_ref then + missing(m.class .. ':' .. m.id, 'capability_unavailable') + else + local opts, opts_err = m.mk_opts(m.field, max_age) + if opts_err ~= "" then + missing(m.class .. ':' .. m.id, opts_err) + else + local value, err = cap_rpc(cap_ref, m.method, opts) + if err ~= "" then + missing(m.class .. ':' .. m.id, err) + elseif m.metric_key == 'cpu_util' then + out.cpu = { utilisation = tonumber(value) or value } + elseif m.metric_key == 'mem_util' then + out.memory = { utilisation = tonumber(value) or value } + elseif m.metric_key == 'temp' then + out.thermal = out.thermal or {} + out.thermal[THERMAL_ZONE0_ID] = { temp_c = tonumber(value) or value } + end end end end + + if #out.missing == 0 then out.missing = nil end + if out.cpu == nil and out.memory == nil and out.thermal == nil then out.state = 'unavailable' end + return out end -local function build_table(keys, values) - local tbl = {} - local head = tbl - for i = 1, #keys-1 do - head[keys[i]] = {} - head = head[keys[i]] - end - head[keys[#keys]] = values - return tbl +local function publish_sysinfo_state(svc, cap_refs, stats_period) + local state = collect_sysinfo_state(svc, cap_refs, stats_period) + svc:_retain(t_state_system_stats(), state) + return state +end + +local function unsubscribe(sub) + if sub and type(sub.unsubscribe) == 'function' then sub:unsubscribe() end end -local function merge_tables(primary, secondary) - for k, v in pairs(secondary) do - if type(v) == "table" and type(primary[k]) == "table" then - merge_tables(primary[k], v) +local function read_platform_identity(platform_cap, timeout) + if not platform_cap then return nil, 'platform_unavailable' end + timeout = timeout or REQUEST_TIMEOUT + + local identity_opts, identity_opts_err = cap_sdk.args.new.PlatformGetOpts('identity', 0) + if identity_opts then + local identity, err = cap_rpc(platform_cap, 'get', identity_opts, timeout) + if err == "" and type(identity) == 'table' then + return identity, "" + end + elseif identity_opts_err ~= "" then + return nil, identity_opts_err + end + + local identity_sub = platform_cap:get_state_sub('identity', { + queue_len = 1, + full = 'drop_oldest', + }) + local identity_msg = perform(op.choice( + identity_sub:recv_op(), + sleep.sleep_op(timeout) + )) + unsubscribe(identity_sub) + if identity_msg and type(identity_msg.payload) == 'table' then + return identity_msg.payload, "" + end + + local out = {} + local got = false + local last_err = nil + for _, metric in ipairs(PLATFORM_IDENTITY_METRICS) do + local opts, opts_err = cap_sdk.args.new.PlatformGetOpts(metric.field, 0) + if opts then + local value, err = cap_rpc(platform_cap, 'get', opts, timeout) + if err == "" then + out[metric.field] = value + got = true + else + last_err = err + end else - primary[k] = v + last_err = opts_err end end + if got then return out, "" end + return nil, last_err or 'platform_identity_unavailable' end ----Periodic gathering a publish of system information -function system_service:_report_sysinfo(ctx) - local time_synced_sub = self.conn:subscribe({ 'time', 'ntp_synced' }) - local time_synced = false - local hw_revision = sysinfo.get_hw_revision() - if hw_revision then - self.model = hw_revision:match('(%S+)') -- this should be put on a channel +local function publish_platform_identity_metrics(svc, platform_cap, timeout) + local identity, err = read_platform_identity(platform_cap, timeout) + if type(identity) ~= 'table' then + svc:obs_log('warn', { what = 'platform_identity_unavailable', err = tostring(err or 'unknown') }) + return false end - local report_period = self.report_period_channel:get() - while not ctx:err() do - local stats = {} - for _, metric in ipairs(METRICS) do - if (metric.needs_time_sync and time_synced) or (not metric.needs_time_sync) then - local result, err - if metric.method then - result, err = metric.method(ctx) + publish_platform_identity_state(svc, identity) + + local published = false + for _, metric in ipairs(PLATFORM_IDENTITY_METRICS) do + if identity[metric.field] ~= nil then + published = publish_system_metric(svc, metric.metric_key, identity[metric.field]) or published + end + end + + if published then + svc:obs_log('debug', 'sysinfo: published platform identity metrics') + else + svc:obs_log('warn', 'sysinfo: platform identity had no publishable fields') + end + return published +end + +local function publish_sysinfo_metrics(svc, cap_refs, report_period) + for _, m in ipairs(SYSINFO_METRICS) do + local cap_ref = get_cap_ref(cap_refs, m.class, m.id) + if cap_ref then + local opts, opts_err = m.mk_opts(m.field, report_period) + if opts_err ~= "" then + svc:obs_log('warn', { + what = 'metric_opts_invalid', + class = m.class, + id = m.id, + field = m.field, + err = opts_err + }) + else + local value, err = cap_rpc(cap_ref, m.method, opts) + if err ~= "" then + svc:obs_log('warn', { + what = 'metric_get_failed', + class = m.class, + id = m.id, + field = m.field, + err = err + }) else - err = "No method defined for metric" - end - if err then - log.error(string.format("System: Failed to get metric for %s: %s", - table.concat(metric.key, '.'), err)) - elseif result then - local keyed_table = build_table(metric.key, result) - merge_tables(stats, keyed_table) + publish_system_metric(svc, m.metric_key, value) end end end + end +end - self.conn:publish_multiple( - { 'system', 'info' }, - stats, - { retained = true } - ) - - op.choice( - sleep.sleep_op(report_period), - self.report_period_channel:get_op():wrap(function(new_period) - report_period = new_period - end), - time_synced_sub:next_msg_op():wrap(function(msg) - time_synced = (msg.payload == true) - end), - ctx:done_op() - ):perform() +local function publish_boot_time_metric(svc, platform_cap, report_period) + if not platform_cap then return end + local uptime_opts, uptime_opts_err = cap_sdk.args.new.PlatformGetOpts('uptime', report_period) + if uptime_opts_err ~= "" then + svc:obs_log('warn', { what = 'uptime_opts_invalid', err = uptime_opts_err }) + else + local uptime, uptime_err = cap_rpc(platform_cap, 'get', uptime_opts) + if uptime == nil or uptime_err ~= "" then + svc:obs_log('warn', { what = 'uptime_get_failed', err = uptime_err }) + else + publish_system_metric(svc, 'boot_time', os.time() - math.floor(uptime)) + end end end ----Performs shutdown or reboot of the system ----@param alarm Alarm -function system_service:_handle_alarm(ctx, alarm) - local name = alarm.payload and alarm.payload.name - local type = alarm.payload and alarm.payload.type - if type ~= 'reboot' and type ~= 'shutdown' then return end - local deadline = sc.monotime() + 10 - self.conn:publish(new_msg( - { '+', 'control', 'shutdown' }, - { reason = name, deadline = deadline }, - { retained = true } - )) +---@param _ Scope +---@param svc ServiceBase +---@param report_period_ch Channel +local function sysinfo_fiber(_, svc, report_period_ch) + local conn = svc.conn + svc:obs_log('debug', 'sysinfo started') - -- need to create context that is independent from the service context - -- as the broadcast shutdown message will also shutdown the system service - local shutdown_timeout = context.with_deadline(context.background(), deadline + 1) - local shutdown_sub = self.conn:subscribe({ '+', 'health' }) + fibers.current_scope():finally(function() + local _, primary = fibers.current_scope():status() + bus_cleanup.unretain(svc.conn, t_state_system_stats()) + bus_cleanup.unretain(svc.conn, t_state_system_identity()) + svc:obs_log('debug', { what = 'sysinfo_stopped', reason = tostring(primary or 'ok') }) + end) - local active_services = {} + local cap_specs = { + { class = 'platform', id = '1' }, + } + for _, metric in ipairs(SYSINFO_METRICS) do + cap_specs[#cap_specs + 1] = { class = metric.class, id = metric.id } + end - while not shutdown_timeout:err() do - local msg, timeout_err = shutdown_sub:next_msg_with_context_op(shutdown_timeout):perform() - if timeout_err then break end - local service_name = msg.topic[1] - if service_name ~= self.name then - if msg.payload.state == 'disabled' and active_services[service_name] then - active_services[service_name] = nil - end - if msg.payload.state ~= 'disabled' and not active_services[service_name] then - active_services[service_name] = true - end + local cap_refs = discover_caps(conn, svc, cap_specs) + local platform_cap = get_cap_ref(cap_refs, 'platform', '1') + local identity_published = false + + -- Block until System Main sends us the initial report_period. + local report_period = report_period_ch:get() + if not report_period then + svc:obs_log('warn', 'sysinfo: report_period channel closed before config received') + return + end + svc:obs_log('debug', { what = 'report_period_set', value = report_period }) + + local function refresh_caps(timeout) + discover_missing_caps(conn, svc, cap_refs, cap_specs, timeout) + platform_cap = get_cap_ref(cap_refs, 'platform', '1') + if not identity_published and platform_cap then + identity_published = publish_platform_identity_metrics(svc, platform_cap, timeout or REQUEST_TIMEOUT) end end - shutdown_sub:unsubscribe() - - for service_name, _ in pairs(active_services) do - local err_msg = string.format("Service %s did not shut down safely due to hanging fibers:\n", service_name) - local service_fibers_status_sub = self.conn:subscribe( - { service_name, 'health', 'fibers', '+' } - ) - while true do - local msg, is_break = service_fibers_status_sub:next_msg_op():perform_alt(function() - return nil, true - end) - if is_break then break end - if msg.payload.state ~= 'disabled' then - err_msg = err_msg .. string.format("\tFiber %s is stuck in state %s\n", msg.topic[4], msg.payload.state) + + -- Subscribe to time sync. + local time_sub = conn:subscribe(t_state_time_synced()) + + local time_synced = false + local stats_period = math.max(1, math.min(tonumber(report_period) or 5, 5)) + local next_metrics_at = fibers.now() + + refresh_caps(CAP_RETRY_TIMEOUT) + publish_sysinfo_state(svc, cap_refs, stats_period) + publish_sysinfo_metrics(svc, cap_refs, report_period) + next_metrics_at = fibers.now() + report_period + + while true do + local choices = { + stats = sleep.sleep_op(stats_period), + period = report_period_ch:get_op(), + time = time_sub:recv_op(), + -- time = time_synced and op.never() or op.always( { payload = true } ) + } + + local which, msg = perform(op.named_choice(choices)) + + if which == 'period' then + if msg then + report_period = msg + stats_period = math.max(1, math.min(tonumber(report_period) or 5, 5)) + next_metrics_at = fibers.now() + svc:obs_log('debug', { what = 'report_period_updated', value = report_period, stats_period = stats_period }) + refresh_caps(CAP_RETRY_TIMEOUT) + publish_sysinfo_state(svc, cap_refs, stats_period) + publish_sysinfo_metrics(svc, cap_refs, report_period) + next_metrics_at = fibers.now() + report_period + else + svc:obs_log('debug', 'sysinfo: report_period channel closed') + return + end + elseif which == 'time' then + if not msg then + svc:obs_log('debug', 'sysinfo: time subscription closed') + return + end + time_synced = (msg.payload == true) + svc:obs_log('debug', { what = 'time_synced_updated', value = time_synced }) + elseif which == 'stats' then + refresh_caps(CAP_RETRY_TIMEOUT) + publish_sysinfo_state(svc, cap_refs, stats_period) + + if fibers.now() >= next_metrics_at then + publish_sysinfo_metrics(svc, cap_refs, report_period) + if time_synced and platform_cap then + publish_boot_time_metric(svc, platform_cap, report_period) + end + next_metrics_at = fibers.now() + report_period end end - service_fibers_status_sub:unsubscribe() - log.debug(err_msg) end +end + +-- ── shutdown orchestration ───────────────────────────── + +local SHUTDOWN_GRACE = 10 -- seconds services have to shut down +local USB3_MODEL = "bigbox-ss" -- only model with controllable USB3 hardware - if type == 'shutdown' then - log.info(string.format("%s - %s: Shutting down system", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - local cmd = exec.command('shutdown', '-h', 'now') - cmd:run() - elseif type == 'reboot' then - log.info(string.format("%s - %s: Rebooting system", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - local cmd = exec.command('reboot') - cmd:run() +---@param svc ServiceBase +---@param power_cap CapabilityReference? +---@param alarm SystemAlarm +local function handle_alarm(svc, power_cap, alarm) + local conn = svc.conn + local payload = alarm.payload or {} + local alarm_name = payload.name or "scheduled alarm" + local alarm_type = payload.type or "reboot" + + svc:obs_log('info', { what = 'alarm_fired', name = alarm_name, type = alarm_type }) + svc:obs_event('alarm_fired', { name = alarm_name, type = alarm_type, ts = svc:now() }) + + if not power_cap then + svc:obs_log('error', { what = 'alarm_aborted', reason = 'power cap unavailable', alarm = alarm_name }) + return + end + + -- Broadcast shutdown signal so all services can clean up within the deadline. + conn:retain(t_state_system_shutdown(), { + reason = alarm_name, + deadline = fibers.now() + SHUTDOWN_GRACE, + }) + + -- After the grace period, issue the power command via the capability. + local ok, spawn_err = fibers.current_scope():spawn(function() + perform(sleep.sleep_op(SHUTDOWN_GRACE)) + svc:obs_log('info', { what = 'power_command', type = alarm_type }) + local power_opts = cap_sdk.args.new.PowerActionOpts() + local _, err = cap_rpc(power_cap, alarm_type, power_opts) + if err ~= "" then + svc:obs_log('error', { what = 'power_command_failed', type = alarm_type, err = err }) + end + end) + if not ok then + svc:obs_log('error', { what = 'power_fiber_spawn_failed', err = tostring(spawn_err) }) end end ---- Main system service loop -function system_service:_system_main(ctx) - -- Subscribe to system-related topics - local config_sub = self.conn:subscribe({ 'config', 'system' }) - local time_sync_sub = self.conn:subscribe({ 'time', 'ntp_synced' }) - - while not ctx:err() do - op.choice( - ctx:done_op(), - config_sub:next_msg_op():wrap(function(config_msg) - self:_handle_config(ctx, config_msg) - end), - time_sync_sub:next_msg_op():wrap(function(msg) - if msg.payload then - self.alarm_manager:sync() - else - self.alarm_manager:desync() +-- ── system main fiber ────────────────────────────── + +---@param svc ServiceBase +---@param report_period_ch Channel +local function system_main(svc, report_period_ch) + local conn = svc.conn + svc:obs_log('debug', 'main started') + + local parent_scope = fibers.current_scope() + parent_scope:finally(function() + local _, primary = parent_scope:status() + svc:obs_log('debug', { what = 'main_stopped', reason = tostring(primary or 'ok') }) + end) + + -- Acquire platform cap to read hw_revision from identity state. + local platform_cap = wait_for_cap(conn, svc, 'platform', '1') + + -- Read hw_revision to gate USB3 control to bigbox-ss hardware only. + local hw_revision = nil + if platform_cap then + local identity, identity_err = read_platform_identity(platform_cap, REQUEST_TIMEOUT) + if type(identity) == 'table' and identity.hw_revision then + hw_revision = identity.hw_revision:match('(%S+)') + svc:obs_log('info', { what = 'hw_revision_detected', value = hw_revision }) + svc:obs_log('info', { + what = 'system_summary', + summary = string.format('system summary hw=%s', tostring(hw_revision)), + hw_revision = hw_revision, + }) + else + svc:obs_log('warn', { + what = 'platform_identity_unavailable', + summary = 'platform identity not available at startup; USB3 control disabled', + err = tostring(identity_err or 'unknown'), + }) + end + end + + -- Acquire USB cap only if this is bigbox-ss hardware. + local usb_cap = nil + if hw_revision == USB3_MODEL then + usb_cap = wait_for_cap(conn, svc, 'usb', 'usb3') + end + + -- Acquire power cap upfront — needed when alarms fire. + local power_cap = wait_for_cap(conn, svc, 'power', '1') + + local alarm_mgr = alarms.AlarmManager.new() + local cfg_sub = conn:subscribe(t_cfg(svc.name)) + local time_sub = conn:subscribe(t_state_time_synced()) + + while true do + local choices = { + cfg = cfg_sub:recv_op(), + time = time_sub:recv_op(), + alarm = alarm_mgr:next_alarm_op(), + } + + local which, msg = perform(op.named_choice(choices)) + + if not msg and (which == 'cfg' or which == 'time') then + svc:obs_log('debug', { what = 'subscription_closed', source = which }) + return + end + + if which == 'cfg' then + local cfg, err = validate_config(msg.payload and msg.payload.data) + if cfg == nil or err ~= "" then + svc:obs_log('warn', { what = 'config_invalid', err = err }) + else + svc:obs_event('config_applied', { ts = svc:now() }) + + -- Forward report_period to sysinfo fiber. + -- Drain any stale unconsumed value first (non-blocking via or_else so + -- the channel is tried first), then put the latest value. + perform(report_period_ch:get_op():or_else(function() return nil end)) + report_period_ch:put(cfg.report_period) + + -- Handle USB3 control (bigbox-ss hardware only). + if usb_cap then + local usb_verb = cfg.usb3_enabled and 'enable' or 'disable' + local _, usb_err = cap_rpc(usb_cap, usb_verb, {}) + if usb_err ~= "" then + svc:obs_log('warn', { what = 'usb3_control_failed', verb = usb_verb, err = usb_err }) + end end - end), - self.alarm_manager:next_alarm_op():wrap(function(alarm) - self:_handle_alarm(ctx, alarm) - end) - ):perform() + + -- Reload alarms. + alarm_mgr:delete_all() + if type(cfg.alarms) == 'table' then + for _, alarm_cfg in ipairs(cfg.alarms) do + local add_err = alarm_mgr:add(alarm_cfg) + if add_err ~= "" then + svc:obs_log('warn', { what = 'alarm_add_failed', err = add_err }) + end + end + end + end + elseif which == 'time' then + local is_synced = (msg.payload == true) + if is_synced then + alarm_mgr:sync() + svc:obs_log('debug', 'alarm manager synced') + else + alarm_mgr:desync() + svc:obs_log('debug', 'alarm manager desynced') + end + elseif which == 'alarm' then + handle_alarm(svc, power_cap, msg) + end end - config_sub:unsubscribe() - time_sync_sub:unsubscribe() end ----Start the system service ----@param ctx Context +-- ── service entry point ────────────────────────────── + +---@class SystemService +local SystemService = {} + ---@param conn Connection -function system_service:start(ctx, conn) - self.conn = conn - self.report_period_channel = channel.new() - self.alarm_manager = alarms.AlarmManager.new() - log.trace("Starting System Service") - service.spawn_fiber('System Main', conn, ctx, function(fctx) - self:_system_main(fctx) - end) - service.spawn_fiber('System Sysinfo', conn, ctx, function(fctx) - self:_report_sysinfo(fctx) +---@param opts? { name?: string, env?: string, heartbeat_s?: number } +function SystemService.start(conn, opts) + opts = opts or {} + + local svc = base.new(conn, { name = opts.name or 'system', env = opts.env }) + local heartbeat_s = (type(opts.heartbeat_s) == 'number') and opts.heartbeat_s or 30.0 + + svc:obs_state('boot', { at = svc:wall(), ts = svc:now(), state = 'entered' }) + svc:obs_log('debug', 'service start() entered') + svc:announce({}) + svc:starting() + svc:spawn_heartbeat(heartbeat_s, 'tick') + + -- Channel carries report_period from System Main → System Sysinfo. + -- Buffer of 1 so a rapid double-update does not block Main. + local report_period_ch = channel.new(1) + + fibers.current_scope():finally(function() + local _, primary = fibers.current_scope():status() + svc:lifecycle('stopped', { ready = false, reason = tostring(primary or 'ok') }) + svc:obs_log('debug', 'service stopped') end) -end -if _G._TEST then - return { - system_service = system_service, - build_table = build_table, - merge_tables = merge_tables - } -else - return system_service + -- Spawn Sysinfo first so it is ready to receive from report_period_ch. + fibers.current_scope():spawn(sysinfo_fiber, svc, report_period_ch) + + svc:running() + svc:obs_log('debug', 'service running') + + -- Run System Main in the calling fiber. + system_main(svc, report_period_ch) end + +SystemService._test = { + publish_platform_identity_metrics = publish_platform_identity_metrics, + read_platform_identity = read_platform_identity, + publish_sysinfo_metrics = publish_sysinfo_metrics, + collect_sysinfo_state = collect_sysinfo_state, + publish_sysinfo_state = publish_sysinfo_state, +} + +return SystemService diff --git a/src/services/system/alarms.lua b/src/services/system/alarms.lua index c84857ed..138a35e6 100644 --- a/src/services/system/alarms.lua +++ b/src/services/system/alarms.lua @@ -1,49 +1,103 @@ -local op = require 'fibers.op' +-- services/system/alarms.lua +-- +-- Alarm and AlarmManager for the system service. +-- Adapted to the current fibers.alarm API for wall-clock scheduling. + +local op = require 'fibers.op' local falarm = require "fibers.alarm" -local sc = require 'fibers.utils.syscall' local REPEAT_TYPES = { - NONE = "none", + NONE = "none", DAILY = "daily", - -- WEEKLY = "weekly", - -- MONTHLY = "monthly" } ----@class Alarm ----@field payload any Any custom data to be held by the alarm ----@field trigger_time table Table with hour and minute fields ----@field repeat_type string The repeat type (none, daily, weekly, monthly) ----@field next_trigger number The monotonic time when this alarm will next trigger +---@class AlarmTriggerTime +---@field hour integer +---@field min integer + +---@param trigger_time AlarmTriggerTime +---@param repeat_type string +---@param last number? +---@param now number +---@return number? +local function next_local_trigger(trigger_time, repeat_type, last, now) + local t + + if last ~= nil then + if repeat_type == REPEAT_TYPES.NONE then + return nil + end + + t = os.date('*t', last) + t.day = t.day + 1 + else + t = os.date('*t', now) + local past = t.hour > trigger_time.hour + or (t.hour == trigger_time.hour and t.min >= trigger_time.min) + if past then + t.day = t.day + 1 + end + end + + ---@cast t osdate + t.hour, t.min, t.sec = trigger_time.hour, trigger_time.min, 0 + return os.time(t) +end + +---@param trigger_time AlarmTriggerTime +---@param repeat_type string +---@return Alarm +local function build_wait_alarm(trigger_time, repeat_type) + return falarm.new { + next_time = function(last, now) + return next_local_trigger(trigger_time, repeat_type, last, now) + end, + label = string.format('system_%s_%02d:%02d', repeat_type, trigger_time.hour, trigger_time.min), + } +end + +---@class AlarmConfig +---@field time string +---@field repeats string? +---@field payload any + +---@class SystemAlarm +---@field payload any Any custom data to be held by the alarm +---@field trigger_time AlarmTriggerTime +---@field repeat_type string "none" or "daily" +---@field next_trigger number Next wall-clock trigger epoch +---@field wait_alarm Alarm local Alarm = {} Alarm.__index = Alarm ---- Create a new alarm from either parameters or a configuration table. ----@param config table Either the name of the alarm or a configuration table ----@return Alarm? alarm A new alarm instance or nil if error ----@return string? error Error message if validation fails +--- Create a new Alarm from a configuration table. +---@param config AlarmConfig +---@return SystemAlarm? alarm +---@return string error function Alarm.new(config) - if not config.time then + if type(config) ~= 'table' then + return nil, "Alarm config must be a table" + end + + if type(config.time) ~= 'string' or config.time == '' then return nil, "Alarm needs a time" end - local hour, minute - if config.time then - -- Parse datetime (expected format: "HH:MM") - hour, minute = config.time:match("^(%d+):(%d+)$") - if not hour or not minute then - return nil, "Invalid datetime format, expected 'HH:MM'" - end - hour, minute = tonumber(hour), tonumber(minute) - if not hour or not minute or hour < 0 or hour > 23 or minute < 0 or minute > 59 then - return nil, "Invalid datetime format, expected 'HH:MM', 0 < HH < 23, 0 < MM < 59" - end - else - hour, minute = 0, 0 + local hour, minute = config.time:match("^(%d+):(%d+)$") + if not hour or not minute then + return nil, "Invalid datetime format, expected 'HH:MM'" + end + hour, minute = tonumber(hour), tonumber(minute) + if not hour or not minute or + hour < 0 or hour > 23 or minute < 0 or minute > 59 then + return nil, "Invalid datetime format, expected 'HH:MM', 0 <= HH <= 23, 0 <= MM <= 59" end - -- Parse repeat type local repeat_type = REPEAT_TYPES.NONE - if config.repeats then + if config.repeats ~= nil then + if type(config.repeats) ~= 'string' then + return nil, "Invalid repeat type" + end local repeat_upper = string.upper(config.repeats) if REPEAT_TYPES[repeat_upper] then repeat_type = REPEAT_TYPES[repeat_upper] @@ -52,128 +106,119 @@ function Alarm.new(config) end end - -- Create trigger_time table - local trigger_time = { - hour = hour, - min = minute - } - - local _, err = falarm.validate_next_table(trigger_time) - if err then return nil, err end + local trigger_time = { hour = hour, min = minute } - -- local time_trigger, _ = falarm.calculate_next(trigger_time, sc.realtime()) + local next_trigger = next_local_trigger(trigger_time, repeat_type, nil, os.time()) + if not next_trigger then + return nil, "Failed to calculate next trigger" + end - -- Create the alarm with parsed values - local alarm = setmetatable({ - payload = config.payload, + return setmetatable({ + payload = config.payload, trigger_time = trigger_time, - repeat_type = repeat_type, - -- next_trigger = time_trigger - }, Alarm) - return alarm + repeat_type = repeat_type, + next_trigger = next_trigger, + wait_alarm = build_wait_alarm(trigger_time, repeat_type), + }, Alarm), "" end +---@return string error function Alarm:calc_next_trigger() - self.next_trigger = falarm.calculate_next(self.trigger_time, sc.realtime()) + local next_trigger = next_local_trigger(self.trigger_time, self.repeat_type, nil, os.time()) + if not next_trigger then + return "Failed to calculate next trigger" + end + + self.next_trigger = next_trigger + self.wait_alarm = build_wait_alarm(self.trigger_time, self.repeat_type) + return "" end ----@class AlarmManager +---@class SystemAlarmManager +---@field alarms SystemAlarm[] sorted by next_trigger ascending +---@field is_synced boolean local AlarmManager = {} AlarmManager.__index = AlarmManager ---- Create a new AlarmManager. ----@return AlarmManager manager A new AlarmManager instance +---@return SystemAlarmManager function AlarmManager.new() - return setmetatable({ - next_alarm = nil, - is_synced = false - }, AlarmManager) + return setmetatable({ alarms = {}, is_synced = false }, AlarmManager) end ---- Add an alarm to the manager. ----@param alarm Alarm|table The alarm to add or a config of an alarm ----@return string? error Error message if alarm is invalid +--- Add an alarm (or a config table that describes one) in sorted order. +---@param alarm SystemAlarm|AlarmConfig +---@return string error function AlarmManager:add(alarm) - -- If alarm is a config table, create an Alarm instance if type(alarm) == "table" and getmetatable(alarm) ~= Alarm then + ---@cast alarm AlarmConfig local new_alarm, err = Alarm.new(alarm) - if not new_alarm then - return err - end + if not new_alarm then return err end alarm = new_alarm end - -- Validate alarm if not alarm or getmetatable(alarm) ~= Alarm then return "Invalid alarm object" end - alarm:calc_next_trigger() + local calc_err = alarm:calc_next_trigger() + if calc_err ~= "" then + return calc_err + end - if self.next_alarm then - local prev = self.next_alarm - local head = self.next_alarm - while head and head.alarm.next_trigger < alarm.next_trigger do - prev = head - head = head.next_alarm - end - if head == self.next_alarm then - self.next_alarm = {alarm = alarm, next_alarm = head} - else - prev.next_alarm = { alarm = alarm, next_alarm = head } - end - else - self.next_alarm = { alarm = alarm, next_alarm = nil } + local i = 1 + -- Insert the alarm in the sorted position based on next_trigger + while i <= #self.alarms and self.alarms[i].next_trigger < alarm.next_trigger do + i = i + 1 end + table.insert(self.alarms, i, alarm) + return "" end ---- Delete all alarms from the manager. +--- Remove all alarms. function AlarmManager:delete_all() - self.next_alarm = nil + self.alarms = {} end +--- Mark the manager as synced, recalculating all trigger times from now. function AlarmManager:sync() if self.is_synced then return end self.is_synced = true - local head = self.next_alarm + local old = self.alarms self:delete_all() - while head do - self:add(head.alarm) - head = head.next_alarm + for _, alarm in ipairs(old) do + self:add(alarm) end end +--- Mark the manager as desynced (alarms will not fire until re-synced). function AlarmManager:desync() self.is_synced = false end ---- Create an operation that waits until the next alarm triggers. ----@return table operation Operation that resolves when the next alarm triggers +--- Return an op that resolves when the next alarm fires, yielding the alarm. +--- If no alarms are pending or the manager is not synced, returns an op that +--- never resolves. +---@return Op operation function AlarmManager:next_alarm_op() - if not self.next_alarm or not self.is_synced then - -- No alarms, create an operation that will never complete - return op.new_base_op( - nil, - function() return false end, - function() end - ) + if #self.alarms == 0 or not self.is_synced then + return op.never() end - -- Create an operation that will sleep until the next alarm triggers - return falarm.wait_absolute_op(self.next_alarm.alarm.next_trigger):wrap(function() - local alarm = self.next_alarm.alarm - self.next_alarm = self.next_alarm.next_alarm - -- Recalculate the next trigger time for repeating alarms + local head = self.alarms[1] + return head.wait_alarm:wait_op():wrap(function() + local alarm = table.remove(self.alarms, 1) + -- Re-queue repeating alarms. if alarm.repeat_type ~= REPEAT_TYPES.NONE then - self:add(alarm) + local add_err = self:add(alarm) + if add_err ~= "" then + error("Failed to re-add repeating alarm: " .. add_err) + end end - - -- Return the alarm that triggered return alarm end) end return { - Alarm = Alarm, + Alarm = Alarm, AlarmManager = AlarmManager, } diff --git a/src/services/time.lua b/src/services/time.lua index 8ab17553..712f80c5 100644 --- a/src/services/time.lua +++ b/src/services/time.lua @@ -1,99 +1,160 @@ --- Importing necessary modules -local fiber = require "fibers.fiber" +-- services/time.lua +-- +-- Time service: +-- - discovers the first HAL time capability via {'cap','time','+','meta','source'} +-- - consumes retained state + sync/unsync events from that capability +-- - publishes retained {'state','time','synced'} for system-wide time trust state +-- - nudges fibers alarm wall-clock handling when sync transitions occur + +local fibers = require "fibers" local op = require "fibers.op" -local exec = require 'fibers.exec' -local alarm = require 'fibers.alarm' -local cjson = require "cjson.safe" -local log = require 'services.log' -local context = require "fibers.context" -local new_msg = require('bus').new_msg - -local time_service = { - name = "time" -} -time_service.__index = time_service - --- Constants -local BUS_TIMEOUT = 5 - -local function ntpd_monitor(ctx) - -- Wait for ubus capability to be active - local ubus_active_sub = time_service.bus_connection:subscribe({'hal', 'capability', 'ubus', '1'}) - ubus_active_sub:next_msg() -- Block until ubus capability is active - - log.trace("TIME: NTP monitor starting") - - -- Restart sysntpd - local err = exec.command("/etc/init.d/sysntpd", "restart"):run() - if err then - log.error("TIME: sysntpd restart failed:", err) - return - end - - -- Start listening for hotplug.ntp events via HAL - local listen_sub = time_service.bus_connection:request(new_msg( - { 'hal', 'capability', 'ubus', '1', 'control', 'listen' }, - { 'hotplug.ntp' } - )) - local listen_response, ctx_err = listen_sub:next_msg_with_context(context.with_timeout(ctx, BUS_TIMEOUT)) - if ctx_err or listen_response.payload.err then - local err = ctx_err or listen_response.payload.err - log.error("TIME: ubus listen failed:", err) - return - end - - local stream_id = listen_response.payload.result.stream_id - local hotplug_sub = time_service.bus_connection:subscribe( - { 'hal', 'capability', 'ubus', '1', 'info', 'stream', stream_id } - ) - local stream_end_sub = time_service.bus_connection:subscribe( - { 'hal', 'capability', 'ubus', '1', 'info', 'stream', stream_id, 'closed' } - ) - - local process_ended = false - - while not ctx:err() and not process_ended do - op.choice( - hotplug_sub:next_msg_op():wrap(function(msg) - local event = msg.payload - if not event then return end - local data = event["hotplug.ntp"] - if data and data.stratum then - log.debug("TIME: ntpd hotplug event received!") - if data.stratum ~= 16 then - log.debug("TIME: ntp time synced!") - time_service.bus_connection:publish(new_msg({ "time", "ntp_synced" }, true, - { retained = true })) - alarm.clock_synced() - else - log.debug("TIME: ntp time desynced") - time_service.bus_connection:publish(new_msg({ "time", "ntp_synced" }, false, - { retained = true })) - alarm.clock_desynced() - end - else - log.warn("TIME: Received unknown hotplug.ntp event") - end - end), - stream_end_sub:next_msg_op():wrap(function(stream_ended) - if stream_ended.payload then - process_ended = true - end - end), - ctx:done_op():wrap(function() - time_service.bus_connection:publish(new_msg( - { 'hal', 'capability', 'ubus', '1', 'control', 'stop_stream' }, - { stream_id } - )) - end) - ):perform() - end +local alarm = require "fibers.alarm" +local time_utils = require "fibers.utils.time" + +local base = require "devicecode.service_base" +local cap_sdk = require "services.hal.sdk.cap" + +local perform = fibers.perform + +local M = {} + +---@param payload any +---@return boolean? synced +local function synced_from_state_payload(payload) + if type(payload) ~= 'table' then return nil end + if type(payload.synced) ~= 'boolean' then return nil end + return payload.synced +end + +---@param state table +---@param conn Connection +---@param svc ServiceBase +---@param is_synced boolean +---@param payload table? +---@return nil +local function apply_sync_state(state, conn, svc, is_synced, payload) + if state.current_synced ~= is_synced then + local what = is_synced and 'time_synced' or 'time_unsynced' + svc:obs_log(is_synced and 'info' or 'warn', { + what = what, + summary = is_synced and 'time synchronised' or 'time synchronisation lost', + from = state.current_synced, + to = is_synced, + }) + conn:retain({ 'state', 'time', 'synced' }, is_synced) + svc:obs_event(is_synced and 'synced' or 'unsynced', payload or {}) + state.current_synced = is_synced + end + + -- New fibers core has set_time_source/time_changed instead of + -- install_alarm_handler/clock_synced/clock_desynced. + if is_synced then + if not state.time_source_installed then + svc:obs_log('debug', 'installing alarm time source from realtime clock') + local ok, err = pcall(alarm.set_time_source, time_utils.realtime) + if ok then + state.time_source_installed = true + svc:obs_log('debug', 'alarm time source installed') + else + svc:obs_log('warn', { what = 'alarm_time_source_failed', err = tostring(err) }) + end + else + alarm.time_changed() + end + end +end + +---@param conn Connection +---@param svc ServiceBase +---@param cap_ref CapabilityReference +---@return nil +local function monitor_capability(conn, svc, cap_ref) + svc:obs_log('debug', { what = 'capability_monitor_start', cap_id = tostring(cap_ref.id) }) + local sub_state = cap_ref:get_state_sub('synced') + local sub_synced = cap_ref:get_event_sub('synced') + local sub_unsynced = cap_ref:get_event_sub('unsynced') + + local state = { + ---@type boolean? + current_synced = nil, + ---@type boolean + time_source_installed = false, + } + + -- Read retained initial state once, then rely on events for transitions. + do + local msg, err = perform(sub_state:recv_op()) + if msg then + svc:obs_log('debug', 'received initial retained sync state') + local is_synced = synced_from_state_payload(msg.payload) + if is_synced ~= nil then + apply_sync_state(state, conn, svc, is_synced, msg.payload) + else + svc:obs_log('warn', { what = 'initial_state_invalid', reason = 'missing boolean synced field' }) + end + else + svc:obs_log('warn', { what = 'initial_state_read_failed', err = tostring(err) }) + end + sub_state:unsubscribe() + end + + while true do + local which, msg, err = perform(op.named_choice({ + synced = sub_synced:recv_op(), + unsynced = sub_unsynced:recv_op(), + })) + + if not msg then + svc:obs_log('warn', { what = 'capability_monitor_closed', err = tostring(err) }) + return + end + + if which == 'synced' then + apply_sync_state(state, conn, svc, true, msg.payload) + elseif which == 'unsynced' then + apply_sync_state(state, conn, svc, false, msg.payload) + else + svc:obs_log('warn', { what = 'unknown_event_source', source = tostring(which) }) + end + end end -function time_service:start(root_ctx, bus_connection) - alarm.install_alarm_handler() - self.bus_connection = bus_connection - fiber.spawn(function() ntpd_monitor(root_ctx) end) +---@param conn Connection +---@param opts table? +---@return nil +function M.start(conn, opts) + opts = opts or {} + local svc = base.new(conn, { name = opts.name or 'time', env = opts.env }) + local heartbeat_s = (type(opts.heartbeat_s) == 'number') and opts.heartbeat_s or 30.0 + local wait_timeout_s = (type(opts.wait_timeout_s) == 'number') and opts.wait_timeout_s or nil + + svc:obs_state('boot', { at = svc:wall(), ts = svc:now(), state = 'entered' }) + svc:obs_log('debug', 'service start() entered') + svc:announce({}) + svc:starting() + svc:spawn_heartbeat(heartbeat_s, 'tick') + + fibers.current_scope():finally(function() + local _, primary = fibers.current_scope():status() + svc:lifecycle('stopped', { ready = false, reason = tostring(primary or 'scope_exit') }) + svc:obs_log('debug', 'service stopped') + end) + + svc:running() + + local cap_listener = cap_sdk.new_cap_listener(conn, 'time', '+') + svc:obs_log('debug', { what = 'waiting_for_time_capability', timeout = wait_timeout_s }) + + local cap_ref, cap_err = cap_listener:wait_for_cap({ timeout = wait_timeout_s }) + cap_listener:close() + + if not cap_ref then + svc:obs_log('warn', { what = 'capability_discovery_failed', err = tostring(cap_err) }) + return + end + + svc:obs_log('debug', { what = 'capability_selected', cap_id = tostring(cap_ref.id) }) + monitor_capability(conn, svc, cap_ref) end -return time_service +return M diff --git a/src/services/ui.lua b/src/services/ui.lua index eb4e100d..1f1e7b0e 100644 --- a/src/services/ui.lua +++ b/src/services/ui.lua @@ -1,501 +1,42 @@ -local fiber = require "fibers.fiber" -local channel = require 'fibers.channel' -local pollio = require "fibers.pollio" -local file = require "fibers.stream.file" -local log = require "services.log" -local http_server = require "http.server" -local http_headers = require "http.headers" -local http_util = require "http.util" -local websocket = require "http.websocket" -local cjson = require "cjson.safe" -local op = require "fibers.op" -local diagnostics = require "services.ui.diagnostics" -require "services.ui.fibers_cqueues" - -local ui_service = { - name = 'ui' -} -ui_service.__index = ui_service - --- TODO move into server config -local api_prefix = "api" -local ws_prefix = "ws" -local sse_prefix = "sse" -local sse_stats = "/" .. sse_prefix .. "/stats" -local sse_logs = "/" .. sse_prefix .. "/logs" -local stats_stream = "stats" -local log_stream = "logs" -local logs_subscription = "logs" -local config_subscription = "config" -local gsm_subscription = "gsm" -local system_subscription = "system" -local net_subscription = "net" -local mcu_subscription = "mcu" -local wifi_subscription = "wifi" --- Runtime state -local ws_clients = {} -- list of active WebSocket clients -local sse_clients = {} -- list of active SSE clients -local stats_messages_cache = {} -local log_messages_cache = {} -- keyed by level, each value is an array of messages -local config = { - mainflux = { - url = '', - key = '', - channels = { - data = '', - control = '', - } - }, - hawkbit = { - url = '', - key = '', - }, - unifi = { - ip = '', - }, - google = { - ip = '8.8.8.8', - } +-- services/ui.lua +-- +-- Public UI service assembly entry point. Semantic owners live under +-- services.ui.*; this file is deliberately thin. + +local M = { + service = require 'services.ui.service', + topics = require 'services.ui.topics', + errors = require 'services.ui.errors', + config = require 'services.ui.config', + sessions = require 'services.ui.sessions', + auth = require 'services.ui.auth', + read_model = require 'services.ui.read_model', + read_model_store = require 'services.ui.read_model_store', + read_model_watches = require 'services.ui.read_model_watches', + queries = require 'services.ui.queries', + local_model = require 'services.ui.local_model', + user_operation = require 'services.ui.user_operation', + http = { + listener = require 'services.ui.http.listener', + request = require 'services.ui.http.request', + routes = require 'services.ui.http.routes', + response = require 'services.ui.http.response', + static = require 'services.ui.http.static', + sse = require 'services.ui.http.sse', + }, + update = { + client = require 'services.ui.update.client', + upload = require 'services.ui.update.upload', + artifact_ingest = require 'services.ui.update.artifact_ingest', + }, } -local function get_static_mimetype(path) - local ext_to_type = { - [".js"] = "application/javascript", - [".css"] = "text/css", - [".png"] = "image/png", - [".jpg"] = "image/jpeg", - [".jpeg"] = "image/jpeg", - [".svg"] = "image/svg+xml", - [".ico"] = "image/x-icon", - [".json"] = "application/json", - [".tff"] = "font/ttf", - [".eot"] = "application/vnd.ms-fontobject", - [".woff"] = "font/woff", - [".webmanifest"] = "application/manifest+json", - [".webp"] = "image/webp", - } - - for ext, mime in pairs(ext_to_type) do - if path:sub(- #ext) == ext then - return mime - end - end - - return nil -end - -local function is_static_asset(path) - return get_static_mimetype(path) ~= nil +function M.start(conn, opts) + return M.service.start(conn, opts) end -local function get_spa_path(req_path) - local base_path = "./www/ui/dist" - - -- If it's a known static asset, serve the file directly. - if is_static_asset(req_path) then - return base_path .. req_path - else - -- Otherwise, it's probably an HTML page and we should serve the index.html of SPA. - return base_path .. "/index.html" - end -end - -local function handle_websocket(ctx, ws) - log.info("Websocket: New client connected") - table.insert(ws_clients, ws) - - while not ctx:err() do - local message, _, _ = ws:receive() - if message == nil then - break - end - -- handle incoming messages if necessary - end - - -- Cleanup - for i, client in ipairs(ws_clients) do - if client == ws then - ws:close() - table.remove(ws_clients, i) - log.info("Websocket: Client disconnected") - break - end - end -end - -local function handle_sse(ctx, stream, subscription) - stream.last_stats_sent = 0 - stream.sse_type = subscription - log.info("SSE: New client connected", subscription) - local sse_channel = channel.new(10) - sse_channel.closed = false - sse_channel.subscription = subscription - table.insert(sse_clients, sse_channel) - - -- Send latest cached messages to new SSE client - local send_cached_message = function(cached_msg) - local sse_msg = "data: " .. cjson.encode(cached_msg) .. "\n\n" - local ok, err = pcall(function() - stream:write_chunk(sse_msg, false) - end) - if not ok then - log.warn("SSE: Failed to send cached message:", err, " when writing to channel", subscription) - return false - end - return true - end - - if subscription == stats_stream then - for _, cached_msg in pairs(stats_messages_cache) do - local ok = send_cached_message(cached_msg) - if not ok then - break - end - end - elseif subscription == log_stream then - local all_logs = {} - for _, level_msgs in pairs(log_messages_cache) do - for _, cached_msg in ipairs(level_msgs) do - table.insert(all_logs, cached_msg) - end - end - table.sort(all_logs, function(a, b) return a.payload.timestamp < b.payload.timestamp end) - for _, cached_msg in ipairs(all_logs) do - local ok = send_cached_message(cached_msg) - if not ok then - break - end - end - end - - while not ctx:err() do - local msg = sse_channel:get() - if msg and msg.subscription == subscription then - local sse_msg = "data: " .. cjson.encode(msg) .. "\n\n" - local ok, err = pcall(function() - stream:write_chunk(sse_msg, false) - end) - -- Pre-emptively check if the stream is not stale - if stream.stats_sent == stream.last_stats_sent or not ok then - sse_channel.closed = true - if not ok then - log.warn("Subscribed ", subscription," SSE: Client disconnected or write error: ", err) - else - log.info("SSE: Stream is stale, closing connection for subscription: ", subscription) - end - stream:shutdown() - break - end - stream.last_stats_sent = stream.stats_sent - end - end -end - -local function onstream(self, stream) - local req_headers = assert(stream:get_headers()) - local req_method = req_headers:get(":method") or "GET" - local req_path = req_headers:get(":path") or "/" - local req_type = req_path:match("^/([^/]+)") - - -- Invalid method - if req_method ~= "GET" and req_method ~= "HEAD" and req_method ~= "PUT" then - local res_headers = http_headers.new() - res_headers:upsert(":status", "405") - assert(stream:write_headers(res_headers, true)) - return - end - - -- Dynamic request handling - if req_type == api_prefix then - if req_path == "/api/diagnostics" then - local res_headers = http_headers.new() - - if req_method == "GET" then - ---@diagnostic disable-next-line: need-check-nil - local diagnostics_reports = diagnostics.get_box_reports(config, stats_messages_cache) - ---@diagnostic disable-next-line: need-check-nil - local diagnostics_logs = diagnostics.get_box_logs() - local resp = { diagnostics_logs = diagnostics_logs, diagnostics = diagnostics_reports } - local msg = cjson.encode(resp) .. "\n\n" - res_headers:append(":status", "200") - assert(stream:write_headers(res_headers, false)) - assert(stream:write_chunk(msg, true)) - else - res_headers:append(":status", "404") - assert(stream:write_headers(res_headers, true)) - end - end - elseif req_type == ws_prefix then - -- Attempt WebSocket upgrade directly - local ws, _ = websocket.new_from_stream(stream, req_headers) - if ws then - log.info("Websocket upgrade request") - -- Successful WebSocket upgrade - ws:accept() - local ctx = { err = function() return false end } - handle_websocket(ctx, ws) - return - end - elseif req_type == sse_prefix then - if req_path == sse_stats or req_path == sse_logs then - local res_headers = http_headers.new() - res_headers:append(":status", "200") - res_headers:append("content-type", "text/event-stream") - assert(stream:write_headers(res_headers, req_method == "HEAD")) - local ctx = { err = function() return false end } - local selected_stream = req_path == sse_stats and stats_stream or log_stream - handle_sse(ctx, stream, selected_stream) - return - end - end - - -- Normal HTTP static file serving - if req_method ~= "GET" then - local res_headers = http_headers.new() - res_headers:append(":status", "405") - res_headers:append("content-type", "text/plain") - assert(stream:write_headers(res_headers, true)) - assert(stream:write_chunk("405 Method Not Allowed\n", true)) - return - end - - local path = get_spa_path(req_path) - local handle, err = file.open(path, "rb") - local res_headers = http_headers.new() - - if not handle then - log.warn("Failed to open file:", err) - res_headers:append(":status", "404") - assert(stream:write_headers(res_headers, true)) - return - end - - res_headers:append(":status", "200") - local mimetype = get_static_mimetype(path) or "text/html" - res_headers:append("content-type", mimetype) - assert(stream:write_headers(res_headers, req_method == "HEAD")) - - if req_method ~= "HEAD" then - while true do - local chunk = handle:read(4096) - if not chunk then break end - assert(stream:write_chunk(chunk, false)) - end - assert(stream:write_chunk("", true)) -- EOF - end - - handle:close() -end - -local function publish_to_ws_clients(payload) - local msg = cjson.encode(payload) - for _, ws in ipairs(ws_clients) do - local ok, err = pcall(function() - ws:send(msg) - end) - if not ok then - log.warn("Failed to send to websocket client:", err) - end - end -end - -local function publish_to_sse_clients(payload) - for i = #sse_clients, 1, -1 do -- Iterate in reverse - if sse_clients[i].closed then - table.remove(sse_clients, i) -- Safely remove the sse_channel - elseif sse_clients[i].subscription == payload.subscription then - sse_clients[i]:put_op(payload):perform_alt(function () end) - end - end -end - -local function bus_listener(ctx, connection) - local sub_gsm = connection:subscribe({ gsm_subscription, "#" }) - local sub_system = connection:subscribe({ system_subscription, "#" }) - local sub_net = connection:subscribe({ net_subscription, "#" }) - local sub_logs = connection:subscribe({ logs_subscription, "#" }) - local sub_mcu = connection:subscribe({ mcu_subscription, "#" }) - local sub_wifi = connection:subscribe({ wifi_subscription, "num_sta" }) - local sub_mainflux_config = connection:subscribe({ config_subscription, "mainflux" }) - local sub_metrics_config = connection:subscribe({ config_subscription, "metrics" }) - local sub_net_config = connection:subscribe({ config_subscription, "net" }) - - local publish_message = function(msg, subscription) - if subscription == stats_stream then - local topic_key = table.concat(msg.topic, ".") - - stats_messages_cache[topic_key] = { - topic = msg.topic, - payload = msg.payload, - subscription = subscription - } - elseif subscription == log_stream then - local level = msg.topic[2] or "unknown" - if not log_messages_cache[level] then - log_messages_cache[level] = {} - end - table.insert(log_messages_cache[level], { - topic = msg.topic, - payload = msg.payload, - subscription = subscription - }) - - -- Keep only the last 200 log messages per level - if #log_messages_cache[level] > 200 then - table.remove(log_messages_cache[level], 1) - end - end - - publish_to_ws_clients({ - topic = msg.topic, - payload = msg.payload - }) - publish_to_sse_clients({ - topic = msg.topic, - payload = msg.payload, - subscription = subscription - }) - end - - while not ctx:err() do - -- TODO unsure how to handle errors here - op.choice( - sub_gsm:next_msg_op():wrap(function(msg) - publish_message(msg, stats_stream) - end), - sub_system:next_msg_op():wrap(function(msg) - publish_message(msg, stats_stream) - end), - sub_net:next_msg_op():wrap(function(msg) - publish_message(msg, stats_stream) - end), - sub_mcu:next_msg_op():wrap(function(msg) - publish_message(msg, stats_stream) - end), - sub_wifi:next_msg_op():wrap(function(msg) - publish_message(msg, stats_stream) - end), - sub_logs:next_msg_op():wrap(function(msg) - publish_message(msg, log_stream) - end), - sub_mainflux_config:next_msg_op():wrap(function(msg) - local function parse_mainflux_config(msg) - local result = { - key = '', - channels = { data = '', control = '' }, - } - - local payload = msg.payload - if payload then - local key = payload.mainflux_key or payload.thing_key - if key then result.key = key end - - local channels = payload.mainflux_channels or payload.channels - for _, ch in ipairs(channels) do - if ch.metadata and ch.metadata.channel_type == "data" then - result.channels.data = ch.id - elseif ch.metadata and ch.metadata.channel_type == "events" then - result.channels.control = ch.id - end - end - end - - return result - end - - local function parse_hawkbit_config(msg) - local result = { - key = '', - url = '' - } - - if msg.payload and msg.payload.content then - local content = msg.payload.content - local content_json, _ = cjson.decode(content) - if not content_json then return result end - content = content_json - - if content.hawkbit.hawkbit_key and content.hawkbit.hawkbit_url then - result.key = content.hawkbit.hawkbit_key - result.url = content.hawkbit.hawkbit_url - end - end - - return result - end - - local mainflux = parse_mainflux_config(msg) - config.mainflux.channels = mainflux.channels - config.mainflux.key = mainflux.key - config.hawkbit = parse_hawkbit_config(msg) - end), - sub_metrics_config:next_msg_op():wrap(function(msg) - local cloud_url = msg.payload and msg.payload.cloud_url - config.mainflux.url = cloud_url - end), - sub_net_config:next_msg_op():wrap(function(msg) - local function parse_unifi_ip(msg) - local domains = msg.payload and msg.payload.dhcp and msg.payload.dhcp.domains - if not domains then return '' end - - for _, domain in ipairs(domains) do - if domain.name == "unifi" then return domain.ip end - end - - return '' - end - config.unifi.ip = parse_unifi_ip(msg) - end) - ):perform() - end - - sub_logs:unsubscribe() - sub_mainflux_config:unsubscribe() - sub_metrics_config:unsubscribe() - sub_net_config:unsubscribe() - sub_gsm:unsubscribe() - sub_system:unsubscribe() - sub_net:unsubscribe() - sub_mcu:unsubscribe() - sub_wifi:unsubscribe() -end - -function ui_service:start(ctx, connection) - log.trace("Starting UI service") - - pollio.install_poll_io_handler() - - local server = assert(http_server.listen { - host = "0.0.0.0", - port = 80, - onstream = onstream, - onerror = function(_, context, operation, err) - log.warn(operation, "on", context, "failed:", err) - end - }) - - function server:add_stream(stream) - fiber.spawn(function() - fiber.yield() - local ok, err = http_util.yieldable_pcall(self.onstream, self, stream) - if not ok then - self:onerror()(self, stream, "onstream", err) - end - -- Ensure the other streams are closed properly - if stream.state ~= "closed" then - log.info("Shutting down stream") - stream:shutdown() - end - end) - end - - fiber.spawn(function() - assert(server:loop()) - end) - - fiber.spawn(function() - bus_listener(ctx, connection) - end) +function M.run(scope, params) + return M.service.run(scope, params) end -return ui_service +return M diff --git a/src/services/ui/auth.lua b/src/services/ui/auth.lua new file mode 100644 index 00000000..a2f5ae29 --- /dev/null +++ b/src/services/ui/auth.lua @@ -0,0 +1,60 @@ +-- services/ui/auth.lua +-- +-- Authentication/verifier boundary. The default verifier is deliberately local +-- and immediate; callers that need blocking authentication should wrap it as a +-- scoped operation outside this module. + +local M = {} +local Verifier = {} +Verifier.__index = Verifier + +function M.new(opts) + opts = opts or {} + if opts.verify ~= nil and type(opts.verify) ~= 'function' then + error('auth.new: verify must be a function', 2) + end + return setmetatable({ + _verify = opts.verify, + _users = opts.users or {}, + }, Verifier) +end + +function Verifier:verify(credentials) + credentials = credentials or {} + if self._verify then + return self._verify(credentials) + end + + local username = credentials.username or credentials.user + local password = credentials.password + local rec = username and self._users[username] or nil + if rec == nil then + return nil, 'unauthenticated' + end + if type(rec) == 'table' then + if rec.password ~= nil and rec.password ~= password then + return nil, 'unauthenticated' + end + return rec.principal or username, nil + end + if rec ~= password then + return nil, 'unauthenticated' + end + return username, nil +end + +function M.verify(verifier, credentials) + if verifier == nil then + return nil, 'auth verifier unavailable' + end + if type(verifier) == 'function' then + return verifier(credentials) + end + if type(verifier) == 'table' and type(verifier.verify) == 'function' then + return verifier:verify(credentials) + end + return nil, 'invalid auth verifier' +end + +M.Verifier = Verifier +return M diff --git a/src/services/ui/config.lua b/src/services/ui/config.lua new file mode 100644 index 00000000..de1333a1 --- /dev/null +++ b/src/services/ui/config.lua @@ -0,0 +1,358 @@ +-- services/ui/config.lua +-- +-- Pure UI configuration normaliser. + +local M = {} + +M.SCHEMA = 'devicecode.config/ui/1' + +local DEFAULTS = { + enabled = true, + http = { + enabled = true, + cap_id = 'main', + host = '0.0.0.0', + port = 80, + }, + static = { + root = 'services/ui/www', + index = 'index.html', + chunk_size = 16384, + }, + sse = { + enabled = true, + queue_len = 256, + replay = false, + }, + sessions = { + prune_interval = 60, + }, + observability = { + status_interval_s = 30, + coalesce_status_s = 0.05, + }, +} + +local ROOT_KEYS = { + schema = true, + enabled = true, + http = true, + static = true, + sse = true, + updates = true, + sessions = true, + observability = true, +} + +local HTTP_KEYS = { + enabled = true, + cap_id = true, + host = true, + port = true, + path = true, + tls = true, + max_accept_queue = true, + max_active_requests = true, +} + +local STATIC_KEYS = { root = true, index = true, chunk_size = true } +local SSE_KEYS = { enabled = true, queue_len = true, max_replay = true, replay = true, pattern = true } +local UPDATE_KEYS = { upload = true, commit = true } +local UPDATE_UPLOAD_KEYS = { + enabled = true, + max_bytes = true, + require_auth = true, + component = true, + create_job = true, + start_job = true, +} +local UPDATE_COMMIT_KEYS = { require_auth = true } +local SESSION_KEYS = { prune_interval = true } +local OBSERVABILITY_KEYS = { status_interval_s = true, coalesce_status_s = true } + +local function fail(msg) return nil, msg end + +local function copy_plain(v) + if type(v) ~= 'table' then return v end + local out = {} + for k, subv in pairs(v) do out[k] = copy_plain(subv) end + return out +end + +local function allowed(t, keys, path) + for k in pairs(t or {}) do + if not keys[k] then return nil, path .. ' has unknown field: ' .. tostring(k) end + end + return true, nil +end + +local function table_or_empty(v, path) + if v == nil then return {}, nil end + if type(v) ~= 'table' then return nil, path .. ' must be a table' end + return v, nil +end + +local function table_required(v, path) + if type(v) ~= 'table' then return nil, path .. ' must be a table' end + return v, nil +end + +local function bool_or_nil(v, path) + if v == nil then return nil, nil end + if type(v) ~= 'boolean' then return nil, path .. ' must be boolean' end + return v, nil +end + +local function non_empty_string_or_nil(v, path) + if v == nil then return nil, nil end + if type(v) ~= 'string' or v == '' then return nil, path .. ' must be a non-empty string' end + return v, nil +end + +local function non_negative_int_or_nil(v, path) + if v == nil then return nil, nil end + if type(v) ~= 'number' or v < 0 or v % 1 ~= 0 then return nil, path .. ' must be a non-negative integer' end + return v, nil +end + +local function positive_number_or_false_or_nil(v, path) + if v == nil then return nil, nil end + if v == false then return false, nil end + if type(v) ~= 'number' or v <= 0 then return nil, path .. ' must be > 0 or false' end + return v, nil +end + +local function normalise_http(raw) + local err + raw, err = table_or_empty(raw, 'http') + if not raw then return nil, err end + local ok + ok, err = allowed(raw, HTTP_KEYS, 'http') + if not ok then return nil, err end + + local out = copy_plain(DEFAULTS.http) + local v + + v, err = bool_or_nil(raw.enabled, 'http.enabled') + if err then return nil, err end + if v ~= nil then out.enabled = v end + + v, err = non_empty_string_or_nil(raw.cap_id, 'http.cap_id') + if err then return nil, err end + if v ~= nil then out.cap_id = v end + + v, err = non_empty_string_or_nil(raw.host, 'http.host') + if err then return nil, err end + if v ~= nil then out.host = v end + + v, err = non_negative_int_or_nil(raw.port, 'http.port') + if err then return nil, err end + if v ~= nil then out.port = v end + + v, err = non_empty_string_or_nil(raw.path, 'http.path') + if err then return nil, err end + if v ~= nil then out.path = v end + + v, err = bool_or_nil(raw.tls, 'http.tls') + if err then return nil, err end + if v ~= nil then out.tls = v end + + v, err = non_negative_int_or_nil(raw.max_accept_queue, 'http.max_accept_queue') + if err then return nil, err end + if v ~= nil then out.max_accept_queue = v end + + v, err = non_negative_int_or_nil(raw.max_active_requests, 'http.max_active_requests') + if err then return nil, err end + if v ~= nil then out.max_active_requests = v end + + return out, nil +end + +local function normalise_static(raw) + local err + raw, err = table_or_empty(raw, 'static') + if not raw then return nil, err end + local ok + ok, err = allowed(raw, STATIC_KEYS, 'static') + if not ok then return nil, err end + + local out = copy_plain(DEFAULTS.static) + local v + v, err = non_empty_string_or_nil(raw.root, 'static.root') + if err then return nil, err end + if v ~= nil then out.root = v end + v, err = non_empty_string_or_nil(raw.index, 'static.index') + if err then return nil, err end + if v ~= nil then out.index = v end + v, err = non_negative_int_or_nil(raw.chunk_size, 'static.chunk_size') + if err then return nil, err end + if v ~= nil then out.chunk_size = v end + return out, nil +end + +local function normalise_sse(raw) + local err + raw, err = table_or_empty(raw, 'sse') + if not raw then return nil, err end + local ok + ok, err = allowed(raw, SSE_KEYS, 'sse') + if not ok then return nil, err end + + local out = copy_plain(DEFAULTS.sse) + local v + v, err = bool_or_nil(raw.enabled, 'sse.enabled') + if err then return nil, err end + if v ~= nil then out.enabled = v end + v, err = non_negative_int_or_nil(raw.queue_len, 'sse.queue_len') + if err then return nil, err end + if v ~= nil then out.queue_len = v end + v, err = non_negative_int_or_nil(raw.max_replay, 'sse.max_replay') + if err then return nil, err end + if v ~= nil then out.max_replay = v end + v, err = bool_or_nil(raw.replay, 'sse.replay') + if err then return nil, err end + if v ~= nil then out.replay = v end + if raw.pattern ~= nil then + if type(raw.pattern) ~= 'table' then return nil, 'sse.pattern must be a table' end + out.pattern = copy_plain(raw.pattern) + end + return out, nil +end + +local function normalise_update_upload(raw) + local err + raw, err = table_required(raw, 'updates.upload') + if not raw then return nil, err end + local ok + ok, err = allowed(raw, UPDATE_UPLOAD_KEYS, 'updates.upload') + if not ok then return nil, err end + local out = {} + local v + v, err = bool_or_nil(raw.enabled, 'updates.upload.enabled') + if err then return nil, err end + if v == nil then return nil, 'updates.upload.enabled is required' end + out.enabled = v + v, err = non_negative_int_or_nil(raw.max_bytes, 'updates.upload.max_bytes') + if err then return nil, err end + if v == nil then return nil, 'updates.upload.max_bytes is required' end + out.max_bytes = v + v, err = bool_or_nil(raw.require_auth, 'updates.upload.require_auth') + if err then return nil, err end + if v == nil then return nil, 'updates.upload.require_auth is required' end + out.require_auth = v + v, err = non_empty_string_or_nil(raw.component, 'updates.upload.component') + if err then return nil, err end + if v == nil then return nil, 'updates.upload.component is required' end + out.component = v + v, err = bool_or_nil(raw.create_job, 'updates.upload.create_job') + if err then return nil, err end + if v == nil then return nil, 'updates.upload.create_job is required' end + out.create_job = v + v, err = bool_or_nil(raw.start_job, 'updates.upload.start_job') + if err then return nil, err end + if v == nil then return nil, 'updates.upload.start_job is required' end + out.start_job = v + if out.start_job == true and out.create_job ~= true then + return nil, 'updates.upload.start_job requires updates.upload.create_job' + end + return out, nil +end + +local function normalise_update_commit(raw) + local err + raw, err = table_required(raw, 'updates.commit') + if not raw then return nil, err end + local ok + ok, err = allowed(raw, UPDATE_COMMIT_KEYS, 'updates.commit') + if not ok then return nil, err end + local out = {} + local v + v, err = bool_or_nil(raw.require_auth, 'updates.commit.require_auth') + if err then return nil, err end + if v == nil then return nil, 'updates.commit.require_auth is required' end + out.require_auth = v + return out, nil +end + +local function normalise_updates(raw) + local err + raw, err = table_required(raw, 'updates') + if not raw then return nil, err end + local ok + ok, err = allowed(raw, UPDATE_KEYS, 'updates') + if not ok then return nil, err end + local upload; upload, err = normalise_update_upload(raw.upload); if not upload then return nil, err end + local commit; commit, err = normalise_update_commit(raw.commit); if not commit then return nil, err end + return { upload = upload, commit = commit }, nil +end + +local function normalise_sessions(raw) + local err + raw, err = table_or_empty(raw, 'sessions') + if not raw then return nil, err end + local ok + ok, err = allowed(raw, SESSION_KEYS, 'sessions') + if not ok then return nil, err end + local out = copy_plain(DEFAULTS.sessions) + local v + v, err = positive_number_or_false_or_nil(raw.prune_interval, 'sessions.prune_interval') + if err then return nil, err end + if v ~= nil then out.prune_interval = v end + return out, nil +end + + +local function normalise_observability(raw) + local err + raw, err = table_or_empty(raw, 'observability') + if not raw then return nil, err end + local ok + ok, err = allowed(raw, OBSERVABILITY_KEYS, 'observability') + if not ok then return nil, err end + local out = copy_plain(DEFAULTS.observability) + local v + v, err = positive_number_or_false_or_nil(raw.status_interval_s, 'observability.status_interval_s') + if err then return nil, err end + if v ~= nil then out.status_interval_s = v end + v, err = positive_number_or_false_or_nil(raw.coalesce_status_s, 'observability.coalesce_status_s') + if err then return nil, err end + if v ~= nil then out.coalesce_status_s = v end + return out, nil +end + +function M.normalise(raw) + if raw == nil then raw = {} end + if type(raw) ~= 'table' then return fail('ui config must be a table') end + local ok, err = allowed(raw, ROOT_KEYS, 'ui config') + if not ok then return nil, err end + if raw.schema ~= nil and raw.schema ~= M.SCHEMA then + return nil, 'ui config schema must be ' .. M.SCHEMA + end + + local enabled + enabled, err = bool_or_nil(raw.enabled, 'enabled') + if err then return nil, err end + + local http; http, err = normalise_http(raw.http); if not http then return nil, err end + local static; static, err = normalise_static(raw.static); if not static then return nil, err end + local sse; sse, err = normalise_sse(raw.sse); if not sse then return nil, err end + local updates; updates, err = normalise_updates(raw.updates); if not updates then return nil, err end + local sessions; sessions, err = normalise_sessions(raw.sessions); if not sessions then return nil, err end + local observability; observability, err = normalise_observability(raw.observability) + if not observability then return nil, err end + + return { + schema = M.SCHEMA, + enabled = enabled ~= false, + http = http, + static = static, + sse = sse, + updates = updates, + sessions = sessions, + observability = observability, + }, nil +end + + +M.DEFAULTS = DEFAULTS +return M diff --git a/src/services/ui/diagnostics.lua b/src/services/ui/diagnostics.lua deleted file mode 100644 index 64e62934..00000000 --- a/src/services/ui/diagnostics.lua +++ /dev/null @@ -1,520 +0,0 @@ -local expected = require 'services.ui.diagnostics_expected' -local helpers = require 'services.ui.diagnostics_helpers' -local request = require 'http.request' -local exec = require 'fibers.exec' -local log = require 'services.log' -local cjson = require "cjson.safe" - -local tests = {} - -local LOG_LEVELS = { - NOTICE = "notice", - WARN = "warn", - ERROR = "error" -} - -local dmesg_level_set = false - --- Configuring Dmesg log level to include Notice -local function ensure_dmesg_console_level() - if dmesg_level_set then return end - local _, err = exec.command("sh", "-c", "dmesg -n 5"):output() - if err then - log.error("UI - Error setting dmesg log level " .. tostring(err)) - else - dmesg_level_set = true - end -end - ----Creates a new report table ----@param exp number Number of expected packages/services ----@param installed number Number of installed/running packages/services ----@param missing string[] Missing packages/services ----@return table report -local function new_report(exp, installed, missing) - return { - expected = exp, - installed = installed, - missing = missing, - } -end - ----Check if the expected packages are installed ----@param expected_packages_installed string[] ----@param installed_packages string[] ----@return table report -local function check_expected_packages_installed(expected_packages_installed, installed_packages) - local total_installed = 0 - local missing = {} - - for _, package in ipairs(expected_packages_installed) do - local installed = installed_packages[package] or false - - if not installed then - table.insert(missing, package) - else - total_installed = total_installed + 1 - end - end - - return new_report(#expected_packages_installed, total_installed, missing) -end - ----Reports the runnning packages ----@param expected_packages_running string[] ----@return table -local function check_expected_packages_running(expected_packages_running) - local total_running = 0 - local missing = {} - for _, package in ipairs(expected_packages_running) do - if helpers.is_process_running(package) then - total_running = total_running + 1 - else - table.insert(missing, package) - end - end - - return new_report(#expected_packages_running, total_running, missing) -end - ----Reports the running services ----@param box_type string getbox|bigbox ----@return table|nil report ----@return string|nil error -local function check_expected_services_running(box_type) - -- Get expected installed packages - local expected_services_running, err = expected.get_expected_services_running(box_type) - if err ~= nil then - return nil, err - end - - local total_running = 0 - local missing = {} - for _, service in ipairs(expected_services_running) do - if helpers.is_service_running(service) then - total_running = total_running + 1 - else - table.insert(missing, service) - end - end - - return new_report(#expected_services_running, total_running, missing) -end - ----Fetches logs from dmesg with the specified log level ----@param log_level string ----@return string[]|nil logs ----@return string|nil error -local function get_dmesg_logs(log_level) - ensure_dmesg_console_level() - - local output, err = exec.command("sh", "-c", 'dmesg | grep -E -i "' .. log_level .. '"'):output() - if err then - return nil, err - end - - local logs = {} - for line in output:gmatch("[^\r\n]+") do - table.insert(logs, '[' .. string.upper(log_level) .. '][dmesg] ' .. line) - end - return logs, nil -end - ----Fetches logs from logread with the specified log level ----@param log_level string ----@return string[]|nil logs ----@return string|nil error -local function get_logread_logs(log_level) - local output, err = exec.command("sh", "-c", 'logread | grep -E -i "' .. log_level .. '"'):output() - if err then - return nil, err - end - - local logs = {} - for line in output:gmatch("[^\r\n]+") do - table.insert(logs, '[' .. string.upper(log_level) .. '][logread] ' .. line) - end - return logs, nil -end - ----Check if the correct number of modems are installed ----@param expected_modems number Number of expected modems ----@return table report ----@return string|nil error -local function check_expected_modems_installed(expected_modems) - local output, err = exec.command("mmcli", "-L"):output() - if err then - return {}, err - end - - local installed_modems = 0 - for line in output:gmatch("[^\r\n]+") do - if line:match("Modem") then - installed_modems = installed_modems + 1 - end - end - - return new_report(expected_modems, installed_modems, {}), nil -end - ----Reports the installed bootstrap files ----@param expected_bootstrap_installed string[] Paths to expected bootstrap files ----@return table report ----@return string|nil error -local function check_expected_bootstrap_installed(expected_bootstrap_installed) - local bootstrap_installed_total = 0 - local missing = {} - - for _, path in ipairs(expected_bootstrap_installed) do - local exists = helpers.file_exists(path) - - if exists then - bootstrap_installed_total = bootstrap_installed_total + 1 - else - table.insert(missing, path) - end - end - - return new_report(#expected_bootstrap_installed, bootstrap_installed_total, missing) -end - ----Reports the installed packages ----@param box_type string getbox|bigbox ----@return table|nil packages_installed ----@return string|nil error -local function get_packages_installed(box_type) - -- Get expected installed packages - local expected_packages_installed, exp_pkg_err = expected.get_expected_packages_installed(box_type) - if exp_pkg_err ~= nil then - return nil, exp_pkg_err - end - - -- Get installed packages - local installed_packages, inst_pkg_err = helpers.get_installed_packages() - if inst_pkg_err ~= nil then - return nil, inst_pkg_err - end - - -- Check if packages are installed - local packages_installed = check_expected_packages_installed(expected_packages_installed, installed_packages) - return packages_installed, nil -end - ----Reports the installed modems ----@param box_type string getbox|bigbox ----@return table|nil modems_installed ----@return string|nil error -local function get_modems_installed(box_type) - -- Get expected modems - local expected_modems, exp_count_err = expected.get_expected_modem_count(box_type) - if exp_count_err ~= nil then - return nil, exp_count_err - end - - -- Check if modems are installed - local modems_installed, exp_inst_err = check_expected_modems_installed(expected_modems) - if exp_inst_err ~= nil then - return nil, exp_inst_err - end - - return modems_installed, nil -end - ----Simple ICMP reachability check for google (BusyBox ping) ----@param config table The google configuration ----@return boolean ----@return string|nil error -function tests.test_google_connectivity(config) - local out, err = exec.command("sh", "-c", "ping -c 1 -W 2 " .. config.ip):output() - if err then return false, "ping error: " .. tostring(err) end - -- BusyBox output usually includes "1 packets transmitted, 1 packets received" or "bytes from" - if out:lower():match("1 packets received") or out:lower():match("bytes from") then - return true, nil - end - return false, "no reply from " .. config.ip -end - ----Test the connectivity to hawkbit ----@param config table The hawkbit configuration ----@return boolean ----@return string|nil error -function tests.test_hawkbit_connectivity(config) - if not config.url or not config.key then - local err = "Hawkbit configuration is missing" - log.error(err) - return false, err - end - - -- Make a request to hawkbit to check connectivity - -- TODO shouldn't read from here - local MAC_PATH = "/sys/class/net/eth0/address" - local mac_address, err = helpers.get_parsed_mac(MAC_PATH) - - if err ~= nil then - log.error('Error getting mac address: ' .. err) - return false, err - end - - local full_path = config.url .. "/default/controller/v1/" .. mac_address - local req = request.new_from_uri(full_path) - - req.headers:upsert(":method", "GET") - req.headers:upsert("authorization", "TargetToken " .. config.key) - req.headers:upsert("content-type", "application/json") - local headers, _ = req:go(10) - - if not headers then - log.error("Request Timeout: No response from the Hawkbit server. Url: " .. full_path) - return false, err - elseif headers:get(":status") ~= "200" then - log.error('Error connecting to hawkbit: ' .. headers:get(":status")) - return false, err - end - - return true, nil -end - ----Test the connectivity to Unifi ----@param config table The unifi configuration ----@return boolean ----@return string|nil error -function tests.test_unifi_connectivity(config) - -- Check if the Unifi IP is set - if not config.ip then - local err = "Unifi IP is not set in the default config" - log.error(err) - return false, err - end - - return true, nil -end - ----Test the connectivity to Mainflux ----@param config table The mainflux configuration ----@return boolean ----@return string|nil error -function tests.test_mainflux_connectivity(config) - if not config.url or not config.key or not config.channels.data or not config.channels.control then - local err = "Mainflux configuration is missing" - log.error(err) - return false, err - end - - local full_path = config.url .. "/http/channels/" .. config.channels.data .. "/messages" - -- Creating a SENML formatted message - local req = request.new_from_uri(full_path) - req.headers:upsert(":method", "POST") - req.headers:upsert("authorization", "Thing " .. config.key) - req.headers:upsert("content-type", "application/senml+json") - req.headers:delete("expect") - req:set_body(cjson.encode({ { - vs = "Testing Mainflux Connectivity", ts = os.time(), n = "Diagnostics" - }})) - - local res_headers, _ = req:go(10) - - if not res_headers then - local err = "Request Timeout: No response from the Mainflux server. Url: " .. full_path - log.error(err) - return false, err - elseif res_headers:get(":status") ~= "202" then - local err = string.format("URL: %s | Response: %s", full_path, res_headers:get(":status")) - log.error(err) - return false, err - end - - return true, nil -end - ----@param box_type string getbox|bigbox ----@param config table The configuration ----@return table|nil report ----@return string|nil error -local function check_expected_cloud_services_reachable(box_type, config) - local expected_tests, exp_tests_err = expected.get_expected_connectivity_tests(box_type) - if exp_tests_err ~= nil then - return nil, exp_tests_err - end - - local expected_cloud_services = 0 - local installed = 0 - local missing = {} - - for name, test_fn_name in pairs(expected_tests) do - expected_cloud_services = expected_cloud_services + 1 - local test_fn = tests[test_fn_name] - if type(test_fn) ~= "function" then - table.insert(missing, name .. " (missing test function)") - else - local ok, _ = test_fn(config[name]) - if ok then installed = installed + 1 else table.insert(missing, name) end - end - end - - return new_report(expected_cloud_services, installed, missing) -end - ----Check if the expected fans are present and working ----@param box_type string Box model type ----@return table|nil report (nil if no fans expected) -local function check_expected_fans(box_type) - local expected_fans, err = expected.get_expected_fan_count(box_type) - if err ~= nil or expected_fans == 0 then - return nil - end - - local _, fan_err = helpers.get_fan_status() - if fan_err then - return new_report(expected_fans, 0, {"fan"}) - end - - return new_report(expected_fans, 1, {}) -end - ----Check which modems are active (SIM present and state machine running) ----@param box_type string Box model type ----@param stats_cache table|nil Flat stats messages cache ----@return table|nil report ----@return string|nil error -local function get_modems_sim_active(box_type, stats_cache) - local expected_modems, err = expected.get_expected_modem_names(box_type) - if err ~= nil then - return nil, err - end - - local installed = 0 - local missing = {} - - if not stats_cache then - return new_report(#expected_modems, 0, expected_modems), nil - end - - for _, modem_name in ipairs(expected_modems) do - local sim_key = "gsm.modem." .. modem_name .. ".sim" - local sim_entry = stats_cache[sim_key] - local sim_present = sim_entry and sim_entry.payload == "present" - - local state_key = "gsm.modem." .. modem_name .. ".state" - local state_entry = stats_cache[state_key] - local has_active_state = state_entry - and state_entry.payload - and state_entry.payload.curr_state - and state_entry.payload.curr_state ~= "" - - local is_active = sim_present and has_active_state - - if is_active then - installed = installed + 1 - else - table.insert(missing, modem_name) - end - end - - return new_report(#expected_modems, installed, missing), nil -end - ----Fetches diagnostics stats from the box ----@param config table The configuration ----@param stats_cache table|nil Flat stats messages cache ----@return table diagnostics -local function get_box_reports(config, stats_cache) - local diagnostics = { - packages_installed = {}, - modems_installed = {}, - modems_sim_active = {}, - services_running = {}, - packages_running = {}, - bootstrap_installed = {}, - cloud_services_reachable = {}, - } - local hardware_info, hardware_info_err = helpers.get_hardware_info("/etc/hwrevision") - - if hardware_info_err == nil then - -- Get packages installed currently - local packages_installed, pkg_inst_err = get_packages_installed(hardware_info.model) - if pkg_inst_err == nil then - diagnostics.packages_installed = packages_installed - else - log.error("UI - error getting packages installed", pkg_inst_err) - end - - -- Check modems - local modems_installed, mod_inst_err = get_modems_installed(hardware_info.model) - if mod_inst_err == nil then - diagnostics.modems_installed = modems_installed - else - log.error("UI - error getting modems installed", mod_inst_err) - end - - -- Check services running - local services_running, exp_svc_err = check_expected_services_running(hardware_info.model) - if exp_svc_err == nil then - diagnostics.services_running = services_running - else - log.error("UI - error getting services running", exp_svc_err) - end - - -- Check cloud services reachable - local cloud_services, cloud_services_err = check_expected_cloud_services_reachable(hardware_info.model, config) - if cloud_services_err ~= nil then - log.error("UI - error checking cloud services reachable", cloud_services_err) - else - diagnostics.cloud_services_reachable = cloud_services - end - - -- Check modem SIM active (present + state machine running) - local modems_sim_active, sim_check_err = get_modems_sim_active(hardware_info.model, stats_cache) - if sim_check_err ~= nil then - log.error("UI - error checking modem SIM active", sim_check_err) - else - diagnostics.modems_sim_active = modems_sim_active - end - - -- Check fan status (only included for devices that have a fan) - local fan_report = check_expected_fans(hardware_info.model) - if fan_report then - diagnostics.fan_status = fan_report - end - else - log.error("UI - error getting hardware info", hardware_info_err) - end - - -- Check if packages are running - -- TODO will need to separate per device - local packages_running = check_expected_packages_running(expected.packages_running) - diagnostics.packages_running = packages_running - - -- Check bootstrapped - local bootstrap_installed = check_expected_bootstrap_installed(expected.bootstrap_installed) - diagnostics.bootstrap_installed = bootstrap_installed - - return diagnostics -end - ----Fetches logs from dmesg and logread ----@return string[] logs -local function get_box_logs() - local logs = {} - - local function log_appender(new_logs) - for _, new_log in ipairs(new_logs) do - table.insert(logs, new_log) - end - end - for _, log_level in pairs(LOG_LEVELS) do - local dmesg_logs, dmesg_read_err = get_dmesg_logs(log_level) - if dmesg_read_err == nil then - log_appender(dmesg_logs) - end - local logread_logs, logread_err = get_logread_logs(log_level) - if logread_err == nil then - log_appender(logread_logs) - end - end - return logs -end - -return { - get_box_reports = get_box_reports, - get_box_logs = get_box_logs, -} diff --git a/src/services/ui/diagnostics_expected.lua b/src/services/ui/diagnostics_expected.lua deleted file mode 100644 index 42e6c8b1..00000000 --- a/src/services/ui/diagnostics_expected.lua +++ /dev/null @@ -1,352 +0,0 @@ -local GETBOX="getbox" -local BIGBOX_SS="bigbox-ss" -local BIGBOX_V1_CM="bigbox-v1-cm" - -local GETBOX_PACKAGES_INSTALLED = { - "mwan3", - "rpcd", - "sqm-scripts", - "lua-lumen", - "modemmanager", - "qmi-utils", - "usb-modeswitch", - "kmod-mii", - "kmod-usb-wdm", - "kmod-usb-serial", - "kmod-usb-net", - "kmod-usb-serial-wwan", - "kmod-usb-serial-option", - "kmod-usb-net-qmi-wwan", - "swupdate", - "lua", - "luasocket", - "luaposix", - "dkjson", - "curl", - "lua-bit32", - "lua-popen3", - "luci-lib-nixio", - "libuci-lua", - "libubus-lua", - "libiwinfo-lua", - "lua-http", - "lua-cqueues", - "lmdb", - "lua-compat53", - "jq", - "block-mount" -} - -local BIGBOX_SS_PACKAGES_INSTALLED = { - "mwan3", - "rpcd", - "sqm-scripts", - "block-mount", - "btrfs-progs", - "kmod-fs-ext4", - "kmod-fs-btrfs", - "fdisk", - "modemmanager", - "qmi-utils", - "mbim-utils", - "libqmi", - "usb-modeswitch", - "kmod-mii", - "kmod-usb-wdm", - "kmod-usb-serial", - "kmod-usb-net", - "kmod-usb-serial-wwan", - "kmod-usb-serial-option", - "kmod-usb-net-qmi-wwan", - "kmod-usb-net-cdc-mbim", - "atinout", - "swupdate", - "lua", - "luasocket", - "luaposix", - "dkjson", - "curl", - "lua-bit32", - "lua-popen3", - "luci-lib-nixio", - "luajit", - "lua-cjson", - "libuci-lua", - "libubus-lua", - "libiwinfo-lua", - "lua-http", - "lua-cqueues", - "lmdb", - "lmdb-test", - "lua-compat53", - "usbutils", - "tree", - "uhubctl", - "jq", -} - -local BIGBOX_V1_CM_PACKAGES_INSTALLED = { - -- Core - "mwan3", - "rpcd", - "sqm-scripts", - - -- Fan / hardware monitoring - "kmod-i2c-core", - - -- Persistent storage - "block-mount", - "kmod-fs-ext4", - "btrfs-progs", - "kmod-fs-btrfs", - "fdisk", - - -- Modem / connectivity - "modemmanager", - "qmi-utils", - "mbim-utils", - "libqmi", - "usb-modeswitch", - "kmod-mii", - "kmod-usb-wdm", - "kmod-usb-serial", - "kmod-usb-net", - "kmod-usb-serial-wwan", - "kmod-usb-serial-option", - "kmod-usb-net-qmi-wwan", - "kmod-usb-net-cdc-mbim", - "atinout", - - -- Wi-Fi support - "kmod-mt7915-firmware", - "kmod-mt7915e", - "pciutils", - "dawn", - "wpad-mbedtls", - - -- Updater - "swupdate", - - -- Lua runtime - "lua", - "luasocket", - "luaposix", - "dkjson", - "curl", - "lua-bit32", - "lua-popen3", - "luci-lib-nixio", - "luajit", - "lua-cjson", - - -- Lua bindings - "libuci-lua", - "libubus-lua", - "libiwinfo-lua", - - -- Experimental Lua / internal libs - "lua-http", - "lua-cqueues", - "lmdb", - "lmdb-test", - "lua-compat53", - - -- Utilities - "usbutils", - "tree", - "uhubctl", - "jq", - - -- Bootloader - "uboot-envtools", -} - -local GETBOX_MODEMS = 1 -local BIGBOX_SS_MODEMS = 2 -local BIGBOX_V1_CM_MODEMS = 2 - -local GETBOX_EXPECTED_FANS = 0 -local BIGBOX_SS_EXPECTED_FANS = 1 -local BIGBOX_V1_CM_EXPECTED_FANS = 0 - -local GETBOX_MODEM_NAMES = {"primary"} -local BIGBOX_SS_MODEM_NAMES = {"primary", "secondary"} -local BIGBOX_V1_CM_MODEM_NAMES = {"primary", "secondary"} - -local BOOTSTRAP_INSTALLED = { - "/data/configs/hawkbit.cfg", - "/data/configs/mainflux.cfg", - "/data/serial", -} - -local PACKAGES_RUNNING = { - "ModemManager", - "main.lua", - "swupdate", - "mwan3" -} - -local GETBOX_SERVICES_RUNNING = { - "dnsmasq", - "dropbear", - "log", - "modemmanager", - "mwan3", - "network", - "odhcpd", - "rpcd", - "sysntpd", - "urngd", - "wpad" -} - -local BIGBOX_SS_SERVICES_RUNNING = { - "dnsmasq", - "dropbear", - "log", - "modemmanager", - "mwan3", - "network", - "odhcpd", - "rpcd", - "sysntpd", - "wpad" -} - -local BIGBOX_V1_CM_SERVICES_RUNNING = { - "dawn", - "dbus", - "dnsmasq", - "dropbear", - "log", - "modemmanager", - "mwan3", - "network", - "odhcpd", - "rpcd", - "sysntpd", - "umdns", - "wpad" -} - -local BIGBOX_SS_CONNECTIVITY_TESTS = { - google = "test_google_connectivity", - hawkbit = "test_hawkbit_connectivity", - mainflux = "test_mainflux_connectivity", - unifi = "test_unifi_connectivity", -} - -local BIGBOX_V1_CM_CONNECTIVITY_TESTS = { - google = "test_google_connectivity", - hawkbit = "test_hawkbit_connectivity", - mainflux = "test_mainflux_connectivity", -} - -local GETBOX_CONNECTIVITY_TESTS = { - google = "test_google_connectivity", - hawkbit = "test_hawkbit_connectivity", - mainflux = "test_mainflux_connectivity", -} - -local function get_expected_connectivity_tests(box_type) - if box_type == GETBOX then - return GETBOX_CONNECTIVITY_TESTS, nil - elseif box_type == BIGBOX_SS then - return BIGBOX_SS_CONNECTIVITY_TESTS, nil - elseif box_type == BIGBOX_V1_CM then - return BIGBOX_V1_CM_CONNECTIVITY_TESTS, nil - end - - return {}, "Unknown box type: " .. box_type -end - ----Get the expected packages installed for the box type ----@param box_type string getbox|bigbox ----@return string[] expected_packages_installed ----@return string|nil error -local function get_expected_packages_installed(box_type) - if box_type == GETBOX then - return GETBOX_PACKAGES_INSTALLED, nil - elseif box_type == BIGBOX_SS then - return BIGBOX_SS_PACKAGES_INSTALLED, nil - elseif box_type == BIGBOX_V1_CM then - return BIGBOX_V1_CM_PACKAGES_INSTALLED, nil - else - return {}, "Unknown box type: " .. box_type - end -end - ----Get the expected services running for the box type ----@param box_type string getbox|bigbox ----@return string[] expected_services_running ----@return string|nil error -local function get_expected_services_running(box_type) - if box_type == GETBOX then - return GETBOX_SERVICES_RUNNING, nil - elseif box_type == BIGBOX_SS then - return BIGBOX_SS_SERVICES_RUNNING, nil - elseif box_type == BIGBOX_V1_CM then - return BIGBOX_V1_CM_SERVICES_RUNNING, nil - else - return {}, "Unknown box type: " .. box_type - end -end - ----Get the number of expected modems installed for the box model ----@param box_type string getbox|bigbox ----@return number expected_modems ----@return string|nil error -local function get_expected_modem_count(box_type) - if box_type == GETBOX then - return GETBOX_MODEMS, nil - elseif box_type == BIGBOX_SS then - return BIGBOX_SS_MODEMS, nil - elseif box_type == BIGBOX_V1_CM then - return BIGBOX_V1_CM_MODEMS, nil - else - return 0, "Unknown box_type: " .. box_type - end -end - ----Get the expected modem names for the box model ----@param box_type string getbox|bigbox-ss|bigbox-v1-cm ----@return string[] expected_modem_names ----@return string|nil error -local function get_expected_modem_names(box_type) - if box_type == GETBOX then - return GETBOX_MODEM_NAMES, nil - elseif box_type == BIGBOX_SS then - return BIGBOX_SS_MODEM_NAMES, nil - elseif box_type == BIGBOX_V1_CM then - return BIGBOX_V1_CM_MODEM_NAMES, nil - else - return {}, "Unknown box_type: " .. box_type - end -end - ----Get the number of expected fans for the box model ----@param box_type string getbox|bigbox-ss|bigbox-v1-cm ----@return number expected_fans ----@return string|nil error -local function get_expected_fan_count(box_type) - if box_type == GETBOX then - return GETBOX_EXPECTED_FANS, nil - elseif box_type == BIGBOX_SS then - return BIGBOX_SS_EXPECTED_FANS, nil - elseif box_type == BIGBOX_V1_CM then - return BIGBOX_V1_CM_EXPECTED_FANS, nil - else - return 0, "Unknown box_type: " .. box_type - end -end - -return { - packages_running = PACKAGES_RUNNING, - bootstrap_installed = BOOTSTRAP_INSTALLED, - get_expected_packages_installed = get_expected_packages_installed, - get_expected_services_running = get_expected_services_running, - get_expected_modem_count = get_expected_modem_count, - get_expected_modem_names = get_expected_modem_names, - get_expected_connectivity_tests = get_expected_connectivity_tests, - get_expected_fan_count = get_expected_fan_count, -} diff --git a/src/services/ui/diagnostics_helpers.lua b/src/services/ui/diagnostics_helpers.lua deleted file mode 100644 index 1b6f4e4b..00000000 --- a/src/services/ui/diagnostics_helpers.lua +++ /dev/null @@ -1,166 +0,0 @@ -local exec = require 'fibers.exec' - ----Check if a process is running ----@param process string ----@return boolean is_running -local function is_process_running(process) - local output, err = exec.command("sh", "-c", "pgrep -af " .. process .. " | grep -v pgrep"):output() - if err then - return false - end - return #output > 0 -end - ----Check if a service is running ----@param service string ----@return boolean is_running -local function is_service_running(service) - local output, err = exec.command("/etc/init.d/" .. service, "status"):output() - if err or #output == 0 then - return false - end - return output:match("running") ~= nil -end - ----Check if a file exists ----@param path string ----@return boolean -local function file_exists(path) - local file = io.open(path, "r") - if file then - file:close() - return true - end - return false -end - ----Get the packages installed on the device ----@return string[] installed_packages ----@return string|nil error -local function get_installed_packages() - local output, err = exec.command("opkg", "list-installed"):output() - if err then - return output, err - end - - local installed_packages = {} - for line in output:gmatch("[^\r\n]+") do - local package = line:match("^(%S+)") - if package then - installed_packages[package] = true - end - end - return installed_packages, nil -end - ----Read a file and return its content ----@param path string ----@param slurp boolean ----@return string|nil content ----@return string|nil error -local function read_file(path, slurp) - local file, error, content - if path == nil then - return nil, "Path is nil" - end - if slurp then - file, error = io.open(path) - else - file, error = io.open(path, "rb") - end - if file then - content = file:read("*all") - file:close() - end - return content, error -end - ----Get the hardware information of the device ----@param hardware_info_path string Path to the hardware info file ----@return table hardware_info {model: string, revision: string} ----@return string|nil error -local function get_hardware_info(hardware_info_path) - -- Can expand this function to also collect information about individual components - local hardware_info = { - model = nil, - revision = nil - } - local info, error = read_file(hardware_info_path, true) - if info then - hardware_info.model, hardware_info.revision = info:match("^(%S+)%s+(%S+)") - return hardware_info, nil - end - return hardware_info, error -end - ----Get the MAC address of the device ----@param mac_address_path string Path to the mac address file ----@return string|nil mac_address ----@return string|nil err -local function get_parsed_mac(mac_address_path) - local mac_address, err = read_file(mac_address_path, true) - if mac_address then - return mac_address:gsub(":", ""):match("^%s*(.-)%s*$"), nil - end - return nil, err -end - ----Find the hwmon directory for a given device name ----@param device_name string The name to match in /sys/class/hwmon/*/name ----@return string|nil hwmon_path The hwmon directory path ----@return string|nil error -local function find_hwmon_device(device_name) - local output, err = exec.command("sh", "-c", - "grep -rl '^" .. device_name .. "$' /sys/class/hwmon/*/name 2>/dev/null"):output() - if err or #output == 0 then - return nil, "hwmon device '" .. device_name .. "' not found" - end - - -- output is e.g. /sys/class/hwmon/hwmon2/name — extract directory - local name_path = output:match("^(%S+)") - if not name_path then - return nil, "hwmon device '" .. device_name .. "' not found" - end - local hwmon_path = name_path:match("(.+)/name$") - if not hwmon_path then - return nil, "could not parse hwmon path from: " .. name_path - end - return hwmon_path, nil -end - ----Check if the fan controller is present and readable ----Tries both "pwmfan" and "rpipoefan" as the device name varies by OpenWrt version ----@return table|nil fan_status {hwmon_path, pwm} ----@return string|nil error -local function get_fan_status() - local hwmon_path, find_err = find_hwmon_device("pwmfan") - if find_err then - hwmon_path, find_err = find_hwmon_device("rpipoefan") - end - if find_err then - return nil, find_err - end - - local pwm_path = hwmon_path .. "/pwm1" - local pwm_str = read_file(pwm_path, true) - local pwm = 0 - if pwm_str then - pwm = tonumber(pwm_str:match("^%s*(.-)%s*$")) or 0 - end - - return { - hwmon_path = hwmon_path, - pwm = pwm, - }, nil -end - -return { - is_process_running = is_process_running, - is_service_running = is_service_running, - file_exists = file_exists, - get_installed_packages = get_installed_packages, - get_hardware_info = get_hardware_info, - get_parsed_mac = get_parsed_mac, - find_hwmon_device = find_hwmon_device, - get_fan_status = get_fan_status, -} diff --git a/src/services/ui/errors.lua b/src/services/ui/errors.lua new file mode 100644 index 00000000..2a2f8233 --- /dev/null +++ b/src/services/ui/errors.lua @@ -0,0 +1,68 @@ +-- services/ui/errors.lua +-- +-- Boundary error vocabulary and simple HTTP mapping helpers. + +local M = {} + +M.codes = { + bad_request = { http = 400, message = 'bad_request' }, + invalid_json = { http = 400, message = 'invalid_json' }, + invalid_body = { http = 400, message = 'invalid_body' }, + request_body_too_large = { http = 413, message = 'request_body_too_large' }, + unsupported_media_type = { http = 415, message = 'unsupported_media_type' }, + unauthenticated = { http = 401, message = 'unauthenticated' }, + forbidden = { http = 403, message = 'forbidden' }, + not_found = { http = 404, message = 'not_found' }, + timeout = { http = 504, message = 'timeout' }, + conflict = { http = 409, message = 'conflict' }, + closed = { http = 499, message = 'closed' }, + internal = { http = 500, message = 'internal_error' }, + upstream_failed = { http = 502, message = 'upstream_failed' }, +} + +local function norm_key(err) + if type(err) == 'table' then + return err.code or err.kind or err.error or 'internal' + end + local s = tostring(err or 'internal') + if M.codes[s] then return s end + if s == 'slot_busy' or s:find('slot_busy', 1, true) or s:find('busy', 1, true) or s:find('conflict', 1, true) then return 'conflict' end + if s:find('timeout', 1, true) then return 'timeout' end + if s:find('unauth', 1, true) then return 'unauthenticated' end + if s:find('forbidden', 1, true) or s:find('permission', 1, true) then return 'forbidden' end + if s:find('not_found', 1, true) or s:find('no_route', 1, true) then return 'not_found' end + if s:find('closed', 1, true) or s:find('cancelled', 1, true) then return 'closed' end + return 'internal' +end + +function M.normalise(err) + local key = norm_key(err) + local spec = M.codes[key] or M.codes.internal + local detail + if type(err) == 'table' then + detail = err.detail or err.reason or err.primary + else + detail = err + end + return { + code = key, + status = spec.http, + message = spec.message, + detail = detail and tostring(detail) or nil, + } +end + +function M.http_status(err) + return M.normalise(err).status +end + +function M.http_body(err) + local e = M.normalise(err) + return { + error = e.message, + code = e.code, + detail = e.detail, + } +end + +return M diff --git a/src/services/ui/fibers_cqueues.lua b/src/services/ui/fibers_cqueues.lua deleted file mode 100644 index fc1af261..00000000 --- a/src/services/ui/fibers_cqueues.lua +++ /dev/null @@ -1,33 +0,0 @@ -package.path="./?.lua;/usr/share/lua/?.lua;/usr/share/lua/?/init.lua;/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua" - .. package.path -package.path = "../../?.lua;../?.lua;" .. package.path - -local fiber = require "fibers.fiber" -local op = require "fibers.op" -local pollio = require "fibers.pollio" -local sleep = require "fibers.sleep" -local cqueues = require "cqueues" - -local old_step; old_step = cqueues.interpose("step", function(self, timeout) - if cqueues.running() then - fiber.yield() - return old_step(self, timeout) - else - local t = self:timeout() or math.huge - if timeout then - t = math.min(t, timeout) - end - - local events = self:events() - local fd = self:pollfd() - - local choices = {} - - if events ~= "w" then table.insert(choices, pollio.fd_readable_op(fd)) end - if events ~= "r" then table.insert(choices, pollio.fd_writable_op(fd)) end - if t ~= math.huge then table.insert(choices, sleep.sleep_op(t)) end - - op.choice(unpack(choices)):perform() - return old_step(self, 0.0) - end -end) diff --git a/src/services/ui/http/listener.lua b/src/services/ui/http/listener.lua new file mode 100644 index 00000000..04e8921a --- /dev/null +++ b/src/services/ui/http/listener.lua @@ -0,0 +1,238 @@ +-- services/ui/http/listener.lua +-- +-- UI HTTP listener consumer. +-- +-- UI does not own the HTTP backend. It obtains an HttpListener local handle +-- from the HTTP capability service, accepts HttpContext handles, and transfers +-- each accepted context into a request scope before any request work runs. + +local fibers = require 'fibers' +local resource = require 'devicecode.support.resource' +local scoped_work = require 'devicecode.support.scoped_work' +local http_sdk = require 'services.http'.sdk +local safe = require 'coxpcall' + +local M = {} + +local function emit(opts, ev) + local port = opts and opts.events_port + if not (port and type(port.emit_required) == 'function') then + error('ui.http.listener requires events_port', 3) + end + local ok, err = port:emit_required(ev, 'ui_http_listener_report_failed') + if ok ~= true then error(err or 'ui_http_listener_report_failed', 3) end + return true, nil +end + +local function terminate(obj, reason) + if obj == nil then return true, nil end + if type(obj) ~= 'table' then return nil, 'resource has no terminate method' end + if type(obj.terminate) == 'function' then return obj:terminate(reason) end + return nil, 'resource has no terminate method' +end + +local function terminate_checked(obj, reason, label) + local ok, err = terminate(obj, reason) + if ok ~= true then error((label or 'resource termination failed') .. ': ' .. tostring(err), 2) end + return true +end + +local function max_active_from(opts) + local n = opts.max_active_requests + if n == nil then return nil end + if type(n) ~= 'number' or n < 0 or n % 1 ~= 0 then + error('http.listener.run: max_active_requests must be a non-negative integer', 3) + end + return n +end + +local function default_run_request(scope, ctx, opts) + local request_mod = require 'services.ui.http.request' + return request_mod.run(scope, ctx, opts) +end + +local function request_id_of(ctx, next_id) + if type(ctx) == 'table' then + if type(ctx.id) == 'function' then + local ok, id = safe.pcall(function () return ctx:id() end) + if ok and id ~= nil then return id end + end + if ctx.id ~= nil then return ctx.id end + end + return next_id +end + +local function reject_overloaded(ctx, opts) + local reason = opts.overload_reason or 'http_request_backpressure' + if type(opts.reject_overloaded_request_now) == 'function' then + return opts.reject_overloaded_request_now(ctx, reason, opts) + end + return terminate(ctx, reason) +end + +local function install_request_owner(request_scope, accepted_owner) + local request_owner + local ctx, err = accepted_owner:handoff(function (value) + request_owner = resource.owned(value, { + label = 'HTTP request context termination', + terminate = function (v, reason) + return terminate(v, reason) + end, + }) + request_scope:finally(function (_, status, primary) + request_owner:terminate_checked(primary or status or 'terminated', 'HTTP request context termination') + end) + return true, nil + end) + if ctx == nil then return nil, err end + + return { + ctx = ctx, + owner = request_owner, + cancel_owned_now = function (reason) + if request_owner and request_owner:is_owned() then + return request_owner:terminate(reason or 'request_cancelled') + end + return terminate(ctx, reason or 'request_cancelled') + end, + }, nil +end + +local function obtain_listener_op(opts) + if opts.listener then + return fibers.always(opts.listener, nil) + end + + if type(opts.obtain_listener_op) == 'function' then + return opts.obtain_listener_op(opts) + end + + local conn = opts.conn + if conn == nil then return fibers.always(nil, 'http.listener.run: conn required') end + + local cap_id = opts.cap_id or 'main' + local listen_args = opts.listen or {} + if type(listen_args) ~= 'table' then return fibers.always(nil, 'http.listener.run: listen opts must be a table') end + local ref = http_sdk.new_ref(conn, cap_id) + return ref:listen_op(listen_args, opts.call_opts):wrap(function (reply, err) + if reply == nil then return nil, err or 'http_listen_failed' end + return reply.listener, nil + end) +end + +function M.run(scope, opts) + opts = opts or {} + + local run_request = opts.run_request or default_run_request + if type(run_request) ~= 'function' then + error('http.listener.run: run_request must be a function', 2) + end + + local listener = opts.listener + + -- A supplied listener is already being handed into this scope. Install the + -- terminating finaliser before any scope-aware perform so an immediate + -- cancellation during start-up cannot leak the handle. For SDK-created + -- listeners the same finaliser is installed first and becomes active as soon + -- as listener is assigned. + scope:finally(function (_, status, primary) + if listener ~= nil then + terminate_checked(listener, primary or status or 'http_listener_closed', 'HTTP listener termination') + end + end) + + if listener == nil then + local listen_err + listener, listen_err = fibers.perform(obtain_listener_op(opts)) + if listener == nil then error(listen_err or 'http.listener.run: listener unavailable', 0) end + end + + if type(listener.accept_op) ~= 'function' then + error('http.listener.run: listener must expose accept_op', 2) + end + + local active = 0 + local next_id = 0 + local max_active = max_active_from(opts) + + while true do + local ctx, err = fibers.perform(listener:accept_op()) + if ctx == nil then error(err or 'http accept failed', 0) end + + next_id = next_id + 1 + local request_id = request_id_of(ctx, next_id) + + if max_active ~= nil and active >= max_active then + local ok, rerr = reject_overloaded(ctx, opts) + emit(opts, { + kind = 'http_request_rejected', + request_id = request_id, + reason = opts.overload_reason or 'http_request_backpressure', + active_requests = active, + max_active_requests = max_active, + cleanup_ok = ok == true, + cleanup_err = rerr, + }) + if ok ~= true then error(rerr or 'http overloaded request cleanup failed', 0) end + else + local accepted_owner = resource.owned(ctx, { + label = 'accepted HTTP context cleanup', + terminate = function (value, reason) + return terminate(value, reason) + end, + }) + active = active + 1 + emit(opts, { + kind = 'http_request_started', + request_id = request_id, + active_requests = active, + }) + + local handle, start_err = scoped_work.start({ + lifetime_scope = scope, + reaper_scope = scope, + report_scope = scope, + identity = { + kind = 'http_request_done', + request_id = request_id, + }, + setup = function (request_scope) + return install_request_owner(request_scope, accepted_owner) + end, + run = function (request_scope, setup) + return run_request(request_scope, setup.ctx, opts) + end, + report = function (ev) + active = math.max(0, active - 1) + ev.active_requests = active + return emit(opts, ev) + end, + }) + + if not handle then + active = math.max(0, active - 1) + if accepted_owner:is_owned() then + accepted_owner:terminate_checked(start_err or 'http_request_start_failed', 'HTTP request start-failure cleanup') + end + emit(opts, { + kind = 'http_request_done', + request_id = request_id, + status = 'failed', + primary = start_err, + active_requests = active, + }) + end + end + end +end + +M._test = { + emit = emit, + max_active_from = max_active_from, + reject_overloaded = reject_overloaded, + install_request_owner = install_request_owner, + obtain_listener_op = obtain_listener_op, + terminate = terminate, +} + +return M diff --git a/src/services/ui/http/request.lua b/src/services/ui/http/request.lua new file mode 100644 index 00000000..2269507d --- /dev/null +++ b/src/services/ui/http/request.lua @@ -0,0 +1,526 @@ +-- services/ui/http/request.lua +-- +-- One HTTP request lifetime. + +local fibers = require 'fibers' +local response_mod = require 'services.ui.http.response' +local routes = require 'services.ui.http.routes' +local static = require 'services.ui.http.static' +local sse = require 'services.ui.http.sse' +local queries = require 'services.ui.queries' +local local_model = require 'services.ui.local_model' +local auth = require 'services.ui.auth' +local user_operation = require 'services.ui.user_operation' +local upload = require 'services.ui.update.upload' +local resource = require 'devicecode.support.resource' +local safe = require 'coxpcall' + +local ok_cjson, cjson = pcall(require, 'cjson.safe') +if not ok_cjson then cjson = require 'cjson' end + +local ok_http_headers, http_headers = pcall(require, 'services.http.headers') +if not ok_http_headers then http_headers = nil end + +local M = {} + +local UPDATE_COMMIT_TOPIC = { 'cap', 'update-manager', 'main', 'rpc', 'commit-job' } +local function monitor_rpc_topic(method) return { 'cap', 'monitor', 'main', 'rpc', method } end + +local function default_encode_json(value) + local encoded, err = cjson.encode(value) + if encoded == nil then error(err or 'json_encode_failed', 0) end + return encoded +end + +local function header_one(headers, name) + if not headers then return nil end + if http_headers and type(http_headers.get_one) == 'function' then + local v = http_headers.get_one(headers, name) + if v ~= nil then return v end + end + if type(headers.get) == 'function' then + local ok, v = safe.pcall(function () return headers:get(string.lower(name)) end) + if ok and v ~= nil then return v end + end + if type(headers) == 'table' then + return headers[name] or headers[string.lower(name)] or headers[string.upper(name)] + end + return nil +end + +local function ensure_request_metadata(ctx) + if type(ctx) ~= 'table' then return ctx end + if type(ctx.get_headers_op) ~= 'function' then return ctx end + + local h, err = ctx.headers, nil + if h == nil then h, err = fibers.perform(ctx:get_headers_op()) end + if not h then error(err or 'request headers unavailable', 0) end + ctx.headers = ctx.headers or h + ctx.method = ctx.method or header_one(h, ':method') or header_one(h, 'method') or 'GET' + ctx.path = header_one(h, ':path') or ctx.path or ctx.uri or '/' + return ctx +end + + +local function ctx_header(ctx, name) + return ctx and header_one(ctx.headers, name) +end + + +local function perform_response(ev) + -- HTTP response writes may yield through the transport bridge. They are + -- performed only inside the HTTP request scope, where that wait is visible. + local ok, err = fibers.perform(ev) + if ok ~= true then + error(err or 'response write failed', 0) + end + return true +end + +local function content_type_is_json(v) + v = tostring(v or ''):lower() + if v == '' then return false end + local mime = v:match('^%s*([^;%s]+)') or v + return mime == 'application/json' or mime:match('%+json$') ~= nil +end + +local function body_table(ctx) + -- Compatibility path for unit tests and internal harnesses which hand a + -- pre-parsed request body to the UI request boundary. Real HTTP contexts + -- must use json_body_table below so parsing and validation remain explicit. + if type(ctx.body) == 'table' then return ctx.body end + if type(ctx.json) == 'table' then return ctx.json end + return {} +end + +local function json_body_table(ctx, deps, opts) + opts = opts or {} + if type(ctx.body) == 'table' then return ctx.body, nil end + if type(ctx.json) == 'table' then return ctx.json, nil end + if type(ctx._ui_json_body) == 'table' then return ctx._ui_json_body, nil end + + local ct = ctx_header(ctx, 'content-type') or ctx_header(ctx, 'Content-Type') + if opts.require_json_content_type ~= false and not content_type_is_json(ct) then + return nil, 'unsupported_media_type' + end + + local raw + if type(ctx.body_string) == 'string' then + raw = ctx.body_string + elseif type(ctx.read_body_as_string_op) == 'function' then + raw = fibers.perform(ctx:read_body_as_string_op()) + elseif type(ctx.read_chars_op) == 'function' then + -- Fallback for older HTTP contexts. This is still one visible wait at the + -- request boundary; command handlers must not hide further body reads. + raw = fibers.perform(ctx:read_chars_op((deps and deps.max_json_body_bytes) or 1024 * 1024)) + else + return nil, 'invalid_body' + end + + if raw == nil then return nil, 'invalid_body' end + raw = tostring(raw or '') + local limit = opts.max_bytes or (deps and deps.max_json_body_bytes) or 1024 * 1024 + if #raw > limit then return nil, 'request_body_too_large' end + if raw == '' then return {}, nil end + + local obj, derr = cjson.decode(raw) + if obj == nil then return nil, 'invalid_json' end + if obj == cjson.null or type(obj) ~= 'table' then return nil, 'invalid_body' end + ctx._ui_json_body = obj + return obj, nil +end + +local function session_id_from(ctx) + return ctx.session_id + or (ctx.cookies and (ctx.cookies.sid or ctx.cookies.session or ctx.cookies.ui_session)) + or ctx_header(ctx, 'x-session-id') +end + +local function principal_from(ctx, deps) + local sid = session_id_from(ctx) + if sid and deps.sessions then + local sess = deps.sessions:get(sid) + if sess then return sess.principal, sess end + end + return nil, nil +end + +local function handle_read(owner, route, deps) + local model = assert(deps.model, 'HTTP read requires model') + local snap = model:snapshot() + local result + if route.query == 'all' then + result = queries.all(snap) + elseif route.query == 'services' then + result = queries.services_snapshot(snap) + elseif route.query == 'fabric' then + result = queries.fabric_status(snap) + elseif route.query == 'update_status' then + result = queries.update_status(snap) + elseif route.query == 'topic' then + result = queries.topic(snap, route.topic) + else + result = queries.all(snap) + end + perform_response(owner:reply_json_op(200, result)) + return { status = 'ok', route = 'read' } +end + + +local function handle_local_ui_bootstrap(owner, deps) + local model = assert(deps.model, 'local UI bootstrap requires model') + local result = local_model.bootstrap(model:snapshot()) + perform_response(owner:reply_json_op(200, result)) + return { status = 'ok', route = 'local_ui_bootstrap' } +end + +local function call_local_rpc(scope, topic, payload, deps, timeout) + return user_operation.run_op { + principal = { kind = 'service', id = 'ui-local', roles = { 'admin' } }, + conn = deps.conn, + connect = deps.connect, + bus = deps.bus, + disconnect_borrowed = false, + timeout = timeout or deps.command_timeout or 5.0, + run_op = function (_, conn) + return conn:call_op(topic, payload or {}, { timeout = false }) + :wrap(function (reply, call_err) + if reply == nil then return nil, call_err or 'upstream_failed' end + if type(reply) == 'table' and type(reply.ok) == 'boolean' then + if reply.ok then return { value = reply.reason }, nil end + return nil, tostring(reply.reason or call_err or 'upstream_failed') + end + return { value = reply }, nil + end) + end, + } +end + +local function handle_gsm_apns_get(scope, owner, deps) + local st, _rep, result_or_primary = fibers.perform(call_local_rpc( + scope, + { 'cap', 'gsm', 'main', 'rpc', 'list-custom-apns' }, + {}, + deps, + deps.apn_timeout or 5.0 + )) + if st ~= 'ok' then + perform_response(owner:reply_error_op(503, result_or_primary or 'gsm_apns_unavailable')) + return { status = 'failed', err = result_or_primary } + end + perform_response(owner:reply_json_op(200, result_or_primary.value or {})) + return { status = 'ok', route = 'gsm_apns_get' } +end + +local function handle_gsm_apns_put(scope, owner, ctx, deps) + local body, berr = json_body_table(ctx, deps, { require_json_content_type = true }) + if not body then + perform_response(owner:reply_error_op(nil, berr)) + return { status = 'bad_request', err = berr } + end + local st, _rep, result_or_primary = fibers.perform(call_local_rpc( + scope, + { 'cap', 'gsm', 'main', 'rpc', 'replace-custom-apns' }, + body, + deps, + deps.apn_timeout or 5.0 + )) + if st ~= 'ok' then + local detail = tostring(result_or_primary or 'gsm_apns_update_failed') + perform_response(owner:reply_json_op(400, { + error = detail, + code = 'gsm_apns_update_failed', + detail = detail, + })) + return { status = 'failed', err = result_or_primary } + end + perform_response(owner:reply_json_op(200, { ok = true, apns = result_or_primary.value or {} })) + return { status = 'ok', route = 'gsm_apns_put' } +end + +local function handle_diagnostics_stub(owner) + perform_response(owner:reply_json_op(200, { + schema = 'devicecode.diagnostics.stub/1', + stub = true, + diagnostics = { + bootstrap_installed = { expected = 0, installed = 0, missing = {} }, + modems_installed = { expected = 0, installed = 0, missing = {} }, + packages_installed = { expected = 0, installed = 0, missing = {} }, + packages_running = { expected = 0, installed = 0, missing = {} }, + services_running = { expected = 0, installed = 0, missing = {} }, + }, + diagnostics_logs = { 'Diagnostics service is not implemented in this build.' }, + })) + return { status = 'ok', route = 'diagnostics_stub' } +end + +local function handle_login(owner, ctx, deps) + local body, berr = json_body_table(ctx, deps, { require_json_content_type = false }) + if not body then + perform_response(owner:reply_error_op(nil, berr)) + return { status = 'bad_request', err = berr } + end + local principal, err = auth.verify(deps.auth, body) + if not principal then + perform_response(owner:reply_error_op(401, err or 'unauthenticated')) + return { status = 'unauthenticated' } + end + local sess = assert(deps.sessions, 'login requires sessions'):create(principal, { + data = { user_agent = ctx_header(ctx, 'user-agent') }, + }) + perform_response(owner:reply_json_op(200, { session = sess })) + return { status = 'ok', session_id = sess.id } +end + +local function handle_logout(owner, ctx, deps) + local sid = session_id_from(ctx) + if sid and deps.sessions then deps.sessions:delete(sid) end + perform_response(owner:reply_json_op(200, { ok = true })) + return { status = 'ok' } +end + +local function handle_session_get(owner, ctx, deps) + local sid = session_id_from(ctx) + local sess = sid and deps.sessions and deps.sessions:get(sid) or nil + if not sess then + perform_response(owner:reply_error_op(401, 'unauthenticated')) + return { status = 'unauthenticated' } + end + perform_response(owner:reply_json_op(200, { session = sess })) + return { status = 'ok' } +end + +local function handle_update_commit(scope, owner, ctx, deps) + local update_deps = deps.update or deps + local principal = nil + if update_deps.commit_require_auth == true then + principal = principal_from(ctx, deps) + if principal == nil then + perform_response(owner:reply_error_op(401, 'unauthenticated')) + return { status = 'unauthenticated' } + end + end + + local payload, perr = json_body_table(ctx, deps, { require_json_content_type = true }) + if not payload then + perform_response(owner:reply_error_op(nil, perr)) + return { status = 'bad_request', err = perr } + end + if type(payload.job_id) ~= 'string' or payload.job_id == '' then + perform_response(owner:reply_error_op(400, 'missing_job_id')) + return { status = 'bad_request', err = 'missing_job_id' } + end + + local op_spec = { + timeout = update_deps.commit_timeout or deps.command_timeout or 5.0, + run_op = function (_, conn) + return conn:call_op(UPDATE_COMMIT_TOPIC, payload, { timeout = false }) + :wrap(function (value, call_err) + if value == nil then return nil, call_err or 'upstream_failed' end + return { value = value }, nil + end) + end, + } + if principal ~= nil then + op_spec.principal = principal + op_spec.connect = update_deps.connect or deps.connect + op_spec.bus = update_deps.bus or deps.bus + if op_spec.connect == nil and op_spec.bus == nil then + op_spec.conn = update_deps.conn or deps.conn + end + else + -- Public prototype update commits are HTTP-public but not bus-anonymous: + -- they borrow the UI service connection supplied by services.ui.service. + op_spec.conn = update_deps.conn or deps.conn + if op_spec.conn == nil then + op_spec.principal = update_deps.principal or deps.principal + op_spec.connect = update_deps.connect or deps.connect + op_spec.bus = update_deps.bus or deps.bus + end + end + + local st, _rep, result_or_primary = fibers.perform(user_operation.run_op(op_spec)) + if st ~= 'ok' then + perform_response(owner:reply_error_op(nil, result_or_primary)) + return { status = 'failed', err = result_or_primary } + end + local result = result_or_primary + perform_response(owner:reply_json_op(200, result)) + return { status = 'ok', route = 'update_commit' } +end + + +local function call_monitor_op(deps, method, payload, timeout) + local conn = assert(deps.conn, 'monitor UI endpoint requires bus connection') + return conn:call_op(monitor_rpc_topic(method), payload or {}, { timeout = timeout or deps.command_timeout or 5.0 }) +end + +local function handle_logs_query(route, owner, deps) + local payload = { limit = 200, min_level = 'info' } + if route and route.boot == true then payload.boot = true end + local reply, err = fibers.perform(call_monitor_op(deps, 'query-logs', payload)) + if reply == nil then + perform_response(owner:reply_error_op(503, err or 'monitor_unavailable')) + return { status = 'failed', err = err or 'monitor_unavailable' } + end + perform_response(owner:reply_json_op(200, reply)) + return { status = 'ok', route = 'logs_query' } +end + +local function handle_logs_follow(scope, owner, deps) + local reply, err = fibers.perform(call_monitor_op( + deps, + 'follow-logs', + { limit = 200, min_level = 'info', replay = true }, + false + )) + if reply == nil or reply.ok ~= true or reply.feed == nil then + perform_response(owner:reply_error_op(503, err or (reply and reply.err) or 'monitor_follow_unavailable')) + return { status = 'failed', err = err or (reply and reply.err) or 'monitor_follow_unavailable' } + end + local feed = reply.feed + scope:finally(function (_, status, primary) + if feed and type(feed.close) == 'function' then feed:close(primary or status or 'ui_log_follow_closed') end + end) + perform_response(owner:write_headers_op(200, { + ['content-type'] = 'text/event-stream', + ['cache-control'] = 'no-cache', + ['connection'] = 'keep-alive', + })) + local encode = deps.encode_json or default_encode_json + while true do + local ev, rerr = fibers.perform(feed:recv_op()) + if ev == nil then return { status = 'closed', err = rerr } end + perform_response(owner:write_chunk_op(sse.frame_event(ev, encode))) + end +end + +local function handle_monitor_profile(scope, owner, ctx, deps) + local principal = principal_from(ctx, deps) + if principal == nil then + perform_response(owner:reply_error_op(401, 'unauthenticated')) + return { status = 'unauthenticated' } + end + local payload, perr = json_body_table(ctx, deps, { require_json_content_type = true }) + if not payload then + perform_response(owner:reply_error_op(nil, perr)) + return { status = 'bad_request', err = perr } + end + local st, _rep, result_or_primary = fibers.perform(user_operation.run_op { + principal = principal, + connect = deps.connect, + bus = deps.bus, + conn = deps.conn, + timeout = deps.command_timeout or 5.0, + run_op = function (_, conn) + return conn:call_op(monitor_rpc_topic('set-profile'), payload, { timeout = false }) + :wrap(function (value, call_err) + if value == nil then return nil, call_err or 'upstream_failed' end + return { value = value }, nil + end) + end, + }) + if st ~= 'ok' then + perform_response(owner:reply_error_op(nil, result_or_primary)) + return { status = 'failed', err = result_or_primary } + end + perform_response(owner:reply_json_op(200, result_or_primary)) + return { status = 'ok', route = 'monitor_profile' } +end + +local function handle_command(scope, owner, ctx, route, deps) + if type(route.topic) ~= 'table' or #route.topic == 0 then + perform_response(owner:reply_error_op(400, 'bad_request')) + return { status = 'bad_request', err = 'missing_command_topic' } + end + local principal = principal_from(ctx, deps) + if principal == nil then + perform_response(owner:reply_error_op(401, 'unauthenticated')) + return { status = 'unauthenticated' } + end + local payload, perr = json_body_table(ctx, deps, { require_json_content_type = true }) + if not payload then + perform_response(owner:reply_error_op(nil, perr)) + return { status = 'bad_request', err = perr } + end + local st, _rep, result_or_primary = fibers.perform(user_operation.run_op { + principal = principal, + connect = deps.connect, + bus = deps.bus, + conn = deps.conn, + timeout = deps.command_timeout or 5.0, + run_op = function (_, conn) + return conn:call_op(route.topic, payload, { timeout = false }) + :wrap(function (value, call_err) + if value == nil then return nil, call_err or 'upstream_failed' end + return { value = value }, nil + end) + end, + }) + if st ~= 'ok' then + perform_response(owner:reply_error_op(nil, result_or_primary)) + return { status = 'failed', err = result_or_primary } + end + local result = result_or_primary + perform_response(owner:reply_json_op(200, result)) + return { status = 'ok' } +end + +function M.run(scope, ctx, deps) + deps = deps or {} + local owner = response_mod.new(ctx, { encode = deps.encode_json or default_encode_json }) + + scope:finally(function (_, status, primary) + resource.terminate_checked(owner, primary or status or 'request_closed', 'HTTP response termination') + end) + + ensure_request_metadata(ctx) + local route = routes.decode(ctx) + + if route.kind == 'read' then + return handle_read(owner, route, deps) + elseif route.kind == 'local_ui_bootstrap' then + return handle_local_ui_bootstrap(owner, deps) + elseif route.kind == 'gsm_apns_get' then + return handle_gsm_apns_get(scope, owner, deps) + elseif route.kind == 'gsm_apns_put' then + return handle_gsm_apns_put(scope, owner, ctx, deps) + elseif route.kind == 'diagnostics_stub' then + return handle_diagnostics_stub(owner) + elseif route.kind == 'login' then + return handle_login(owner, ctx, deps) + elseif route.kind == 'logout' then + return handle_logout(owner, ctx, deps) + elseif route.kind == 'session_get' then + return handle_session_get(owner, ctx, deps) + elseif route.kind == 'logs_query' then + return handle_logs_query(route, owner, deps) + elseif route.kind == 'logs_follow' then + return handle_logs_follow(scope, owner, deps) + elseif route.kind == 'monitor_profile' then + return handle_monitor_profile(scope, owner, ctx, deps) + elseif route.kind == 'command' then + return handle_command(scope, owner, ctx, route, deps) + elseif route.kind == 'update_commit' then + return handle_update_commit(scope, owner, ctx, deps) + elseif route.kind == 'upload' then + local update_deps = deps.update or deps + if update_deps.require_auth == true then + local principal = principal_from(ctx, deps) + if principal == nil then + perform_response(owner:reply_error_op(401, 'unauthenticated')) + return { status = 'unauthenticated' } + end + end + return upload.run(scope, owner, ctx, update_deps) + elseif route.kind == 'sse' then + return sse.run(scope, owner, route, deps) + elseif route.kind == 'static' then + return static.run(scope, owner, route, deps) + else + perform_response(owner:reply_error_op(404, 'not_found')) + return { status = 'not_found' } + end +end + +return M diff --git a/src/services/ui/http/response.lua b/src/services/ui/http/response.lua new file mode 100644 index 00000000..d878d17a --- /dev/null +++ b/src/services/ui/http/response.lua @@ -0,0 +1,326 @@ +-- services/ui/http/response.lua +-- +-- Response owner for one HTTP request. +-- +-- The response owner is the only UI object that writes an HTTP response. It +-- owns the response state machine and exposes Op-returning write methods for +-- request-owned scopes. Finalisers must use terminate(reason) only; they +-- never write bytes. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local errors = require 'services.ui.errors' +local tablex = require 'shared.table' +local safe = require 'coxpcall' + +local ok_http_headers, http_headers = pcall(require, 'services.http.headers') +if not ok_http_headers then http_headers = nil end + +local M = {} +local Response = {} +Response.__index = Response + +local TERMINAL = { + replied = true, + ended = true, + abandoned = true, +} + +local shallow_copy = tablex.shallow_copy + +local function normalise_write_result(ok, err, fallback) + if ok == false or ok == nil then + return nil, err or fallback or 'response write failed' + end + return true, nil +end + +local function terminate_ctx(ctx, reason) + if not ctx then return true, nil end + local fn = ctx.terminate or ctx.abandon_now + if type(fn) ~= 'function' then return nil, 'response context has no terminate' end + local called, ok, err = safe.pcall(fn, ctx, reason) + if not called then return nil, ok or 'response context termination failed' end + if ok == false or ok == nil then return nil, err or 'response context termination failed' end + return true, nil +end + +local function is_http_service_context(ctx) + return type(ctx) == 'table' and (type(ctx._raw_context) == 'function' or type(ctx.registry_id) == 'function') +end + +local function response_headers(status, fields) + if http_headers and type(http_headers.status) == 'function' then + return http_headers.status(status, fields or {}) + end + local out = shallow_copy(fields) + out[':status'] = tostring(status or 200) + return out, nil +end + +local function call_write_headers_op(ctx, fn, status, headers, opts) + if is_http_service_context(ctx) then + local h, herr = response_headers(status, headers) + if not h then return fibers.always(nil, herr or 'response headers construction failed') end + return fn(ctx, h, not not (opts and opts.end_stream)) + end + return fn(ctx, status, headers, opts) +end + +local function call_write_chunk_op(ctx, fn, chunk, opts) + if is_http_service_context(ctx) then + return fn(ctx, chunk or '', not not (opts and opts.end_stream)) + end + return fn(ctx, chunk or '', opts) +end + +local function ensure_method(self, name) + if not (self._ctx and type(self._ctx[name]) == 'function') then + return nil, 'response context has no ' .. name + end + return self._ctx[name] +end + +local function mark_abandoned(self, reason) + if TERMINAL[self._state] then return false, 'response already resolved' end + self._state = 'abandoned' + self._abandoned = reason or 'abandoned' + return true, nil +end + +local function new_start_token(self) + local response_ref = self + local token = { active = true } + + function token:release() + if not self.active then return false end + self.active = false + if response_ref._start_token == token then + response_ref._start_token = nil + end + return true + end + + return token +end + +local function acquire_start_token_op(self) + local function try() + if self._start_token ~= nil then + return true, nil, 'response already starting' + end + if self._state ~= 'unresolved' then + return true, nil, 'response already started' + end + + -- Do not mutate response state during readiness probing. The token is + -- committed by the primitive wrap only after this arm has won. + return true, new_start_token(self), nil + end + + local function block() + error('response start token op should never block', 0) + end + + local function wrap(token, err) + if not token then return nil, err end + if self._start_token ~= nil then + return nil, 'response already starting' + end + if self._state ~= 'unresolved' then + return nil, 'response already started' + end + self._start_token = token + return token, nil + end + + return op.new_primitive(wrap, try, block) +end + +function M.new(ctx, opts) + opts = opts or {} + return setmetatable({ + _ctx = ctx or {}, + _state = 'unresolved', + _status = nil, + _abandoned = nil, + _start_token = nil, + _encode = opts.encode, + }, Response) +end + +function Response:state() + return self._state +end + +function Response:done() + return TERMINAL[self._state] or false +end + +function Response:write_headers_op(status, headers, opts) + status = status or 200 + headers = shallow_copy(headers) + opts = opts or {} + + return fibers.run_scope_op(function (scope) + local fn, missing = ensure_method(self, 'write_headers_op') + if not fn then return { ok = nil, err = missing } end + + local token, token_err = fibers.perform(acquire_start_token_op(self)) + if not token then return { ok = nil, err = token_err } end + + local finished = false + scope:finally(function () + if not finished then + token:release() + mark_abandoned(self, 'response_headers_aborted') + terminate_ctx(self._ctx, 'response_headers_aborted') + end + end) + + local ok, err = fibers.perform(call_write_headers_op(self._ctx, fn, status, headers, { + end_stream = not not opts.end_stream, + timeout = opts.timeout, + })) + + finished = true + token:release() + + local write_ok, write_err = normalise_write_result(ok, err, 'response headers write failed') + if write_ok ~= true then + self._state = 'abandoned' + self._abandoned = write_err + return { ok = nil, err = write_err } + end + + self._status = status + self._state = opts.end_stream and 'ended' or 'headers_sent' + return { ok = true } + end):wrap(function (st, _rep, result_or_primary) + if st ~= 'ok' then + return nil, result_or_primary or st + end + local result = result_or_primary or {} + return result.ok, result.err + end) +end + +function Response:write_chunk_op(chunk, opts) + opts = opts or {} + + return fibers.guard(function () + if self._state ~= 'headers_sent' then + return fibers.always(nil, 'response stream is not open') + end + + local fn, missing = ensure_method(self, 'write_chunk_op') + if not fn then return fibers.always(nil, missing) end + + local finished = false + return call_write_chunk_op(self._ctx, fn, chunk or '', { + end_stream = not not opts.end_stream, + timeout = opts.timeout, + }):wrap(function (ok, err) + finished = true + local write_ok, write_err = normalise_write_result(ok, err, 'response chunk write failed') + if write_ok ~= true then + self._state = 'abandoned' + self._abandoned = write_err + return nil, write_err + end + if opts.end_stream then self._state = 'ended' end + return true, nil + end):on_abort(function () + if not finished then + mark_abandoned(self, 'response_chunk_aborted') + terminate_ctx(self._ctx, 'response_chunk_aborted') + end + end) + end) +end + +function Response:end_stream_op(opts) + opts = opts or {} + opts.end_stream = true + return self:write_chunk_op('', opts) +end + +function Response:reply_op(status, body, headers, opts) + status = status or 200 + body = body or '' + headers = shallow_copy(headers) + opts = opts or {} + + return fibers.guard(function () + if self._state ~= 'unresolved' then + return fibers.always(nil, 'response already resolved') + end + + local no_body = body == '' or body == nil or not not opts.no_body + return fibers.run_scope_op(function () + local ok, err = fibers.perform(self:write_headers_op(status, headers, { + end_stream = no_body, + timeout = opts.timeout, + })) + if ok ~= true then error(err or 'response headers write failed', 0) end + + if not no_body then + ok, err = fibers.perform(self:write_chunk_op(body, { + end_stream = true, + timeout = opts.timeout, + })) + if ok ~= true then error(err or 'response chunk write failed', 0) end + end + + self._state = 'replied' + return { ok = true } + end):on_abort(function () + if not TERMINAL[self._state] then + mark_abandoned(self, 'response_aborted') + terminate_ctx(self._ctx, 'response_aborted') + end + end):wrap(function (st, _rep, result_or_primary) + if st ~= 'ok' then + if not TERMINAL[self._state] then + mark_abandoned(self, result_or_primary or st) + terminate_ctx(self._ctx, result_or_primary or st) + end + return nil, result_or_primary or st + end + return result_or_primary and result_or_primary.ok == true, nil + end) + end) +end + +function Response:reply_json_op(status, body, headers, opts) + headers = shallow_copy(headers) + headers['content-type'] = headers['content-type'] or 'application/json' + local payload = body + if self._encode then payload = self._encode(body) end + return self:reply_op(status, payload, headers, opts) +end + +function Response:reply_error_op(status, err) + local e = err + if type(status) ~= 'number' then + e = err or status + status = errors.http_status(e) + elseif e == nil then + e = status + end + return self:reply_json_op(status, errors.http_body(e)) +end + +function Response:abandon_now(reason) + local ok, err = mark_abandoned(self, reason or 'abandoned') + if ok ~= true then return ok, err end + return terminate_ctx(self._ctx, self._abandoned) +end + +function Response:terminate(reason) + if TERMINAL[self._state] then return true, nil end + return self:abandon_now(reason or 'response_terminated') +end + +M.Response = Response +return M diff --git a/src/services/ui/http/routes.lua b/src/services/ui/http/routes.lua new file mode 100644 index 00000000..f747a931 --- /dev/null +++ b/src/services/ui/http/routes.lua @@ -0,0 +1,153 @@ +-- services/ui/http/routes.lua +-- +-- Pure route decoding for UI HTTP requests. + +local safe = require 'coxpcall' + +local M = {} + +local function split_path(path) + path = tostring(path or '/') + path = path:match('^([^?#]*)') or path + local out = {} + for part in path:gmatch('[^/]+') do + out[#out + 1] = part + end + return out +end + +local function query_flag(path, name) + local query = tostring(path or ''):match('%?([^#]*)') + if not query then return false end + for pair in query:gmatch('[^&]+') do + local key, value = pair:match('^([^=]*)=?(.*)$') + if key == name then + value = tostring(value or ''):lower() + return value == '' or value == '1' or value == 'true' or value == 'yes' + end + end + return false +end + +local function method_of(ctx) + if ctx and type(ctx.method) == 'function' then + local ok, v = safe.pcall(function () return ctx:method() end) + if ok and v ~= nil then return string.upper(tostring(v)) end + end + return string.upper(tostring((ctx and (ctx.method or ctx.verb)) or 'GET')) +end + +local function header_one(headers, name) + if not headers then return nil end + if type(headers.get) == 'function' then + local ok, v = safe.pcall(function () return headers:get(string.lower(name)) end) + if ok and v ~= nil then return v end + end + if type(headers) == 'table' then + return headers[name] or headers[string.lower(name)] or headers[string.upper(name)] + end + return nil +end + +local function path_of(ctx) + local header_path = ctx and header_one(ctx.headers, ':path') + if header_path ~= nil then return header_path end + if ctx and type(ctx.path) == 'function' then + local ok, v = safe.pcall(function () return ctx:path() end) + if ok and v ~= nil then return v end + end + return (ctx and (ctx.path or ctx.uri)) or '/' +end + +function M.decode(ctx) + local method = method_of(ctx) + local path = path_of(ctx) + local parts = split_path(path) + + if #parts == 0 then + return { kind = 'static', path = '/index.html' } + end + + if parts[1] == 'events' and method == 'GET' then + return { kind = 'sse' } + end + + if parts[1] ~= 'api' then + return { kind = 'static', path = '/' .. table.concat(parts, '/') } + end + + if parts[2] == 'login' and method == 'POST' then + return { kind = 'login' } + end + + if parts[2] == 'session' then + if method == 'GET' then return { kind = 'session_get' } end + if method == 'DELETE' or method == 'POST' then return { kind = 'logout' } end + end + + if parts[2] == 'local-ui' and parts[3] == 'bootstrap' and method == 'GET' then + return { kind = 'local_ui_bootstrap' } + end + + if parts[2] == 'gsm' and parts[3] == 'apns' and parts[4] == 'custom' then + if method == 'GET' then return { kind = 'gsm_apns_get' } end + if method == 'PUT' then return { kind = 'gsm_apns_put' } end + end + + if parts[2] == 'diagnostics' and method == 'GET' then + return { kind = 'diagnostics_stub' } + end + + if parts[2] == 'state' and method == 'GET' then + local topic = {} + for i = 3, #parts do topic[#topic + 1] = parts[i] end + if #topic == 0 then + return { kind = 'read', query = 'all' } + end + return { kind = 'read', query = 'topic', topic = topic } + end + + if parts[2] == 'services' and method == 'GET' then + return { kind = 'read', query = 'services' } + end + + if parts[2] == 'fabric' and method == 'GET' then + return { kind = 'read', query = 'fabric' } + end + + + if parts[2] == 'logs' and method == 'GET' then + if parts[3] == 'follow' or parts[3] == 'tail' then + return { kind = 'logs_follow' } + end + return { kind = 'logs_query', boot = query_flag(path, 'boot') } + end + + if parts[2] == 'monitor' and parts[3] == 'profile' and method == 'POST' then + return { kind = 'monitor_profile' } + end + + if parts[2] == 'update' and method == 'GET' then + if parts[3] == nil or parts[3] == 'status' then + return { kind = 'read', query = 'update_status' } + end + end + + if parts[2] == 'update' and parts[3] == 'upload' and method == 'POST' then + return { kind = 'upload' } + end + + if parts[2] == 'update' and parts[3] == 'commit' and method == 'POST' then + return { kind = 'update_commit' } + end + + if parts[2] == 'call' and method == 'POST' then + local topic = {} + for i = 3, #parts do topic[#topic + 1] = parts[i] end + return { kind = 'command', topic = topic } + end + + return { kind = 'not_found' } +end + +return M diff --git a/src/services/ui/http/sse.lua b/src/services/ui/http/sse.lua new file mode 100644 index 00000000..76d17833 --- /dev/null +++ b/src/services/ui/http/sse.lua @@ -0,0 +1,198 @@ +-- services/ui/http/sse.lua +-- +-- Server-sent-event response owner for UI read-model watches. +-- +-- SSE is request-owned streaming HTTP work. It observes the local UI read-model +-- watch owner and writes framed events through the response owner. It does not +-- subscribe to retained bus state directly and it does not know about the HTTP +-- backend implementation. + +local fibers = require 'fibers' +local local_model = require 'services.ui.local_model' + +local ok_cjson, cjson = pcall(require, 'cjson.safe') +if not ok_cjson then cjson = require 'cjson' end + +local M = {} + +local function default_encode_json(v) + local encoded, err = cjson.encode(v) + if encoded == nil then error(err or 'json_encode_failed', 0) end + return encoded +end + +local function perform_required(ev, label) + local ok, err = fibers.perform(ev) + if ok ~= true then error(err or label or 'sse write failed', 0) end + return true +end + +local function topic_to_string(topic) + local parts = {} + for i = 1, #(topic or {}) do parts[i] = tostring(topic[i]) end + return table.concat(parts, '/') +end + +local function copy_topic(topic) + local out = {} + for i = 1, #(topic or {}) do out[i] = topic[i] end + return out +end + +local function pattern_from_prefix(prefix) + local out = copy_topic(prefix) + out[#out + 1] = '#' + return out +end + +local function copy_patterns(patterns) + local out = {} + for i, pattern in ipairs(patterns or {}) do out[i] = copy_topic(pattern) end + return out +end + +local function default_patterns() + local out = {} + for _, prefix in ipairs(local_model.ALLOW_PREFIXES or {}) do + if #prefix > 0 then out[#out + 1] = pattern_from_prefix(prefix) end + end + for _, pattern in ipairs(local_model.LIVE_EVENT_PATTERNS or {}) do + if #pattern > 0 then out[#out + 1] = copy_topic(pattern) end + end + return out +end + +local function patterns_for(route, opts) + route = route or {} + opts = opts or {} + local sse_cfg = opts.sse or {} + if route.pattern then return { copy_topic(route.pattern) } end + if route.patterns then return copy_patterns(route.patterns) end + if sse_cfg.pattern then return { copy_topic(sse_cfg.pattern) } end + if sse_cfg.patterns then return copy_patterns(sse_cfg.patterns) end + return default_patterns() +end + +local function frame_event(ev, encode) + encode = encode or default_encode_json + local name = (ev and ev.op) or (ev and ev.kind) or 'message' + local data = encode(ev or {}) + local id = ev and ev.topic and topic_to_string(ev.topic) or nil + local out = {} + if id and id ~= '' then out[#out + 1] = 'id: ' .. id .. '\n' end + out[#out + 1] = 'event: ' .. tostring(name) .. '\n' + data = tostring(data) + if data == '' then + out[#out + 1] = 'data: \n' + else + local start = 1 + while start <= #data do + local nl = data:find('\n', start, true) + local line + if nl then + line = data:sub(start, nl - 1) + start = nl + 1 + else + line = data:sub(start) + start = #data + 1 + end + out[#out + 1] = 'data: ' .. line .. '\n' + end + end + out[#out + 1] = '\n' + return table.concat(out) +end + +local function event_allowed(ev) + if type(ev) ~= 'table' or ev.topic == nil then return true end + return local_model.allowed_event(ev.topic) +end + +local function project_event(ev) + if type(ev) ~= 'table' or ev.topic == nil then return ev end + return local_model.project_event(ev) +end + +local function terminate_watches(watches, reason) + for i = #watches, 1, -1 do + local watch = watches[i] + if watch and type(watch.terminate) == 'function' then + pcall(function () watch:terminate(reason or 'sse_closed') end) + end + end +end + +local function open_watches(watch_owner, patterns, opts) + local watches = {} + for _, pattern in ipairs(patterns) do + local watch, err = watch_owner:watch_open(pattern, opts) + if not watch then + terminate_watches(watches, err or 'watch_open_failed') + return nil, err or 'watch_open_failed' + end + watches[#watches + 1] = watch + end + return watches, nil +end + +local function next_event_op(watches) + local ops = {} + for i, watch in ipairs(watches) do + ops[i] = watch:recv_op() + end + return fibers.first_ready(ops) +end + +function M.run(scope, owner, route, opts) + opts = opts or {} + local watch_owner = assert(opts.watch_owner, 'SSE requires watch_owner') + local encode = opts.encode_json or opts.encode or default_encode_json + local patterns = patterns_for(route, opts) + local replay = (opts.sse and opts.sse.replay == true) or false + if #patterns == 0 then + perform_required(owner:reply_error_op(503, 'no_sse_patterns'), 'SSE no-patterns error response failed') + return { status = 'failed', err = 'no_sse_patterns' } + end + + local watch_opts = { + replay = replay, + queue_len = (opts.sse and opts.sse.queue_len) or 32, + full = 'drop_oldest', + max_replay = opts.sse and opts.sse.max_replay, + } + local watches, err = open_watches(watch_owner, patterns, watch_opts) + if not watches then + perform_required(owner:reply_error_op(503, err or 'watch_open_failed'), 'SSE watch-open error response failed') + return { status = 'failed', err = err or 'watch_open_failed' } + end + + scope:finally(function (_, status, primary) + terminate_watches(watches, primary or status or 'sse_closed') + end) + + perform_required(owner:write_headers_op(200, { + ['content-type'] = 'text/event-stream', + ['cache-control'] = 'no-cache', + ['connection'] = 'keep-alive', + }), 'SSE headers write failed') + + while true do + local idx, ev, rerr = fibers.perform(next_event_op(watches)) + if ev == nil then + terminate_watches(watches, rerr or 'sse_watch_closed') + return { status = 'closed', err = rerr, pattern = patterns[idx] } + end + local projected = project_event(ev) + if projected then + perform_required(owner:write_chunk_op(frame_event(projected, encode)), 'SSE event write failed') + end + end +end + +M.frame_event = frame_event +M.event_allowed = event_allowed +M.project_event = project_event +M.default_patterns = default_patterns +M.patterns_for = patterns_for +M.default_encode_json = default_encode_json +return M diff --git a/src/services/ui/http/static.lua b/src/services/ui/http/static.lua new file mode 100644 index 00000000..e8f787a2 --- /dev/null +++ b/src/services/ui/http/static.lua @@ -0,0 +1,97 @@ +-- services/ui/http/static.lua +-- +-- Static file response worker for one HTTP request scope. Static responses are +-- streamed through the response owner; this module never writes transport bytes +-- directly. + +local fibers = require 'fibers' +local file_io = require 'fibers.io.file' +local resource = require 'devicecode.support.resource' + +local M = {} + +local TYPES = { + ['.html'] = 'text/html', + ['.css'] = 'text/css', + ['.js'] = 'application/javascript', + ['.json'] = 'application/json', + ['.webmanifest'] = 'application/manifest+json', + ['.png'] = 'image/png', + ['.svg'] = 'image/svg+xml', + ['.ico'] = 'image/x-icon', + ['.webp'] = 'image/webp', + ['.woff'] = 'font/woff', + ['.woff2'] = 'font/woff2', + ['.ttf'] = 'font/ttf', + ['.txt'] = 'text/plain', +} + +local function content_type(path) + local ext = tostring(path or ''):match('(%.[^.]+)$') + return (ext and TYPES[ext]) or 'application/octet-stream' +end + +local function clean_path(root, path) + path = tostring(path or '/index.html') + path = path:gsub('%.%.+', '') + if path == '/' then path = '/index.html' end + return (root or '.') .. path +end + +local function perform_required(op, label) + local ok, err = fibers.perform(op) + if ok ~= true then error(err or label or 'response write failed', 0) end + return true +end + +function M.run(scope, owner, route, opts) + opts = opts or {} + local requested_path = route.path + local filename = clean_path(opts.root or '.', requested_path) + local f, err = file_io.open(filename, 'r') + if not f then + local has_ext = tostring(requested_path or ''):match('/[^/]*%.[^/%.]+$') ~= nil + if opts.spa_fallback ~= false and not has_ext then + filename = clean_path(opts.root or '.', '/index.html') + f, err = file_io.open(filename, 'r') + if f then requested_path = '/index.html' end + end + end + if not f then + perform_required(owner:reply_error_op(404, 'not_found'), 'static not found response failed') + return { status = 'not_found', path = route.path, err = err } + end + + local file_owner = resource.owned(f, { + label = 'static file cleanup', + terminate = function (file) + if file and type(file.close) == 'function' then return file:close() end + return true, nil + end, + }) + scope:finally(function (_, status, primary) + file_owner:terminate_checked(primary or status or 'request_closed', 'static file cleanup') + end) + + perform_required(owner:write_headers_op(200, { ['content-type'] = content_type(filename) }), 'static headers write failed') + + local bytes = 0 + local chunk_size = opts.chunk_size or 16384 + while true do + local chunk, rerr = fibers.perform(f:read_some_op(chunk_size)) + if rerr ~= nil then + owner:abandon_now(rerr) + error(rerr, 0) + end + if chunk == nil then break end + bytes = bytes + #chunk + perform_required(owner:write_chunk_op(chunk), 'static chunk write failed') + end + + perform_required(owner:end_stream_op(), 'static end write failed') + file_owner:terminate_checked('done', 'static file cleanup') + f = nil + return { status = 'ok', path = requested_path, bytes = bytes } +end + +return M diff --git a/src/services/ui/local-ui b/src/services/ui/local-ui deleted file mode 160000 index 10e0cdcc..00000000 --- a/src/services/ui/local-ui +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 10e0cdcce06fc225ca291397a64a231069138872 diff --git a/src/services/ui/local_model.lua b/src/services/ui/local_model.lua new file mode 100644 index 00000000..456d13b5 --- /dev/null +++ b/src/services/ui/local_model.lua @@ -0,0 +1,150 @@ +-- services/ui/local_model.lua +-- +-- Pure retained-state projection for the temporary Vue/Tailwind local UI. +-- This module deliberately selects a small allow-list of useful retained topics +-- rather than exposing the full /api/state model to ordinary local pages. + +local tablex = require 'shared.table' +local topics = require 'services.ui.topics' + +local M = {} + +local copy = tablex.deep_copy +local function prefix_matches(topic, prefix) + if type(topic) ~= 'table' or type(prefix) ~= 'table' then return false end + if #prefix > #topic then return false end + for i = 1, #prefix do + local p = prefix[i] + if p == '#' then return true end + if p ~= '+' and topic[i] ~= p then return false end + end + return true +end + +local DROP_KEYS = { + raw = true, + raw_facts = true, +} + +local ALLOW_PREFIXES = { + { 'svc', '+', 'status' }, + { 'state', 'device' }, + { 'state', 'net' }, + { 'state', 'gsm' }, + { 'state', 'system' }, + { 'state', 'fabric' }, + { 'state', 'update' }, + { 'state', 'workflow', 'update-job' }, +} + +local LIVE_EVENT_PATTERNS = {} + +local DENY_PREFIXES = { + { 'cfg' }, + { 'raw' }, + { 'state', 'ui' }, + { 'svc', 'ui' }, + { 'obs', 'v1', 'ui' }, +} + +local function is_log_topic(topic) + return type(topic) == 'table' + and #topic == 5 + and topic[1] == 'obs' + and topic[2] == 'v1' + and topic[4] == 'event' + and topic[5] == 'log' +end + +local function allowed(topic) + if is_log_topic(topic) then return false end + for _, prefix in ipairs(DENY_PREFIXES) do + if prefix_matches(topic, prefix) then return false end + end + for _, prefix in ipairs(ALLOW_PREFIXES) do + if prefix_matches(topic, prefix) then return true end + end + return false +end + +local function allowed_event(topic) + for _, prefix in ipairs(DENY_PREFIXES) do + if prefix_matches(topic, prefix) then return false end + end + return allowed(topic) or is_log_topic(topic) +end + +local function sorted_items(snapshot) + local out = {} + for _, msg in pairs((snapshot and snapshot.items) or {}) do + if type(msg) == 'table' and type(msg.topic) == 'table' and allowed(msg.topic) then + out[#out + 1] = { + topic = copy(msg.topic), + payload = M.project_payload(msg.topic, msg.payload), + origin = copy(msg.origin), + } + end + end + table.sort(out, function(a, b) + return topics.topic_key(a.topic) < topics.topic_key(b.topic) + end) + return out +end + +local function strip_payload(value) + if type(value) ~= 'table' then return value end + local out = {} + for k, v in pairs(value) do + if not DROP_KEYS[k] then + out[k] = strip_payload(v) + end + end + return out +end + +function M.project_payload(topic, payload) + local out = strip_payload(payload) + if type(out) ~= 'table' or type(topic) ~= 'table' then return out end + + if prefix_matches(topic, { 'state', 'device' }) then + local kind = topic[3] + if kind == 'identity' or kind == 'components' then + out.components = nil + elseif kind == 'component' then + out.observed = nil + end + end + + return out +end + +function M.project_event(ev) + if type(ev) ~= 'table' or ev.topic == nil then return copy(ev) end + if not allowed_event(ev.topic) then return nil end + + local out = copy(ev) + out.topic = copy(ev.topic) + out.payload = M.project_payload(ev.topic, ev.payload) + out.origin = copy(ev.origin) + return out +end + +function M.bootstrap(snapshot) + local out = {} + for _, msg in ipairs(sorted_items(snapshot)) do + out[topics.topic_string(msg.topic)] = msg + end + return { + schema = 'devicecode.ui.local-bootstrap/1', + version = snapshot and snapshot.version or 0, + items = out, + } +end + +M.allowed = allowed +M.allowed_event = allowed_event +M.ALLOW_PREFIXES = ALLOW_PREFIXES +M.DENY_PREFIXES = DENY_PREFIXES +M.LIVE_EVENT_PATTERNS = LIVE_EVENT_PATTERNS + +return M diff --git a/src/services/ui/queries.lua b/src/services/ui/queries.lua new file mode 100644 index 00000000..123117d3 --- /dev/null +++ b/src/services/ui/queries.lua @@ -0,0 +1,360 @@ +-- services/ui/queries.lua +-- +-- Pure read-only projections over read-model snapshots. + +local topics = require 'services.ui.topics' +local tablex = require 'shared.table' +local topicx = require 'shared.topic' + +local M = {} + +local copy_value = tablex.deep_copy + +local topic_has_prefix = topicx.starts_with + +local function sorted_items(snapshot, pred) + local out = {} + for _, msg in pairs((snapshot and snapshot.items) or {}) do + if not pred or pred(msg) then + out[#out + 1] = copy_value(msg) + end + end + table.sort(out, function(a, b) + return topics.topic_key(a.topic) < topics.topic_key(b.topic) + end) + return out +end + +function M.all(snapshot) + return { + version = snapshot and snapshot.version or 0, + items = sorted_items(snapshot), + } +end + +function M.topic(snapshot, topic) + local key = topics.topic_key(topic) + local item = snapshot and snapshot.items and snapshot.items[key] + return item and copy_value(item) or nil +end + +function M.pattern(snapshot, pattern, matcher) + if matcher then + return sorted_items(snapshot, function(msg) return matcher(pattern, msg.topic) end) + end + return sorted_items(snapshot, function(msg) + return topic_has_prefix(msg.topic, pattern) + end) +end + +function M.services_snapshot(snapshot) + return { + version = snapshot and snapshot.version or 0, + services = sorted_items(snapshot, function(msg) + return topic_has_prefix(msg.topic, { 'svc' }) + end), + } +end + +function M.fabric_status(snapshot) + return { + version = snapshot and snapshot.version or 0, + status = sorted_items(snapshot, function(msg) + return topic_has_prefix(msg.topic, { 'state', 'fabric' }) + or topic_has_prefix(msg.topic, { 'svc', 'fabric' }) + end), + } +end + +function M.update_jobs_snapshot(snapshot) + return { + version = snapshot and snapshot.version or 0, + jobs = sorted_items(snapshot, function(msg) + return topic_has_prefix(msg.topic, { 'state', 'workflow', 'update-job' }) + or topic_has_prefix(msg.topic, { 'state', 'update' }) + end), + } +end + + +local function payload_of(msg) + return type(msg) == 'table' and type(msg.payload) == 'table' and msg.payload or nil +end + +local function topic_len(topic) + return type(topic) == 'table' and #topic or 0 +end + +local function topic_at(topic, i) + return type(topic) == 'table' and topic[i] or nil +end + +local function is_update_job_topic(topic) + return topic_len(topic) == 4 + and topic_at(topic, 1) == 'state' + and topic_at(topic, 2) == 'workflow' + and topic_at(topic, 3) == 'update-job' +end + +local function is_update_timeline_topic(topic) + return topic_len(topic) == 5 + and topic_at(topic, 1) == 'state' + and topic_at(topic, 2) == 'workflow' + and topic_at(topic, 3) == 'update-job' + and topic_at(topic, 5) == 'timeline' +end + +local function is_fabric_transfer_topic(topic) + return topic_len(topic) == 4 + and topic_at(topic, 1) == 'state' + and topic_at(topic, 2) == 'fabric' + and topic_at(topic, 3) == 'transfer' +end + +local function is_fabric_transfer_component_topic(topic) + return topic_len(topic) == 6 + and topic_at(topic, 1) == 'state' + and topic_at(topic, 2) == 'fabric' + and topic_at(topic, 3) == 'link' + and topic_at(topic, 5) == 'component' + and (topic_at(topic, 6) == 'transfer' or topic_at(topic, 6) == 'transfer_manager') +end + +local function transfer_payload_from_component(payload) + if type(payload) ~= 'table' or type(payload.snapshot) ~= 'table' then return nil end + local snapshot = payload.snapshot + local rec = type(snapshot.active) == 'table' and snapshot.active or type(snapshot.last) == 'table' and snapshot.last or nil + if type(rec) ~= 'table' then return nil end + local result = type(rec.result) == 'table' and rec.result or {} + local xfer_id = rec.xfer_id or result.xfer_id + if type(xfer_id) ~= 'string' or xfer_id == '' then return nil end + local meta = type(rec.meta) == 'table' and rec.meta or {} + return { + kind = 'fabric.transfer', + link_id = payload.link_id, + link_generation = payload.link_generation, + xfer_id = xfer_id, + request_id = rec.request_id or result.request_id, + direction = rec.direction, + state = rec.status, + status = rec.status, + target = rec.target or result.target, + size = result.size or rec.size, + sent_bytes = result.sent_bytes, + received_bytes = result.received_bytes, + digest_alg = result.digest_alg or rec.digest_alg, + digest = result.digest or rec.digest, + retransmits = result.retransmits, + chunk_retries = result.chunk_retries, + chunks_sent = result.chunks_sent, + max_frame_queue_ms = result.max_frame_queue_ms, + max_need_to_chunk_ms = result.max_need_to_chunk_ms, + max_source_read_ms = result.max_source_read_ms, + max_send_ms = result.max_send_ms, + progress = type(rec.progress) == 'table' and copy_value(rec.progress) or nil, + error = rec.primary, + correlation = { + job_id = result.job_id or meta.job_id, + component = result.component or meta.component, + image_id = result.image_id or meta.image_id, + xfer_id = xfer_id, + request_id = rec.request_id or result.request_id, + }, + ts = payload.ts, + } +end + +local function is_terminal_job_state(state) + return state == 'succeeded' + or state == 'failed' + or state == 'cancelled' + or state == 'timed_out' + or state == 'discarded' +end + +local function transfer_bytes(progress, payload) + progress = type(progress) == 'table' and progress or {} + payload = type(payload) == 'table' and payload or {} + return payload.sent_bytes + or payload.received_bytes + or progress.sent_bytes + or progress.received_bytes + or progress.last_tx_next + or progress.requested_next + or progress.pending_next + or progress.last_rx_next +end + +local function transfer_total(progress, payload) + progress = type(progress) == 'table' and progress or {} + payload = type(payload) == 'table' and payload or {} + return payload.size or progress.size or progress.total_bytes +end + +local function augment_transfer(payload) + local out = copy_value(payload or {}) + local progress = type(out.progress) == 'table' and out.progress or {} + local bytes = transfer_bytes(progress, out) + local total = transfer_total(progress, out) + out.bytes_transferred = bytes + out.total_bytes = total + if type(bytes) == 'number' and type(total) == 'number' and total > 0 then + out.percent = math.floor((bytes * 10000 / total) + 0.5) / 100 + end + out.last_rx = { + type = progress.last_rx_type, + next = progress.last_rx_next, + at = progress.last_rx_at, + retry = progress.retry, + reason = progress.reason, + } + out.last_tx = { + type = progress.last_tx_type, + offset = progress.last_tx_offset, + next = progress.last_tx_next, + at = progress.last_tx_at, + } + out.timing = { + frame_queue_ms = progress.frame_queue_ms, + need_to_chunk_ms = progress.need_to_chunk_ms, + source_read_ms = progress.source_read_ms, + send_ms = progress.send_ms, + max_frame_queue_ms = out.max_frame_queue_ms or progress.max_frame_queue_ms, + max_need_to_chunk_ms = out.max_need_to_chunk_ms or progress.max_need_to_chunk_ms, + max_source_read_ms = out.max_source_read_ms or progress.max_source_read_ms, + max_send_ms = out.max_send_ms or progress.max_send_ms, + } + return out +end + +local function transfer_job_id(transfer) + local corr = type(transfer) == 'table' and type(transfer.correlation) == 'table' and transfer.correlation or nil + return corr and corr.job_id or nil +end + +local function transfer_is_active(transfer) + local st = transfer and (transfer.state or transfer.status) + return st == 'leased' or st == 'sending' or st == 'receiving' or st == 'staging' +end + +local function newest_job_first(a, b) + local av = type(a) == 'table' and (a.updated_seq or a.seq or 0) or 0 + local bv = type(b) == 'table' and (b.updated_seq or b.seq or 0) or 0 + if av == bv then return tostring(a and a.job_id or '') > tostring(b and b.job_id or '') end + return av > bv +end + +local function newest_transfer_first(a, b) + local av = type(a) == 'table' and (a.ts or 0) or 0 + local bv = type(b) == 'table' and (b.ts or 0) or 0 + if av == bv then return tostring(a and a.xfer_id or '') > tostring(b and b.xfer_id or '') end + return av > bv +end + +--- HTTP-friendly update status, including live Fabric transfer progress. +--- +--- This is deliberately a projection over retained state rather than another +--- update-service RPC. It lets curl-based harnesses poll one endpoint while an +--- upload/start request is already in flight. +function M.update_status(snapshot) + local items = snapshot and snapshot.items or {} + local summary = nil + local jobs = {} + local timelines = {} + local transfers = {} + local transfers_by_xfer = {} + local active_job = nil + local active_transfer = nil + + local function add_transfer(raw) + if type(raw) ~= 'table' then return end + local transfer = augment_transfer(raw) + local key = transfer.xfer_id or transfer.request_id or tostring(#transfers + 1) + local prev = transfers_by_xfer[key] + if prev == nil or newest_transfer_first(transfer, prev) then + transfers_by_xfer[key] = transfer + end + end + + for _, msg in pairs(items) do + local topic = msg and msg.topic or nil + local payload = payload_of(msg) + if payload ~= nil then + if topic_has_prefix(topic, { 'state', 'update', 'summary' }) then + summary = copy_value(payload) + elseif is_update_job_topic(topic) then + local job = copy_value(payload) + jobs[#jobs + 1] = job + if not is_terminal_job_state(job.state) then + if active_job == nil or newest_job_first(job, active_job) then active_job = job end + end + elseif is_update_timeline_topic(topic) then + timelines[topic_at(topic, 4)] = copy_value(payload) + elseif is_fabric_transfer_topic(topic) then + add_transfer(payload) + elseif is_fabric_transfer_component_topic(topic) then + add_transfer(transfer_payload_from_component(payload)) + end + end + end + + for _, transfer in pairs(transfers_by_xfer) do + transfers[#transfers + 1] = transfer + end + + table.sort(jobs, newest_job_first) + table.sort(transfers, newest_transfer_first) + for _, transfer in ipairs(transfers) do + if transfer_is_active(transfer) then + if active_transfer == nil or newest_transfer_first(transfer, active_transfer) then active_transfer = transfer end + end + end + + local transfers_by_job = {} + for _, transfer in ipairs(transfers) do + local job_id = transfer_job_id(transfer) + if type(job_id) == 'string' and job_id ~= '' and transfers_by_job[job_id] == nil then + transfers_by_job[job_id] = transfer + end + end + + for _, job in ipairs(jobs) do + if type(job) == 'table' then + job.timeline = timelines[job.job_id] + job.transfer_status = transfers_by_job[job.job_id] + end + end + + if active_job ~= nil and type(active_job) == 'table' then + active_job.timeline = timelines[active_job.job_id] + active_job.transfer_status = transfers_by_job[active_job.job_id] + end + + return { + version = snapshot and snapshot.version or 0, + update = summary, + active_job = active_job, + active_transfer = active_transfer, + jobs = jobs, + transfers = transfers, + } +end + +function M.summary(snapshot, sessions_count, extra) + local items = snapshot and snapshot.items or {} + local services = 0 + for _, msg in pairs(items) do + if topic_has_prefix(msg.topic, { 'svc' }) then services = services + 1 end + end + local out = { + version = snapshot and snapshot.version or 0, + services = services, + sessions = sessions_count or 0, + closed = snapshot and snapshot.closed or false, + reason = snapshot and snapshot.reason or nil, + } + for k, v in pairs(extra or {}) do out[k] = v end + return out +end + +return M diff --git a/src/services/ui/read_model.lua b/src/services/ui/read_model.lua new file mode 100644 index 00000000..d6c6ba27 --- /dev/null +++ b/src/services/ui/read_model.lua @@ -0,0 +1,181 @@ +-- services/ui/read_model.lua +-- +-- Read-model component assembly. +-- +-- Fabric-grade split: +-- * read_model_store.lua : pure retained-state projection/model +-- * read_model_watches.lua : local watch owner and fanout boundary +-- +-- This module wires retained bus feeds into the store/watch owner. It remains as +-- the public read-model entry point for UI callers. + +local fibers = require 'fibers' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local local_model = require 'services.ui.local_model' +local store_mod = require 'services.ui.read_model_store' +local watches_mod = require 'services.ui.read_model_watches' +local topics = require 'services.ui.topics' + +local M = {} + + +local function event_topic(ev) + if type(ev) ~= 'table' then return nil end + return ev.topic +end + +local function topic_is_excluded(topic, excluded_patterns, opts) + opts = opts or {} + if type(topic) ~= 'table' then return false end + for _, pattern in ipairs(excluded_patterns or {}) do + if store_mod.match_topic(pattern, topic, opts.s_wild or '+', opts.m_wild or '#') then + return true + end + end + return false +end + +local function should_ingest_event(ev, opts) + opts = opts or {} + if type(ev) ~= 'table' then return true end + if ev.op == 'replay_done' then return true end + local topic = event_topic(ev) + local excluded = opts.exclude_patterns or opts.excluded_patterns or topics.default_excluded_retained_patterns() + return not topic_is_excluded(topic, excluded, opts) +end + +local function start_feed_owner(scope, target, conn, pattern, opts) + local watch, err = bus_cleanup.watch_retained(conn, pattern, { + replay = true, + queue_len = opts.feed_queue_len or 64, + full = opts.feed_full or 'reject_newest', + }) + if not watch then + return nil, err or 'watch_retained failed' + end + + scope:finally(function () + bus_cleanup.unwatch_retained(conn, watch) + end) + + local ok, spawn_err = fibers.spawn(function () + while true do + local ev, recv_err = fibers.perform(watch:recv_op()) + if ev == nil then + error(recv_err or 'read model retained feed closed', 0) + end + if should_ingest_event(ev, opts) then + local changed, _, _, ingest_err = target:ingest(ev) + if changed == nil then + error(ingest_err or 'read model ingest failed', 0) + end + end + end + end) + if not ok then + bus_cleanup.unwatch_retained(conn, watch) + return nil, spawn_err + end + + return true, nil +end + +local function start_live_event_owner(scope, target, conn, pattern, opts) + local sub, err = bus_cleanup.subscribe(conn, pattern, { + queue_len = opts.event_queue_len or 256, + full = opts.event_full or 'drop_oldest', + }) + if not sub then + return nil, err or 'subscribe failed' + end + + scope:finally(function () + bus_cleanup.unsubscribe(conn, sub) + end) + + local ok, spawn_err = fibers.spawn(function () + while true do + local msg, recv_err = fibers.perform(sub:recv_op()) + if msg == nil then + error(recv_err or 'read model live event feed closed', 0) + end + if local_model.allowed_event(msg.topic) then + local notified, notify_err = target:notify('set', msg) + if notified == nil then + error(notify_err or 'read model live event notify failed', 0) + end + end + end + end) + if not ok then + bus_cleanup.unsubscribe(conn, sub) + return nil, spawn_err + end + + return true, nil +end + +--- Create a pure retained-state projection store. +function M.new(opts) + return store_mod.new(opts) +end + +--- Create a local watch owner for an existing store. +function M.new_watches(store, opts) + return watches_mod.new(store, opts) +end + +--- Start retained-feed owners inside an already-created read-model scope. +--- +--- opts.model is the pure projection store. +--- opts.watch_owner is the local watch fanout owner. +--- +--- Returns the store and watch owner immediately; feed owners run as child +--- fibres in the same scope. +function M.start(scope, conn, opts) + opts = opts or {} + local store = opts.model or M.new(opts) + local watch_owner = opts.watch_owner + if watch_owner == nil then + watch_owner = M.new_watches(store, opts) + end + + scope:finally(function (_, status, primary) + local reason = primary or status or 'read_model_closed' + watch_owner:terminate(reason) + store:terminate(reason) + end) + + if conn ~= nil then + local patterns = opts.patterns or topics.default_retained_patterns(opts) + for _, pattern in ipairs(patterns) do + local ok, err = start_feed_owner(scope, watch_owner, conn, pattern, opts) + if not ok then error(err or 'read_model feed start failed', 2) end + end + + local event_patterns = opts.event_patterns + if event_patterns == nil then event_patterns = local_model.LIVE_EVENT_PATTERNS end + if event_patterns ~= false then + for _, pattern in ipairs(event_patterns) do + local ok, err = start_live_event_owner(scope, watch_owner, conn, pattern, opts) + if not ok then error(err or 'read_model live event feed start failed', 2) end + end + end + end + + return store, watch_owner +end + +M.store = store_mod +M.watches = watches_mod +M.Store = store_mod.Store +M.WatchOwner = watches_mod.WatchOwner +M.Watch = watches_mod.Watch +M.match_topic = store_mod.match_topic +M._test = { + should_ingest_event = should_ingest_event, + start_live_event_owner = start_live_event_owner, + topic_is_excluded = topic_is_excluded, +} + +return M diff --git a/src/services/ui/read_model_store.lua b/src/services/ui/read_model_store.lua new file mode 100644 index 00000000..6b02ed13 --- /dev/null +++ b/src/services/ui/read_model_store.lua @@ -0,0 +1,381 @@ +-- services/ui/read_model_store.lua +-- +-- Pure retained-state projection store for UI. +-- +-- Contract: +-- * stores retained messages by literal topic key +-- * snapshot(), items(), get(), and query() return copies +-- * set/delete/ingest are immediate and non-yielding +-- * changed_op(seen) returns a versioned snapshot, or a close reason +-- * terminate(reason) wakes observers +-- * no queue, mailbox, bus, publish, or service calls live here + +local op = require 'fibers.op' +local cond = require 'fibers.cond' +local topics = require 'services.ui.topics' +local tablex = require 'shared.table' +local topicx = require 'shared.topic' + +local M = {} + +local Store = {} +Store.__index = Store + +local copy_value = tablex.deep_copy +local deep_equal = tablex.deep_equal +local topic_copy = topicx.copy + +local function copy_msg(msg) + if not msg then return nil end + return { + topic = topic_copy(msg.topic), + payload = copy_value(msg.payload), + origin = copy_value(msg.origin), + } +end + +local function token_matches(pattern_tok, topic_tok, s_wild, m_wild) + if pattern_tok == s_wild then return true end + if pattern_tok == m_wild then return true end + return pattern_tok == topic_tok +end + +local function match_topic(pattern, topic, s_wild, m_wild) + pattern = pattern or {} + topic = topic or {} + local pi, ti = 1, 1 + while pi <= #pattern do + local p = pattern[pi] + if p == m_wild then + return true + end + if ti > #topic then return false end + if not token_matches(p, topic[ti], s_wild, m_wild) then + return false + end + pi = pi + 1 + ti = ti + 1 + end + return ti > #topic +end + +local function index_token_key(tok) + local ty = type(tok) + return ty .. ':' .. tostring(tok) +end + +local function first_literal_token(pattern, s_wild, m_wild) + local tok = pattern and pattern[1] or nil + if tok == nil or tok == s_wild or tok == m_wild then return nil end + return tok +end + +local function sorted_msgs_from_keys(self, keys, pattern) + local out = {} + for _, key in ipairs(keys) do + local msg = self._items[key] + if msg and match_topic(pattern, msg.topic, self._s_wild, self._m_wild) then + out[#out + 1] = copy_msg(msg) + end + end + table.sort(out, function(a, b) + return topics.topic_key(a.topic) < topics.topic_key(b.topic) + end) + return out +end + +local function assert_topic(topic, name, level) + if type(topic) ~= 'table' then + error((name or 'topic') .. ' must be a table', (level or 1) + 1) + end +end + +local function assert_seen(seen, level) + if type(seen) ~= 'number' or seen < 0 or seen % 1 ~= 0 then + error('read_model_store.changed_op: seen must be a non-negative integer', (level or 1) + 1) + end +end + +function Store:version() + return self._version +end + +function Store:is_closed() + return self._closed +end + +function Store:why() + return self._close_reason +end + +function Store:_bump() + self._version = self._version + 1 + self._changed:signal() + self._changed = cond.new() + return self._version +end + +function Store:_index_add(key, topic) + local first = topic and topic[1] + if first == nil then return end + local ikey = index_token_key(first) + local bucket = self._index_first[ikey] + if not bucket then + bucket = {} + self._index_first[ikey] = bucket + end + bucket[key] = true +end + +function Store:_index_remove(key, topic) + local first = topic and topic[1] + if first == nil then return end + local ikey = index_token_key(first) + local bucket = self._index_first[ikey] + if not bucket then return end + bucket[key] = nil + if next(bucket) == nil then self._index_first[ikey] = nil end +end + +function Store:_candidate_keys(pattern) + local literal = first_literal_token(pattern, self._s_wild, self._m_wild) + local out = {} + if literal ~= nil then + local bucket = self._index_first[index_token_key(literal)] or {} + for key in pairs(bucket) do out[#out + 1] = key end + else + for key in pairs(self._items) do out[#out + 1] = key end + end + table.sort(out) + return out +end + +--- Set a retained item. +--- +--- Return shapes: +--- true, msg, version changed +--- false, msg, version unchanged +--- nil, reason closed +function Store:set(topic, payload, origin) + assert_topic(topic, 'read_model_store:set topic', 2) + if self._closed then return nil, tostring(self._close_reason or 'closed') end + + local key = topics.topic_key(topic) + local old = self._items[key] + local msg = { + topic = topic_copy(topic), + payload = copy_value(payload), + origin = copy_value(origin), + } + + local same = false + if old and topics.topic_key(old.topic) == key then + same = deep_equal(old.payload, msg.payload) and deep_equal(old.origin, msg.origin) + end + + if old then self:_index_remove(key, old.topic) end + self._items[key] = msg + self:_index_add(key, msg.topic) + if same then + return false, copy_msg(msg), self._version + end + + local version = self:_bump() + return true, copy_msg(msg), version +end + +--- Delete a retained item. +--- +--- Return shapes: +--- true, msg, version changed +--- false, nil, version unchanged / absent +--- nil, reason closed +function Store:delete(topic, origin) + assert_topic(topic, 'read_model_store:delete topic', 2) + if self._closed then return nil, tostring(self._close_reason or 'closed') end + + local key = topics.topic_key(topic) + local old = self._items[key] + if old == nil then return false, nil, self._version end + + self._items[key] = nil + self:_index_remove(key, old.topic) + local msg = { + topic = topic_copy(topic), + payload = nil, + origin = copy_value(origin), + } + local version = self:_bump() + return true, copy_msg(msg), version +end + +--- Ingest a retained lifecycle event. +--- +--- Return shapes: +--- changed, msg, op_name, version_or_reason +--- nil, nil, nil, reason invalid/closed +function Store:ingest(ev) + if self._closed then return nil, nil, nil, tostring(self._close_reason or 'closed') end + if type(ev) ~= 'table' then return nil, nil, nil, 'invalid retained event' end + + if ev.op == 'retain' then + local changed, msg, version_or_reason = self:set(ev.topic, ev.payload, ev.origin) + if changed == nil then return nil, nil, nil, msg end + return changed, msg, 'set', version_or_reason + elseif ev.op == 'unretain' then + local changed, msg, version_or_reason = self:delete(ev.topic, ev.origin) + if changed == nil then return nil, nil, nil, msg end + return changed, msg, 'delete', version_or_reason + elseif ev.op == 'replay_done' then + return false, nil, 'replay_done', self._version + elseif ev.topic ~= nil and ev.payload ~= nil then + local changed, msg, version_or_reason = self:set(ev.topic, ev.payload, ev.origin) + if changed == nil then return nil, nil, nil, msg end + return changed, msg, 'set', version_or_reason + end + + return nil, nil, nil, 'unknown retained event' +end + +function Store:snapshot() + local out = {} + for key, msg in pairs(self._items) do + out[key] = copy_msg(msg) + end + return { + version = self._version, + items = out, + closed = self._closed, + reason = self._close_reason, + } +end + +function Store:items() + local out = {} + for _, msg in pairs(self._items) do out[#out + 1] = copy_msg(msg) end + table.sort(out, function(a, b) + return topics.topic_key(a.topic) < topics.topic_key(b.topic) + end) + return out +end + +function Store:get(topic) + assert_topic(topic, 'read_model_store:get topic', 2) + return copy_msg(self._items[topics.topic_key(topic)]) +end + +function Store:query(pattern) + return sorted_msgs_from_keys(self, self:_candidate_keys(pattern), pattern) +end + +local function each_candidate_key_unsorted(self, pattern, fn) + local literal = first_literal_token(pattern, self._s_wild, self._m_wild) + if literal ~= nil then + local bucket = self._index_first[index_token_key(literal)] or {} + for key in pairs(bucket) do + if fn(key) == false then return false end + end + return true + end + + for key in pairs(self._items) do + if fn(key) == false then return false end + end + return true +end + +function Store:query_limited(pattern, limit) + if limit ~= false and limit ~= nil then + if type(limit) ~= 'number' or limit < 0 or limit % 1 ~= 0 then + error('read_model_store:query_limited limit must be a non-negative integer, nil, or false', 2) + end + end + + if limit == false or limit == nil then + return self:query(pattern), nil + end + + local out = {} + local exceeded = false + -- Work is bounded by the first limit + 1 matching retained entries. We do + -- not materialise or sort a broader replay before reporting overflow. + each_candidate_key_unsorted(self, pattern, function (key) + local msg = self._items[key] + if msg and match_topic(pattern, msg.topic, self._s_wild, self._m_wild) then + if #out >= limit then + exceeded = true + return false + end + out[#out + 1] = copy_msg(msg) + end + return true + end) + + if exceeded then + return nil, 'query_limit_exceeded' + end + + table.sort(out, function(a, b) + return topics.topic_key(a.topic) < topics.topic_key(b.topic) + end) + return out, nil +end + +--- Wait until version > seen, or until closed. +--- +--- Return shapes: +--- changed: version, snapshot, nil +--- closed : nil, nil, reason +function Store:changed_op(seen) + assert_seen(seen, 2) + + return op.guard(function () + if self._closed then + return op.always(nil, nil, tostring(self._close_reason or 'closed')) + end + if self._version > seen then + return op.always(self._version, self:snapshot(), nil) + end + local c = self._changed + return c:wait_op():wrap(function () + if self._closed then + return nil, nil, tostring(self._close_reason or 'closed') + end + return self._version, self:snapshot(), nil + end) + end) +end + +function Store:terminate(reason) + if self._closed then return true end + self._closed = true + self._close_reason = reason or 'closed' + self:_bump() + return true +end + +function M.new(opts) + opts = opts or {} + return setmetatable({ + _items = {}, + _index_first = {}, -- first-token index used to bound watch replay candidate scans + _version = 0, + _changed = cond.new(), + _closed = false, + _close_reason = nil, + _s_wild = opts.s_wild or '+', + _m_wild = opts.m_wild or '#', + }, Store) +end + +M.Store = Store +M.copy_value = copy_value +M.topic_copy = topic_copy +M.copy_msg = copy_msg +M.match_topic = match_topic +M._test = { + candidate_count = function (store, pattern) return #store:_candidate_keys(pattern) end, + index_token_key = index_token_key, +} + +return M diff --git a/src/services/ui/read_model_watches.lua b/src/services/ui/read_model_watches.lua new file mode 100644 index 00000000..920b9567 --- /dev/null +++ b/src/services/ui/read_model_watches.lua @@ -0,0 +1,218 @@ +-- services/ui/read_model_watches.lua +-- +-- Local UI read-model watch owner and fanout boundary. +-- +-- This module deliberately owns bounded watch queues and replay/fanout policy. +-- The projection store remains pure in read_model_store.lua. +-- Replay is bounded by default. Callers may set max_replay/watch_max_replay +-- explicitly; max_replay=false opts out deliberately for tests/special cases. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local queue = require 'devicecode.support.queue' +local store_mod = require 'services.ui.read_model_store' + +local M = {} + +local DEFAULT_MAX_REPLAY = 32 + + +local function resolve_max_replay(opts) + if opts.max_replay ~= nil then return opts.max_replay end + if opts.watch_max_replay ~= nil then return opts.watch_max_replay end + if opts.replay_limit ~= nil then return opts.replay_limit end + return DEFAULT_MAX_REPLAY +end + +local WatchOwner = {} +WatchOwner.__index = WatchOwner + +local Watch = {} +Watch.__index = Watch + +local function copy_value(v) + return store_mod.copy_value(v) +end + +local function topic_copy(topic) + return store_mod.topic_copy(topic) +end + +local function msg_event(op_name, msg) + return { + kind = 'read_model_event', + op = op_name, + topic = msg and topic_copy(msg.topic) or nil, + payload = msg and copy_value(msg.payload) or nil, + origin = msg and copy_value(msg.origin) or nil, + } +end + +local function terminate_watch(watch, reason) + if watch._closed then return true end + watch._closed = true + watch._reason = reason or 'closed' + if watch._tx then watch._tx:close(watch._reason) end + if watch._owner then + watch._owner._watches[watch] = nil + end + watch._owner = nil + return true +end + +local function terminate_all(owner, reason) + local pending = {} + for watch in pairs(owner._watches) do pending[#pending + 1] = watch end + for _, watch in ipairs(pending) do terminate_watch(watch, reason) end +end + +function WatchOwner:store() + return self._store +end + +function WatchOwner:watch_count() + local n = 0 + for _ in pairs(self._watches) do n = n + 1 end + return n +end + +function WatchOwner:notify(op_name, msg) + if self._closed then return nil, tostring(self._reason or 'closed') end + if not msg or type(msg.topic) ~= 'table' then return false, 'no_topic' end + + local targets = {} + for watch in pairs(self._watches) do + if store_mod.match_topic(watch._pattern, msg.topic, self._s_wild, self._m_wild) then + targets[#targets + 1] = watch + end + end + + local ev = msg_event(op_name, msg) + for _, watch in ipairs(targets) do + if watch._owner == self and not watch._closed then + local ok = queue.try_admit_now(watch._tx, copy_value(ev)) + if ok ~= true then + terminate_watch(watch, 'watch_overflow') + end + end + end + return true, nil +end + +function WatchOwner:set(topic, payload, origin) + local changed, msg, version_or_reason = self._store:set(topic, payload, origin) + if changed == true then self:notify('set', msg) end + return changed, msg, version_or_reason +end + +function WatchOwner:delete(topic, origin) + local changed, msg, version_or_reason = self._store:delete(topic, origin) + if changed == true then self:notify('delete', msg) end + return changed, msg, version_or_reason +end + +function WatchOwner:ingest(ev) + local changed, msg, op_name, version_or_reason = self._store:ingest(ev) + if changed == true and (op_name == 'set' or op_name == 'delete') then + self:notify(op_name, msg) + end + return changed, msg, op_name, version_or_reason +end + +function WatchOwner:watch_open(pattern, opts) + opts = opts or {} + if self._closed then return nil, tostring(self._reason or 'closed') end + if self._store:is_closed() then return nil, tostring(self._store:why() or 'closed') end + + local qlen = opts.queue_len or self._default_queue_len + local full = opts.full or self._default_full + local tx, rx = mailbox.new(qlen, { full = full }) + local watch = setmetatable({ + _owner = self, + _pattern = topic_copy(pattern), + _tx = tx, + _rx = rx, + _closed = false, + _reason = nil, + }, Watch) + + if opts.replay ~= false then + local max_replay = opts.max_replay + if max_replay == nil then max_replay = self._max_replay end + local replay, qerr = self._store:query_limited(pattern, max_replay) + if not replay then + terminate_watch(watch, 'watch_replay_overflow') + return nil, (qerr == 'query_limit_exceeded') and 'watch_replay_overflow' or qerr + end + for _, msg in ipairs(replay) do + local ok = queue.try_admit_now(tx, msg_event('set', msg)) + if ok ~= true then + terminate_watch(watch, 'watch_replay_overflow') + return nil, 'watch_replay_overflow' + end + end + end + + self._watches[watch] = true + return watch, nil +end + +function WatchOwner:terminate(reason) + if self._closed then return true end + self._closed = true + self._reason = reason or 'closed' + terminate_all(self, self._reason) + return true +end + +function WatchOwner:why() + return self._reason +end + +function Watch:recv_op() + return self._rx:recv_op():wrap(function (ev) + if ev == nil then + return nil, tostring(self._reason or self._rx:why() or 'closed') + end + return ev, nil + end) +end + +function Watch:recv() + return fibers.perform(self:recv_op()) +end + +function Watch:terminate(reason) + return terminate_watch(self, reason or 'closed') +end + +function Watch:why() + return self._reason or self._rx:why() +end + +function Watch:pattern() + return topic_copy(self._pattern) +end + +function M.new(store, opts) + if store == nil then error('read_model_watches.new: store required', 2) end + opts = opts or {} + return setmetatable({ + _store = store, + _watches = {}, + _closed = false, + _reason = nil, + _s_wild = opts.s_wild or '+', + _m_wild = opts.m_wild or '#', + _default_queue_len = opts.queue_len or opts.watch_queue_len or 32, + _default_full = opts.full or opts.watch_full or 'reject_newest', + _max_replay = resolve_max_replay(opts), + }, WatchOwner) +end + +M.DEFAULT_MAX_REPLAY = DEFAULT_MAX_REPLAY +M.WatchOwner = WatchOwner +M.Watch = Watch +M.msg_event = msg_event + +return M diff --git a/src/services/ui/service.lua b/src/services/ui/service.lua new file mode 100644 index 00000000..cbb7a2c3 --- /dev/null +++ b/src/services/ui/service.lua @@ -0,0 +1,891 @@ +-- services/ui/service.lua +-- +-- Top-level UI service coordinator. It owns service-level status and starts +-- long-lived scoped components. It does not own HTTP request work, +-- upstream calls, or upload streams. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local service_base = require 'devicecode.service_base' +local scoped_work = require 'devicecode.support.scoped_work' +local sleep = require 'fibers.sleep' +local runtime = require 'fibers.runtime' +local queue = require 'devicecode.support.queue' +local read_model = require 'services.ui.read_model' +local queries = require 'services.ui.queries' +local sessions_mod = require 'services.ui.sessions' +local auth_mod = require 'services.ui.auth' +local topics = require 'services.ui.topics' +local supervision = require 'services.ui.supervision' +local config_watch = require 'devicecode.support.config_watch' +local config_mod = require 'services.ui.config' +local service_events = require 'devicecode.support.service_events' +local dep_slot = require 'devicecode.support.dependency_slot' +local dep_failure = require 'devicecode.support.dependency_failure' +local tablex = require 'shared.table' + +local M = {} + +local shallow_copy = tablex.shallow_copy + +local function dependency_snapshot(state) + return dep_slot.snapshot(state, 'http_deps') +end + +local function sorted_keys(t) + local keys = {} + for k in pairs(t or {}) do keys[#keys + 1] = k end + table.sort(keys, function (a, b) return tostring(a) < tostring(b) end) + return keys +end + +local function stable_status_value(v, key) + if key == 'at' or key == 'ts' or key == 'run_id' or key == 'updated_at' then return '' end + if type(v) ~= 'table' then return tostring(v) end + local parts = { '{' } + for _, k in ipairs(sorted_keys(v)) do + if k ~= 'at' and k ~= 'ts' and k ~= 'run_id' and k ~= 'updated_at' then + parts[#parts + 1] = tostring(k) + parts[#parts + 1] = '=' + parts[#parts + 1] = stable_status_value(v[k], k) + parts[#parts + 1] = ';' + end + end + parts[#parts + 1] = '}' + return table.concat(parts) +end + +local function lifecycle_status_key(service_state, payload) + return stable_status_value({ state = service_state, payload = payload }, nil) +end + +local function ui_observability(state) + local cfg = state and state.config and state.config.observability or nil + return cfg or config_mod.DEFAULTS.observability +end + +local function should_publish_lifecycle(state, key) + state.lifecycle_obs = state.lifecycle_obs or {} + local obs = state.lifecycle_obs + if obs.status_key ~= key then return true end + local interval = ui_observability(state).status_interval_s + if interval == false then return false end + interval = tonumber(interval) or 30 + if interval <= 0 then return false end + return (runtime.now() - (obs.last_status_emit_at or 0)) >= interval +end + +local function note_lifecycle_published(state, key) + state.lifecycle_obs = state.lifecycle_obs or {} + state.lifecycle_obs.status_key = key + state.lifecycle_obs.last_status_emit_at = runtime.now() +end + + +local function component_summary(components) + local out = {} + for name, c in pairs(components or {}) do + out[name] = { + status = c.status, + primary = c.primary, + reason = c.reason, + } + end + return out +end + +local function build_summary(state) + local model_snapshot = state.model and state.model:snapshot() or nil + return queries.summary(model_snapshot, + state.sessions and state.sessions:count() or 0, + { + active_requests = state.active_requests, + rejected_requests = state.rejected_requests or 0, + last_rejection_reason = state.last_rejection_reason, + read_model_status = state.read_model_status, + listener_status = state.listener_status, + config_status = state.config_status, + config_generation = state.config_generation, + service_status = state.service_status, + components = component_summary(state.components), + dependencies = dependency_snapshot(state), + last_error = state.last_error, + } + ) +end + +local function publish_summary(state) + -- UI summaries are presentation projections. They are intentionally not + -- retained back to the bus because the read model consumes retained state/#. + -- Publishing derived UI state there creates a self-ingesting projection loop. + return build_summary(state) +end +local function record_cleanup_error(state, kind, err) + local rec = { + kind = kind or 'cleanup_error', + err = tostring(err or 'cleanup failed'), + } + -- Cleanup errors are diagnostic state, not history. Keep the last error only + -- to avoid accumulating records in long-running services. + state.last_cleanup_error = rec + + return rec +end + +local function component_identity(component, generation) + return { + kind = 'ui_component_done', + component = component, + generation = generation, + } +end + +local function record_component_started(state, component, generation) + state.components = state.components or {} + local c = state.components[component] or {} + if generation ~= nil and c.generation ~= nil and c.generation ~= generation then return false, 'stale_component_start' end + c.status = 'running' + if generation ~= nil then c.generation = generation end + c.reason = nil + c.primary = nil + c.outcome = nil + state.components[component] = c + if component == 'read_model' then + state.read_model_status = 'running' + elseif component == 'http_listener' then + state.listener_status = 'running' + end +end + +local function record_component_done(state, ev) + state.components = state.components or {} + local name = ev.component + if type(name) ~= 'string' or name == '' then + return false, 'missing_component' + end + local c = state.components[name] or {} + if ev.generation ~= nil and c.generation ~= nil and ev.generation ~= c.generation then + return false, 'stale_component_completion' + end + if c.status == 'ok' or c.status == 'failed' or c.status == 'cancelled' then + return false, 'stale_component_completion' + end + c.status = ev.status + c.outcome = ev + if ev.status == 'ok' then + c.result = ev.result + c.reason = type(ev.result) == 'table' and (ev.result.reason or ev.result.status) or nil + else + c.primary = ev.primary + c.reason = ev.primary + end + state.components[name] = c + + if name == 'read_model' then + state.read_model_status = ev.status + elseif name == 'http_listener' then + state.listener_status = ev.status + end + return true, nil +end + +local function apply_service_policy(state, ev) + local decision = supervision.classify_service_component_done(state, ev) + local action = decision.action or 'continue' + if action == 'continue' then + return { publish = true } + elseif action == 'degrade_service' then + state.service_status = 'degraded' + state.last_error = decision.reason or ev.primary + return { publish = true } + elseif action == 'fail_service' then + state.service_status = 'failed' + state.last_error = decision.reason or ev.primary or 'ui component failed' + return { publish = true, fail = state.last_error } + elseif action == 'cancel_service' then + state.service_status = 'stopping' + state.last_error = decision.reason or ev.primary or 'ui component cancelled' + if state.scope and state.scope.cancel then state.scope:cancel(state.last_error) end + return { publish = true } + end + error('ui.service: unknown supervision action: ' .. tostring(action), 0) +end + +local function start_component(state, component, run, generation) + state.components[component] = { status = 'starting', generation = generation } + local handle, err = scoped_work.start({ + lifetime_scope = state.scope, + reaper_scope = state.scope, + report_scope = state.scope, + identity = component_identity(component, generation), + run = function (scope) + return run(scope) + end, + report = service_events.reporter( + service_events.port(state.done_tx, { + service_id = state.service_id or 'ui', + source = 'ui_component', + source_id = component, + component = component, + generation = generation, + }, { label = 'ui_component_done_report_failed' }), + 'ui_component_done_report_failed' + ), + }) + if not handle then error(err or ('failed to start ' .. component), 0) end + return handle +end + + + +local function run_session_pruner(_, sessions, interval) + while true do + fibers.perform(sleep.sleep_op(interval)) + sessions:prune() + end +end + +local start_configured_listener + +local function listener_config_key(cfg) + if not cfg or not cfg.enabled or not cfg.http or cfg.http.enabled == false then return 'disabled' end + local h = cfg.http + return table.concat({ + tostring(h.cap_id or 'main'), + tostring(h.host or ''), + tostring(h.port or ''), + tostring(h.path or ''), + tostring(h.tls or false), + tostring(h.max_accept_queue or ''), + }, '|') +end + + +local function listener_http_cap_id(cfg) + local h = cfg and cfg.http or nil + return (type(h) == 'table' and h.cap_id) or 'main' +end + +local function listener_needs_http_dependency(state, cfg) + local params = state.params or {} + if params.listener ~= nil or params.obtain_listener_op ~= nil then return false end + if not cfg or cfg.enabled == false then return false end + if not cfg.http or cfg.http.enabled == false then return false end + return true +end + +local function terminate_http_deps(state, reason) + dep_slot.terminate(state, 'http_deps', reason or 'ui_http_dependency_closed') + state.http_dep_cap_id = nil + return true +end + +local function ensure_http_dependency(state, cfg) + if not listener_needs_http_dependency(state, cfg) then + terminate_http_deps(state, 'ui_http_dependency_not_required') + return true, nil + end + + local cap_id = listener_http_cap_id(cfg) + if state.http_deps and state.http_dep_cap_id == cap_id then return true, nil end + local ok, err = dep_slot.replace(state, 'http_deps', state.conn, { + { key = 'http', class = 'http', id = cap_id, required = true }, + }, { + replace_reason = 'ui_http_dependency_replaced', + changed_kind = 'http_dependency_changed', + closed_kind = 'http_dependency_closed', + queue_len = (state.params and state.params.dependency_queue_len) or 8, + full = 'drop_oldest', + }) + if ok ~= true then return nil, err or 'http_dependency_open_failed' end + state.http_dep_cap_id = cap_id + return true, nil +end +local function http_dependency_available(state) + return state.http_deps == nil or dep_slot.available(state, 'http_deps', 'http') +end + +local function set_listener_waiting_for_http(state, reason) + state.listener_status = 'waiting_for_http' + state.last_error = reason or 'http_unavailable' + state.components = state.components or {} + state.components.http_listener = { + status = 'waiting_for_dependency', + reason = state.last_error, + generation = state.listener_generation, + } + return true +end + +local function maybe_start_pending_listener(state, reason) + if state.listener_handle ~= nil then return true, nil end + local cfg = state.pending_listener_cfg + if cfg == nil then return true, nil end + if not http_dependency_available(state) then + set_listener_waiting_for_http(state, reason or 'http_unavailable') + return true, nil + end + state.pending_listener_cfg = nil + return start_configured_listener(state, cfg, state.config_generation) +end + +local function listener_required(state) + local cfg = state and state.config or nil + if not cfg or cfg.enabled == false then return false end + return listener_config_key(cfg) ~= 'disabled' +end + +local function lifecycle_readiness(state) + local service_state = state.service_status or 'running' + if service_state == 'disabled' then return 'disabled', false, 'ui_disabled' end + if service_state == 'failed' or service_state == 'degraded' or service_state == 'stopping' then + return service_state, false, state.last_error or service_state + end + if state.config_status ~= 'ok' then return service_state, false, 'waiting_for_config' end + if state.read_model_status ~= 'running' then return service_state, false, 'read_model_not_running' end + if listener_required(state) and state.listener_status ~= 'running' then + if state.listener_status == 'waiting_for_http' then + return service_state, false, state.last_error or 'http_unavailable' + end + return service_state, false, 'http_listener_not_running' + end + return service_state, true, nil +end + + +local function log_ui_summary(state, reason) + if not state.svc then return end + local sessions = state.sessions and state.sessions:count() or 0 + local summary = string.format('ui summary ready=%s listener=%s read_model=%s sessions=%d active_requests=%d', + tostring(state._last_operator_ready == true), tostring(state.listener_status), tostring(state.read_model_status), sessions, tonumber(state.active_requests) or 0) + local tnow = fibers.now() + if state._operator_summary_key == summary and (tnow - (state._operator_summary_at or 0)) < 600 then return end + state._operator_summary_key = summary + state._operator_summary_at = tnow + state.svc:info('ui_summary', { summary = summary, reason = reason, sessions = sessions, listener_status = state.listener_status, read_model_status = state.read_model_status }) +end + +local function update_lifecycle(state) + local lifecycle = state.lifecycle + if not lifecycle then return true, nil end + local service_state, ready, reason = lifecycle_readiness(state) + local payload = { + ready = ready == true, + reason = reason, + read_model_status = state.read_model_status, + listener_status = state.listener_status, + config_status = state.config_status, + config_generation = state.config_generation, + dependencies = dependency_snapshot(state), + } + local key = lifecycle_status_key(service_state, payload) + if not should_publish_lifecycle(state, key) then return true, nil end + note_lifecycle_published(state, key) + if state.svc and ready == true and state._last_operator_ready ~= true then + state.svc:info('ui_ready', { + summary = string.format('ui ready listener=%s read_model=%s', tostring(state.listener_status), tostring(state.read_model_status)), + listener_status = state.listener_status, + read_model_status = state.read_model_status, + config_generation = state.config_generation, + }) + state._last_operator_ready = true + elseif ready ~= true then + state._last_operator_ready = false + end + if service_state == 'disabled' then return lifecycle:status('disabled', payload) end + if service_state == 'degraded' then return lifecycle:degraded(payload) end + if service_state == 'failed' then return lifecycle:failed(reason or 'ui_failed', payload) end + if service_state == 'stopped' then return lifecycle:stopped(payload) end + return lifecycle:running(payload) +end + + +local function schedule_lifecycle_publish(state, reason) + state.lifecycle_publish_pending = true + state.lifecycle_publish_reason = reason or state.lifecycle_publish_reason + local delay = ui_observability(state).coalesce_status_s + if delay == false then delay = 0.05 end + delay = tonumber(delay) or 0.05 + if delay < 0 then delay = 0 end + local due = runtime.now() + delay + if state.lifecycle_publish_due_at == nil or due < state.lifecycle_publish_due_at then + state.lifecycle_publish_due_at = due + end + return true +end + +local function clear_lifecycle_publish_due(state) + state.lifecycle_publish_pending = false + state.lifecycle_publish_due_at = nil + state.lifecycle_publish_reason = nil + return true +end + +local function build_listener_opts(state, cfg, listener_generation, listener_id) + local params = state.params or {} + local h = cfg.http or {} + local s = cfg.static or {} + local sse_cfg = cfg.sse or {} + local updates = cfg.updates or {} + local upload = updates.upload or {} + local commit = updates.commit or {} + + local listen = { + host = h.host, + port = h.port, + path = h.path, + tls = h.tls, + max_accept_queue = h.max_accept_queue, + } + + local update_opts = shallow_copy(params.update or {}) + update_opts.max_bytes = upload.max_bytes + update_opts.enabled = upload.enabled + update_opts.require_auth = upload.require_auth + update_opts.component = upload.component + update_opts.create_job = upload.create_job + update_opts.start_job = upload.start_job + update_opts.commit_require_auth = commit.require_auth + update_opts.connect = update_opts.connect or params.connect + update_opts.bus = update_opts.bus or params.bus + -- Public prototype update routes still make protected internal bus calls. + -- Borrow the UI service connection so those calls carry the UI service + -- principal rather than a nil HTTP-user principal. + update_opts.conn = update_opts.conn or state.conn + + return { + listener = params.listener, + conn = state.conn, + listen = listen, + cap_id = h.cap_id or 'main', + call_opts = params.http_call_opts, + bus = params.bus, + model = state.model, + watch_owner = state.watch_owner, + sessions = state.sessions, + auth = state.auth, + connect = params.connect, + events_port = service_events.port(state.done_tx, { + service_id = state.service_id or 'ui', + source = 'ui_http_listener', + source_id = listener_id or ('http_listener:' .. tostring(listener_generation or state.listener_generation or 0)), + listener_id = listener_id or ('http_listener:' .. tostring(listener_generation or state.listener_generation or 0)), + generation = listener_generation or state.listener_generation, + }, { label = 'ui_http_listener_event_report_failed' }), + root = s.root, + chunk_size = s.chunk_size, + encode_json = params.encode_json, + max_active_requests = h.max_active_requests, + overload_reason = params.overload_reason, + reject_overloaded_request_now = params.reject_overloaded_request_now, + sse = { + enabled = sse_cfg.enabled, + queue_len = sse_cfg.queue_len, + max_replay = sse_cfg.max_replay, + replay = sse_cfg.replay, + pattern = sse_cfg.pattern, + }, + update = update_opts, + } +end + +local function cancel_listener(state, reason) + if state.listener_handle and type(state.listener_handle.cancel) == 'function' then + state.listener_handle:cancel(reason or 'ui_config_changed') + end + state.listener_handle = nil + state.listener_config_key = nil + state.current_listener_cfg = nil + state.listener_generation = (state.listener_generation or 0) + 1 + state.components = state.components or {} + state.components.http_listener = { status = 'disabled', generation = state.listener_generation } + state.listener_status = 'disabled' + state.active_requests = 0 + return true +end + +function start_configured_listener(state, cfg, generation) + local key = listener_config_key(cfg) + if key == 'disabled' then + cancel_listener(state, 'ui_http_disabled') + return true + end + if state.listener_handle and state.listener_config_key == key then return true end + cancel_listener(state, 'ui_http_reconfigured') + state.listener_generation = generation or ((state.listener_generation or 0) + 1) + state.listener_config_key = key + state.listener_status = 'starting' + state.current_listener_cfg = cfg + local listener_generation = state.listener_generation + local listener_id = 'http_listener:' .. tostring(listener_generation) + state.listener_handle = start_component(state, 'http_listener', function (component_scope) + queue.assert_admit_required(state.done_tx, { + kind = 'component_started', + component = 'http_listener', + generation = listener_generation, + }, 'ui_http_listener_start_report_failed') + local listener_mod = require 'services.ui.http.listener' + listener_mod.run(component_scope, build_listener_opts(state, cfg, listener_generation, listener_id)) + return { status = 'stopped' } + end, listener_generation) + return true +end + +local function cancel_session_pruner(state, reason) + if state.session_pruner_handle and type(state.session_pruner_handle.cancel) == 'function' then + state.session_pruner_handle:cancel(reason or 'ui_session_pruner_reconfigured') + end + state.session_pruner_handle = nil + state.session_pruner_key = nil + state.session_pruner_generation = (state.session_pruner_generation or 0) + 1 + state.components = state.components or {} + state.components.session_pruner = { status = 'disabled', generation = state.session_pruner_generation } + return true +end + +local function start_configured_session_pruner(state, cfg, generation) + local sessions_cfg = cfg and cfg.sessions or {} + local interval = sessions_cfg.prune_interval + if interval == false or interval == nil then + cancel_session_pruner(state, 'ui_session_pruner_disabled') + return true + end + local key = tostring(interval) + if state.session_pruner_handle and state.session_pruner_key == key then return true end + cancel_session_pruner(state, 'ui_session_pruner_reconfigured') + state.session_pruner_generation = generation or ((state.session_pruner_generation or 0) + 1) + state.session_pruner_key = key + local pruner_generation = state.session_pruner_generation + state.session_pruner_handle = start_component(state, 'session_pruner', function (component_scope) + queue.assert_admit_required(state.done_tx, { + kind = 'component_started', + component = 'session_pruner', + generation = pruner_generation, + }, 'ui_session_pruner_start_report_failed') + return run_session_pruner(component_scope, state.sessions, interval) + end, pruner_generation) + return true +end + +local function apply_config(state, raw, generation) + local cfg, err = config_mod.normalise(raw or {}) + if not cfg then + state.config_status = 'invalid' + state.last_error = tostring(err or 'invalid_config') + state.service_status = 'degraded' + return { publish = true } + end + + state.config = cfg + state.config_generation = generation or ((state.config_generation or 0) + 1) + state.config_status = 'ok' + if state.service_status == 'starting' or state.service_status == 'degraded' then + state.service_status = 'running' + end + + if cfg.enabled == false then + cancel_listener(state, 'ui_disabled') + cancel_session_pruner(state, 'ui_disabled') + terminate_http_deps(state, 'ui_disabled') + state.pending_listener_cfg = nil + state.service_status = 'disabled' + return { publish = true } + end + + local ok_dep, dep_err = ensure_http_dependency(state, cfg) + if ok_dep ~= true then + state.service_status = 'degraded' + state.last_error = dep_err or 'http_dependency_open_failed' + return { publish = true } + end + + local key = listener_config_key(cfg) + if key == 'disabled' then + state.pending_listener_cfg = nil + cancel_listener(state, 'ui_http_disabled') + else + if not state.listener_handle or state.listener_config_key ~= key then + cancel_listener(state, 'ui_http_reconfigured') + state.pending_listener_cfg = cfg + maybe_start_pending_listener(state, 'config_changed') + end + end + start_configured_session_pruner(state, cfg, state.config_generation) + return { publish = true } +end + +local function cfg_event_from_watch(ev) + if ev == nil then return nil end + if ev.kind == 'config_closed' then + return { kind = 'config_closed', err = ev.err } + end + return { + kind = 'config_changed', + generation = ev.generation, + rev = ev.rev, + raw = ev.raw, + } +end + + +local function is_session_event(ev) + local k = ev and ev.kind + return k == 'session_created' + or k == 'session_touched' + or k == 'session_deleted' + or k == 'session_pruned' + or k == 'session_count_changed' +end + +local function reduce_event(state, ev) + if ev.kind == 'component_started' then + record_component_started(state, ev.component, ev.generation) + return { publish = true } + elseif ev.kind == 'config_changed' then + return apply_config(state, ev.raw, ev.generation) + elseif ev.kind == 'config_closed' then + state.config_status = 'closed' + state.last_error = ev.err + return { publish = true } + elseif ev.kind == 'ui_component_done' then + local accepted = record_component_done(state, ev) + if not accepted then return {} end + if ev.component == 'http_listener' and state.http_deps and dep_failure.is_no_route(ev.primary, ev.result, ev.report) then + state.http_deps:classify_call_failure('http', ev.result, ev.primary) + state.listener_handle = nil + state.listener_config_key = nil + state.pending_listener_cfg = state.current_listener_cfg or state.config + set_listener_waiting_for_http(state, 'http_route_missing') + return { publish = true } + end + return apply_service_policy(state, ev) + elseif ev.kind == 'http_dependency_changed' or ev.kind == 'http_dependency_closed' then + if ev.key == 'http' then + if ev.available == true then + state.last_error = nil + local ok, err = maybe_start_pending_listener(state, 'http_available') + if ok ~= true then state.service_status = 'degraded'; state.last_error = err or 'http_listener_start_failed' end + else + if state.pending_listener_cfg ~= nil and state.listener_handle == nil then + set_listener_waiting_for_http(state, ev.reason or 'http_unavailable') + end + end + end + return { coalesce_publish = true, reason = ev.kind } + elseif ev.kind == 'read_model_changed' then + state.model_seen = ev.version or state.model_seen + return { coalesce_publish = true, reason = 'read_model_changed' } + elseif ev.kind == 'session_changed' then + state.sessions_seen = ev.version or state.sessions_seen + state.last_session_event = ev.last_event or ev + return {} + elseif is_session_event(ev) then + state.last_session_event = ev + return { publish = true } + elseif ev.kind == 'http_request_done' then + if ev.generation ~= nil and state.listener_generation ~= nil and ev.generation ~= state.listener_generation then return {} end + if type(ev.active_requests) == 'number' then + state.active_requests = ev.active_requests + else + state.active_requests = math.max(0, (state.active_requests or 0) - 1) + end + return { publish = true } + elseif ev.kind == 'http_request_started' then + if ev.generation ~= nil and state.listener_generation ~= nil and ev.generation ~= state.listener_generation then return {} end + if type(ev.active_requests) == 'number' then + state.active_requests = ev.active_requests + else + state.active_requests = (state.active_requests or 0) + 1 + end + return { publish = true } + elseif ev.kind == 'http_request_rejected' then + if ev.generation ~= nil and state.listener_generation ~= nil and ev.generation ~= state.listener_generation then return {} end + state.rejected_requests = (state.rejected_requests or 0) + 1 + state.last_rejection_reason = ev.reason + return { publish = true } + elseif ev.kind == 'read_model_closed' then + state.read_model_status = 'closed' + return { publish = true } + elseif ev.kind == 'lifecycle_publish_due' then + clear_lifecycle_publish_due(state) + return { publish = true } + end + return {} +end + +local function next_event_op(state) + local arms = { + done = state.done_rx:recv_op(), + cfg = state.cfg_watch:recv_op():wrap(cfg_event_from_watch), + model = state.model:changed_op(state.model_seen):wrap(function (version, snapshot, err) + if version == nil then + return { kind = 'read_model_closed', err = err } + end + return { kind = 'read_model_changed', version = version, snapshot = snapshot } + end), + } + + if state.sessions and type(state.sessions.changed_op) == 'function' then + arms.sessions = state.sessions:changed_op(state.sessions_seen or 0):wrap(function (version, snapshot, err) + if version == nil then + return { kind = 'session_model_closed', err = err } + end + return { + kind = 'session_changed', + version = version, + snapshot = snapshot, + last_event = snapshot and snapshot.last_event or nil, + } + end) + end + + local src = dep_slot.event_source(state, 'http_deps', { name = 'http' }) + if src then arms.http_dependency = src.recv_op() end + if state.lifecycle_publish_pending == true then + arms.lifecycle_publish_due = sleep.sleep_until_op(state.lifecycle_publish_due_at or runtime.now()):wrap(function () + return { kind = 'lifecycle_publish_due' } + end) + end + + return fibers.named_choice(arms):wrap(function (_, ev) + return ev + end) +end + +function M.run(scope, params) + params = params or {} + local conn = assert(params.conn, 'ui.service.run: conn required') + local done_tx, done_rx = mailbox.new(params.done_queue_len or 128, { full = 'reject_newest' }) + local sessions = params.sessions or sessions_mod.new(params.session_opts) + local auth = params.auth or auth_mod.new(params.auth_opts or {}) + local model = params.model or read_model.new(params.read_model_opts) + local watch_owner = params.watch_owner or params.watches or read_model.new_watches(model, params.watch_opts or params.read_model_opts) + local read_model_owns_model = false + + local cfg_watch, cfg_err = config_watch.open(conn, 'ui', { + topic = params.config_topic or { 'cfg', 'ui' }, + queue_len = params.config_queue_len or 4, + full = 'reject_newest', + }) + if not cfg_watch then error(cfg_err or 'ui config subscribe failed', 2) end + + local state = { + scope = scope, + params = params, + service_id = params.service_id or 'ui', + conn = conn, + lifecycle = params.lifecycle, + svc = params.lifecycle, + cfg_watch = cfg_watch, + done_tx = done_tx, + done_rx = done_rx, + sessions = sessions, + auth = auth, + model = model, + watch_owner = watch_owner, + model_seen = model:version(), + sessions_seen = (type(sessions.version) == 'function') and sessions:version() or 0, + service_status = 'starting', + read_model_status = 'starting', + listener_status = 'disabled', + listener_generation = 0, + pending_listener_cfg = nil, + http_deps = nil, + http_dep_cap_id = nil, + config_status = 'waiting', + config_generation = 0, + active_requests = 0, + rejected_requests = 0, + components = {}, + lifecycle_obs = {}, + last_error = nil, + } + + scope:finally(function (_, status, primary) + local reason = primary or status or 'ui_service_closed' + cancel_listener(state, reason) + cancel_session_pruner(state, reason) + terminate_http_deps(state, reason) + cfg_watch:close() + done_tx:close(reason) + if not read_model_owns_model then + watch_owner:terminate(reason) + model:terminate(reason) + end + end) + + local read_handle = start_component(state, 'read_model', function (component_scope) + local read_model_opts = shallow_copy(params.read_model_opts or {}) + read_model_opts.model = model + read_model_opts.watch_owner = watch_owner + read_model_owns_model = true + read_model.start(component_scope, conn, read_model_opts) + queue.assert_admit_required(done_tx, { kind = 'component_started', component = 'read_model' }, 'ui_read_model_start_report_failed') + fibers.perform(fibers.never()) + end) + state.read_model_handle = read_handle + + state.service_status = 'running' + + if params.config ~= nil then + local decision = apply_config(state, params.config, params.rev or params.config_rev or 1) or {} + if decision.fail then error(decision.fail, 0) end + end + + publish_summary(state) + update_lifecycle(state) + + while true do + local ev = fibers.perform(next_event_op(state)) + if ev == nil then error('ui service event source closed', 0) end + local decision = reduce_event(state, ev) + if decision.coalesce_publish then + schedule_lifecycle_publish(state, decision.reason) + end + if decision.publish then + clear_lifecycle_publish_due(state) + publish_summary(state) + update_lifecycle(state) + end + if decision.fail then error(decision.fail, 0) end + end +end + +function M.start(conn, opts) + opts = opts or {} + if conn == nil then error('ui.start: conn required', 2) end + local scope = fibers.current_scope() + if not scope then error('ui.start must be called inside a fiber', 2) end + + local svc = service_base.new(conn, { + name = opts.name or 'ui', + env = opts.env, + meta = opts.meta, + announce = opts.announce, + }) + svc:starting({ ready = false }) + + svc:running({ ready = false }) + + local params = shallow_copy(opts) + params.conn = conn + params.lifecycle = svc + return M.run(scope, params) +end + +M._test = { + reduce_event = reduce_event, + record_component_done = record_component_done, + apply_service_policy = apply_service_policy, + publish_summary = publish_summary, + record_cleanup_error = record_cleanup_error, + apply_config = apply_config, + lifecycle_readiness = lifecycle_readiness, + update_lifecycle = update_lifecycle, + lifecycle_status_key = lifecycle_status_key, + should_publish_lifecycle = should_publish_lifecycle, + schedule_lifecycle_publish = schedule_lifecycle_publish, +} + + +return M diff --git a/src/services/ui/sessions.lua b/src/services/ui/sessions.lua new file mode 100644 index 00000000..cc139413 --- /dev/null +++ b/src/services/ui/sessions.lua @@ -0,0 +1,228 @@ +-- services/ui/sessions.lua +-- +-- Immediate in-memory session store. +-- +-- Session mutations are versioned with a Pulse. Observers should use +-- changed_op(seen) and recompute from the current snapshot/count. + +local ok_uuid, uuid = pcall(require, 'uuid') +local pulse = require 'fibers.pulse' +local tablex = require 'shared.table' + +local M = {} +local Store = {} +Store.__index = Store + +local function default_now() + local ok, fibers = pcall(require, 'fibers') + if ok and fibers and type(fibers.now) == 'function' then + return fibers.now() + end + return os.time() +end + +local function new_id() + if ok_uuid and uuid and type(uuid.new) == 'function' then + return tostring(uuid.new()) + end + return ('sess-%d-%d'):format(os.time(), math.random(1, 1000000000)) +end + +local shallow_copy = tablex.shallow_copy +local copy_array = tablex.array_copy + +local function copy_event(ev) + if type(ev) ~= 'table' then return ev end + local out = {} + for k, v in pairs(ev) do + if type(v) == 'table' then + if k == 'session_ids' then + out[k] = copy_array(v) + elseif k == 'sessions' then + local arr = {} + for i = 1, #v do arr[i] = shallow_copy(v[i]) end + out[k] = arr + else + out[k] = shallow_copy(v) + end + else + out[k] = v + end + end + return out +end + +local function public_view(rec) + if not rec then return nil end + return { + id = rec.id, + principal = rec.principal, + created_at = rec.created_at, + expires_at = rec.expires_at, + last_seen = rec.last_seen, + data = shallow_copy(rec.data), + } +end + +local function is_expired(self, rec, now) + if not rec then return false end + now = now or self:now() + return rec.expires_at ~= nil and rec.expires_at <= now +end + +function M.new(opts) + opts = opts or {} + return setmetatable({ + _items = {}, + _now = opts.now or default_now, + _default_ttl = opts.default_ttl or 3600, + _pulse = pulse.new(), + _last_event = nil, + }, Store) +end + +function Store:_notify(ev) + if type(ev) ~= 'table' then ev = { kind = tostring(ev) } end + ev.count = self:count() + self._last_event = copy_event(ev) + self._pulse:signal() + return true, nil +end + +function Store:version() + return self._pulse:version() +end + +function Store:last_event() + return copy_event(self._last_event) +end + + +function Store:snapshot() + return { + version = self:version(), + count = self:count(), + sessions = self:list(), + last_event = self:last_event(), + } +end + +function Store:changed_op(seen) + return self._pulse:changed_op(seen):wrap(function (version, reason) + if version == nil then + return nil, nil, reason + end + return version, self:snapshot(), nil + end) +end + +function Store:now() + return self._now() +end + +function Store:create(principal, opts) + opts = opts or {} + local now = self:now() + local ttl = opts.ttl or self._default_ttl + local id = opts.id or new_id() + local rec = { + id = id, + principal = principal, + created_at = now, + last_seen = now, + expires_at = now + ttl, + data = shallow_copy(opts.data), + } + self._items[id] = rec + local view = public_view(rec) + self:_notify({ kind = 'session_created', session_id = id, principal = principal, session = view }) + return view +end + +function Store:get(id) + local rec = self._items[id] + if not rec or is_expired(self, rec) then return nil end + return public_view(rec) +end + +function Store:touch(id, opts) + opts = opts or {} + local rec = self._items[id] + if not rec then return nil, 'not_found' end + local now = self:now() + if is_expired(self, rec, now) then + return nil, 'expired' + end + rec.last_seen = now + if opts.ttl then rec.expires_at = now + opts.ttl end + if opts.data then rec.data = shallow_copy(opts.data) end + local view = public_view(rec) + self:_notify({ kind = 'session_touched', session_id = id, principal = rec.principal, session = view }) + return view, nil +end + +function Store:delete(id) + local rec = self._items[id] + local existed = rec ~= nil + self._items[id] = nil + if existed then + self:_notify({ + kind = 'session_deleted', + session_id = id, + principal = rec.principal, + session = public_view(rec), + }) + end + return existed +end + +function Store:prune(now) + now = now or self:now() + local removed = {} + local removed_sessions = {} + for id, rec in pairs(self._items) do + if rec.expires_at and rec.expires_at <= now then + self._items[id] = nil + removed[#removed + 1] = id + removed_sessions[#removed_sessions + 1] = public_view(rec) + end + end + table.sort(removed, function (a, b) return tostring(a) < tostring(b) end) + if #removed > 0 then + self:_notify({ + kind = 'session_pruned', + session_ids = copy_array(removed), + sessions = removed_sessions, + removed = #removed, + }) + end + return removed +end + +function Store:count() + local n = 0 + local now = self:now() + for _, rec in pairs(self._items) do + if not is_expired(self, rec, now) then n = n + 1 end + end + return n +end + +function Store:list() + local out = {} + local now = self:now() + for _, rec in pairs(self._items) do + if not is_expired(self, rec, now) then + out[#out + 1] = public_view(rec) + end + end + table.sort(out, function(a, b) return tostring(a.id) < tostring(b.id) end) + return out +end + +function Store:public_view(id) + return self:get(id) +end + +M.Store = Store +return M diff --git a/src/services/ui/supervision.lua b/src/services/ui/supervision.lua new file mode 100644 index 00000000..ecbd3672 --- /dev/null +++ b/src/services/ui/supervision.lua @@ -0,0 +1,55 @@ +-- services/ui/supervision.lua +-- +-- Pure supervision policy helpers for UI components. +-- +-- These functions classify completed scoped work as data. Coordinators record +-- the completion first, then apply the returned action. No function here may +-- perform Ops or touch transport resources. + +local M = {} + +local function result(ev) + return type(ev) == 'table' and type(ev.result) == 'table' and ev.result or {} +end + +local function reason_from(ev, fallback) + local r = result(ev) + return ev.primary or r.reason or r.err or r.status or ev.status or fallback +end + +local function decision(class, action, reason) + return { + class = class, + action = action or 'continue', + reason = reason, + } +end + + +local function component_name(ev) + return type(ev) == 'table' and ev.component or nil +end + +function M.classify_service_component_done(_, ev) + local component = component_name(ev) or 'component' + + if ev.status == 'ok' then + return decision('normal_close', 'continue', reason_from(ev, component .. '_stopped')) + end + + if ev.status == 'failed' then + return decision('failed', 'fail_service', ('%s failed: %s'):format(component, tostring(reason_from(ev, 'failed')))) + end + + if ev.status == 'cancelled' then + return decision('cancelled_unexpected', 'fail_service', ('%s cancelled unexpectedly: %s'):format(component, tostring(reason_from(ev, 'cancelled')))) + end + + return decision('failed', 'fail_service', ('%s ended with invalid status: %s'):format(component, tostring(ev.status))) +end + +M._test = { + reason_from = reason_from, +} + +return M diff --git a/src/services/ui/topics.lua b/src/services/ui/topics.lua new file mode 100644 index 00000000..17c203bb --- /dev/null +++ b/src/services/ui/topics.lua @@ -0,0 +1,64 @@ +-- services/ui/topics.lua +-- +-- Pure topic helpers for the UI service. + +local M = {} + +local function t(...) + return { ... } +end + +function M.svc_status() + return t('svc', 'ui', 'status') +end + +function M.svc_meta() + return t('svc', 'ui', 'meta') +end + + +function M.default_retained_patterns(opts) + opts = opts or {} + local patterns = { + t('state', '#'), + t('svc', '+', 'status'), + } + if opts.include_raw == true then patterns[#patterns + 1] = t('raw', '#') end + return patterns +end + + +function M.default_excluded_retained_patterns() + return { + t('state', 'ui', '#'), + t('svc', 'ui', '#'), + t('obs', 'v1', 'ui', '#'), + } +end + +function M.topic_key(topic) + assert(type(topic) == 'table', 'topic_key: topic must be a table') + local out = {} + for i = 1, #topic do + local v = topic[i] + local tv = type(v) + if tv == 'string' then + out[#out + 1] = 's' .. #v .. ':' .. v + elseif tv == 'number' then + local s = tostring(v) + out[#out + 1] = 'n' .. #s .. ':' .. s + else + error('topic_key: topic tokens must be strings or numbers', 2) + end + end + return table.concat(out, '|') +end + +function M.topic_string(topic) + if type(topic) ~= 'table' then return tostring(topic) end + local out = {} + for i = 1, #topic do out[i] = tostring(topic[i]) end + return table.concat(out, '/') +end + +return M diff --git a/src/services/ui/update/artifact_ingest.lua b/src/services/ui/update/artifact_ingest.lua new file mode 100644 index 00000000..22354155 --- /dev/null +++ b/src/services/ui/update/artifact_ingest.lua @@ -0,0 +1,159 @@ +-- services/ui/update/artifact_ingest.lua +-- +-- Strict boundary around artifact ingest providers. +-- +-- Production ingest providers must expose Op-returning methods for all +-- potentially blocking work and an immediate abort_now() method for finalisers. +-- This module deliberately does not wrap raw append/write/commit/close methods +-- in guards, because that would hide blocking provider calls behind an Op shape. + +local op = require 'fibers.op' + +local M = {} + +local function is_op(v) + return type(v) == 'table' and getmetatable(v) == op.Op +end + +local function require_op_method(obj, method, owner) + if type(obj) ~= 'table' or type(obj[method]) ~= 'function' then + return nil, (owner or 'artifact ingest object') .. ' must expose ' .. method + end + return obj[method], nil +end + +local function call_op_method(obj, method, owner, ...) + local fn, err = require_op_method(obj, method, owner) + if not fn then return op.always(nil, err) end + + local ok, ev_or_err = pcall(function (...) + return fn(obj, ...) + end, ...) + + if not ok then + return op.always(nil, tostring(ev_or_err)) + end + if not is_op(ev_or_err) then + return op.always(nil, (owner or 'artifact ingest object') .. ' ' .. method .. ' must return an Op') + end + return ev_or_err +end + +function M.open_ingest_op(client, opts) + return call_op_method(client, 'open_ingest_op', 'artifact ingest client', opts) +end + +function M.append_chunk_op(handle, chunk) + return call_op_method(handle, 'append_chunk_op', 'artifact ingest handle', chunk) +end + +function M.commit_op(handle) + return call_op_method(handle, 'commit_op', 'artifact ingest handle') +end + +function M.abort_now(handle, reason) + if handle == nil then return true, nil end + if type(handle) ~= 'table' or type(handle.abort_now) ~= 'function' then + return nil, 'artifact ingest handle must expose immediate abort_now' + end + + local ok, a, b = pcall(function () + return handle:abort_now(reason) + end) + if not ok then + return nil, tostring(a) + end + if is_op(a) then + return nil, 'artifact ingest abort_now must be immediate and must not return an Op' + end + if a == nil and b == nil then return true, nil end + if a == true then return true, nil end + if a == false or a == nil then return nil, b or 'artifact ingest abort_now failed' end + return true, nil +end + + +local BusHandle = {} +BusHandle.__index = BusHandle + +local function ingest_rpc(method) + return { 'cap', 'artifact-ingest', 'main', 'rpc', method } +end + +local function request_timeout(opts) + local out = {} + for k, v in pairs(opts or {}) do out[k] = v end + if out.timeout == nil and out.deadline == nil then out.timeout = false end + return out +end + +local function ingest_id_from(reply, fallback) + local rec = type(reply) == 'table' and (reply.ingest or reply.record or reply) or nil + return rec and rec.ingest_id or fallback +end + +function BusHandle:append_chunk_op(chunk) + if self._closed then return op.always(nil, 'artifact_ingest_handle_closed') end + return self._conn:call_op(ingest_rpc('append'), { + ingest_id = self.ingest_id, + chunk = chunk, + }, request_timeout(self._opts)):wrap(function (reply, err) + return reply, err + end) +end + +function BusHandle:commit_op() + if self._closed then return op.always(nil, 'artifact_ingest_handle_closed') end + self._closed = true + return self._conn:call_op(ingest_rpc('commit'), { + ingest_id = self.ingest_id, + }, request_timeout(self._opts)):wrap(function (reply, err) + if reply == nil or err ~= nil then return nil, err end + if type(reply) ~= 'table' then return nil, 'invalid_artifact_ingest_commit_reply' end + local committed = reply.commit + if type(committed) ~= 'table' then return nil, 'invalid_artifact_ingest_commit_reply' end + local artifact = committed.artifact + if type(artifact) == 'table' then + return artifact.artifact_ref or artifact.artifact_id or artifact.ref or artifact.id or artifact + end + return committed.artifact_ref or committed.artifact_id or committed.ref or committed.id or artifact + end) +end + +function BusHandle:abort_now(reason) + self._closed = true + self._abort_reason = reason or 'artifact_ingest_aborted' + return true, nil +end + +local BusClient = {} +BusClient.__index = BusClient + +function BusClient:open_ingest_op(opts) + opts = opts or {} + local ingest_id = opts.ingest_id or ('ui-upload-' .. tostring(math.floor((os.clock() or 0) * 1000000))) + local sink = opts.sink + return self._conn:call_op(ingest_rpc('create'), { + ingest_id = ingest_id, + component = opts.component, + sink = sink, + metadata = opts.metadata, + }, request_timeout(opts)):wrap(function (reply, err) + if reply == nil or err ~= nil then return nil, err end + return setmetatable({ + _conn = self._conn, + _opts = opts, + ingest_id = ingest_id_from(reply, ingest_id), + _closed = false, + }, BusHandle) + end) +end + +function M.bus_client(conn) + if not conn then return nil, 'artifact ingest bus client requires conn' end + return setmetatable({ _conn = conn }, BusClient) +end + +M._test = { is_op = is_op } + +return M diff --git a/src/services/ui/update/client.lua b/src/services/ui/update/client.lua new file mode 100644 index 00000000..a21ba3fd --- /dev/null +++ b/src/services/ui/update/client.lua @@ -0,0 +1,41 @@ +-- services/ui/update/client.lua +-- +-- Update service client boundary. Reusable upstream waits are exposed as Ops. + +local M = {} + +local function call_opts(opts, _default_timeout) + local out = {} + for k, v in pairs(opts or {}) do out[k] = v end + if out.timeout == nil and out.deadline == nil then out.timeout = false end + return out +end + +local function update_manager_rpc(method) + return { 'cap', 'update-manager', 'main', 'rpc', method } +end + +function M.create_job_op(conn, artifact_id, opts) + opts = opts or {} + local method = update_manager_rpc('create-job') + return conn:call_op(method, { + artifact_id = artifact_id, + artifact_ref = artifact_id, + component = opts.component, + options = opts.options, + metadata = opts.metadata, + job_id = opts.job_id, + }, call_opts(opts, 10.0)):wrap(function (reply, err) + if reply == nil or err ~= nil then return nil, err end + if type(reply) == 'table' and reply.job ~= nil then return reply.job end + return reply + end) +end + +function M.start_job_op(conn, job_id, opts) + opts = opts or {} + local method = update_manager_rpc('start-job') + return conn:call_op(method, { job_id = job_id }, call_opts(opts, 10.0)) +end + +return M diff --git a/src/services/ui/update/upload.lua b/src/services/ui/update/upload.lua new file mode 100644 index 00000000..88cab467 --- /dev/null +++ b/src/services/ui/update/upload.lua @@ -0,0 +1,192 @@ +-- services/ui/update/upload.lua +-- +-- Update upload scoped operation. It owns ingest before commit, detaches +-- abort cleanup after commit, and may hand off to the update service. +-- +-- Upload timeout is an ownership decision: waits inside the upload scope are +-- raced against the deadline. If the deadline wins, the upload scope is +-- cancelled with reason "timeout" and the uncommitted ingest finaliser aborts +-- immediately. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local ingest = require 'services.ui.update.artifact_ingest' +local client = require 'services.ui.update.client' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local resource = require 'devicecode.support.resource' +local scope_mod = require 'fibers.scope' + +local M = {} + +local function connect_update_conn(scope, opts) + if opts.update_conn then + return opts.update_conn, false, nil + end + if opts.conn then + return opts.conn, false, nil + end + local conn, err + if type(opts.connect) == 'function' then + conn, err = opts.connect(opts.principal, opts) + elseif opts.bus and type(opts.bus.connect) == 'function' then + conn, err = opts.bus:connect({ principal = opts.principal }) + end + if not conn then return nil, false, err or 'update connection unavailable' end + local conn_owner = resource.owned(conn, { + label = 'update upload connection cleanup', + terminate = function (value) + return bus_cleanup.disconnect(value) + end, + }) + scope:finally(function (_, status, primary) + conn_owner:terminate_checked(primary or status or 'upload_closed', 'update upload connection cleanup') + end) + return conn, true, nil +end + +local function read_chunk_op(body, max) + if type(body) == 'table' and type(body.read_chunk_op) == 'function' then + return body:read_chunk_op(max) + end + return fibers.always(nil, 'request body has no read_chunk_op') +end + +local function deadline_from_opts(opts) + if type(opts.deadline) == 'number' then + return opts.deadline + end + local timeout = opts.upload_timeout + if timeout == nil then timeout = opts.timeout end + if type(timeout) == 'number' then + return fibers.now() + timeout + end + return nil +end + + +local function cancel_for_timeout(scope, on_timeout) + if on_timeout then on_timeout() end + if scope and type(scope.cancel) == 'function' then + scope:cancel('timeout') + end + error(scope_mod.cancelled('timeout'), 0) +end + +local function perform_with_deadline(scope, ev, deadline, on_timeout) + if deadline == nil then + return fibers.perform(ev) + end + + local dt = deadline - fibers.now() + if dt <= 0 then + cancel_for_timeout(scope, on_timeout) + end + + local completed, a, b, c = fibers.perform(fibers.boolean_choice(ev, sleep.sleep_op(dt))) + if completed then + return a, b, c + end + + cancel_for_timeout(scope, on_timeout) +end + +local function upload_body_op(ctx, opts, deadline) + return fibers.run_scope_op(function (scope) + local timed_out = false + local function mark_timeout() timed_out = true end + + local ingest_client = opts.ingest + if ingest_client == nil then + local conn, _, conn_err = connect_update_conn(scope, opts) + if not conn then error(conn_err or 'update connection unavailable', 0) end + opts.update_conn = opts.update_conn or conn + local built, build_err = ingest.bus_client(conn) + if not built then error(build_err or 'artifact ingest bus client unavailable', 0) end + ingest_client = built + end + + local handle, open_err = perform_with_deadline(scope, ingest.open_ingest_op(ingest_client, opts), deadline, mark_timeout) + if not handle then error(open_err or 'artifact ingest open failed', 0) end + + local ingest_owner = resource.owned(handle, { + label = 'upload ingest abort', + terminate = function (value, reason) + return ingest.abort_now(value, reason) + end, + }) + scope:finally(function (_, status, primary) + ingest_owner:terminate_checked( + timed_out and 'timeout' or primary or status or 'upload_closed', + 'upload ingest abort' + ) + end) + + local body = ctx.body_stream or ctx.body or ctx.stream or ctx + if body == nil then error('request body has no chunk reader', 0) end + local uploaded_bytes = 0 + local uploaded_chunks = 0 + while true do + local chunk, rerr = perform_with_deadline(scope, read_chunk_op(body, opts.chunk_size or 65536), deadline, mark_timeout) + if rerr then error(rerr, 0) end + if chunk == nil or chunk == '' then break end + local ok, werr = perform_with_deadline(scope, ingest.append_chunk_op(handle, chunk), deadline, mark_timeout) + if ok == nil or ok == false then error(werr or 'artifact append failed', 0) end + uploaded_chunks = uploaded_chunks + 1 + uploaded_bytes = uploaded_bytes + #chunk + end + + local artifact_id, cerr = perform_with_deadline(scope, ingest.commit_op(handle), deadline, mark_timeout) + if not artifact_id then error(cerr or 'artifact commit failed', 0) end + local _committed_handle, detach_err = ingest_owner:detach() + if detach_err then error(detach_err, 0) end + + local out = { status = 'ok', artifact_id = artifact_id } + if opts.create_job then + local conn, _, conn_err = connect_update_conn(scope, opts) + if not conn then error(conn_err or 'update connection unavailable', 0) end + + local call_opts = {} + for k, v in pairs(opts) do call_opts[k] = v end + call_opts.timeout = false + + local job, jerr = perform_with_deadline(scope, client.create_job_op(conn, artifact_id, call_opts), deadline, mark_timeout) + if not job then error(jerr or 'update job create failed', 0) end + if type(job) ~= 'table' or type(job.job_id) ~= 'string' or job.job_id == '' then + error('create_job_reply_missing_job_id', 0) + end + out.job_id = job.job_id + if opts.start_job then + call_opts.timeout = false + local started, serr = perform_with_deadline(scope, client.start_job_op(conn, job.job_id, call_opts), deadline, mark_timeout) + if not started then error(serr or 'update job start failed', 0) end + out.started = started + end + end + return out + end) +end + +function M.run(scope, owner, ctx, opts) + opts = opts or {} + local st, _, result_or_primary = fibers.perform(M.run_op(ctx, opts)) + if st ~= 'ok' then + local ok, werr = fibers.perform(owner:reply_error_op(nil, result_or_primary or st)) + if ok ~= true then error(werr or 'response write failed', 0) end + return { status = st, err = result_or_primary or st } + end + local ok, werr = fibers.perform(owner:reply_json_op(200, result_or_primary)) + if ok ~= true then error(werr or 'response write failed', 0) end + return result_or_primary +end + +function M.run_op(ctx, opts) + ctx = ctx or {} + opts = opts or {} + + return fibers.guard(function () + local deadline = deadline_from_opts(opts) + return upload_body_op(ctx, opts, deadline) + end) +end + +return M diff --git a/src/services/ui/user_operation.lua b/src/services/ui/user_operation.lua new file mode 100644 index 00000000..807c28fe --- /dev/null +++ b/src/services/ui/user_operation.lua @@ -0,0 +1,119 @@ +-- services/ui/user_operation.lua +-- +-- Authenticated scoped upstream calls. This is for request/client operation +-- scopes, not service coordinators. +-- +-- Timeout is modelled as a race against the whole operation scope. If the +-- timeout wins, run_scope_op's abort path cancels and joins the operation +-- scope, running the owned connection finaliser before the boundary returns +-- cancelled, "timeout". + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local resource = require 'devicecode.support.resource' + +local M = {} + +local function connect_for(spec) + if type(spec.connect) == 'function' then + return spec.connect(spec.principal, spec) + end + if spec.bus and type(spec.bus.connect) == 'function' then + return spec.bus:connect({ principal = spec.principal }) + end + if spec.conn ~= nil then + return spec.conn + end + return nil, 'no user_operation connection source' +end + +local function disconnect_if_owned(conn, spec) + if spec.conn ~= nil and conn == spec.conn and not spec.disconnect_borrowed then + return true, nil + end + return bus_cleanup.disconnect(conn) +end + +local function normalise_result(result) + if type(result) == 'table' then return result end + return { value = result } +end + +local function timeout_report() + return { id = 0, extra_errors = {}, children = {}, timeout = true } +end + +local function deadline_from_spec(spec) + if type(spec.deadline) == 'number' then + return spec.deadline + end + if type(spec.timeout) == 'number' then + return fibers.now() + spec.timeout + end + return nil +end + +local function operation_body_op(spec) + return fibers.run_scope_op(function (scope) + local conn, cerr = connect_for(spec) + if not conn then error(cerr or 'user connection failed', 0) end + + local conn_owner = resource.owned(conn, { + label = 'user operation connection cleanup', + terminate = function (value) + return disconnect_if_owned(value, spec) + end, + }) + + scope:finally(function (_, status, primary) + conn_owner:terminate_checked( + primary or status or 'user_operation_closed', + 'user operation connection cleanup' + ) + end) + + if type(spec.run_op) == 'function' then + local result, err = fibers.perform(spec.run_op(scope, conn)) + if result == nil then error(err or 'user_operation_failed', 0) end + return normalise_result(result) + end + + return normalise_result(spec.run(scope, conn)) + end) +end + +function M.run_op(spec) + spec = spec or {} + if type(spec.run_op) ~= 'function' and type(spec.run) ~= 'function' then + error('user_operation.run_op: run_op or run function required', 2) + end + + return fibers.guard(function () + local deadline = deadline_from_spec(spec) + local work = operation_body_op(spec) + if deadline == nil then + return work + end + + local dt = deadline - fibers.now() + if dt <= 0 then + return fibers.always('cancelled', timeout_report(), 'timeout') + end + + return fibers.boolean_choice(work, sleep.sleep_op(dt)):wrap(function (work_won, st, rep, value_or_primary) + if work_won then + return st, rep, value_or_primary + end + return 'cancelled', timeout_report(), 'timeout' + end) + end) +end + +function M.run(spec) + local st, rep, a = fibers.perform(M.run_op(spec)) + if st == 'ok' then return a, nil, rep end + return nil, a, rep +end + +return M diff --git a/src/services/update.lua b/src/services/update.lua new file mode 100644 index 00000000..ec361655 --- /dev/null +++ b/src/services/update.lua @@ -0,0 +1,29 @@ +-- services/update.lua +-- Public update service entry point. + +local M = { + service = require 'services.update.service', + generation = require 'services.update.generation', + events = require 'services.update.events', + model = require 'services.update.model', + projection = require 'services.update.projection', + publisher = require 'services.update.publisher', + topics = require 'services.update.topics', + config = require 'services.update.config', + job_repository = require 'services.update.job_repository', + job_store_memory = require 'services.update.job_store_memory', + job_store_control_store = require 'services.update.job_store_control_store', + manager_requests = require 'services.update.manager_requests', + active_runtime = require 'services.update.active_runtime', + active_job = require 'services.update.active_job', +} + +function M.start(conn, opts) + M.service.start(conn, opts) +end + +function M.run(scope, params) + return M.service.run(scope, params) +end + +return M diff --git a/src/services/update/active_job.lua b/src/services/update/active_job.lua new file mode 100644 index 00000000..4622673b --- /dev/null +++ b/src/services/update/active_job.lua @@ -0,0 +1,402 @@ +-- services/update/active_job.lua +-- +-- Active update worker bodies. +-- +-- These functions run inside an active-job scope. They may perform backend Ops +-- and observer waits because the active-job scope owns the update phase +-- lifetime. They return one result table on success and let ordinary failures +-- escape through the scope machinery. + +local safe = require 'coxpcall' +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' + +local M = {} + +local function backend_method(backend, name, required) + if type(backend) ~= 'table' then + error('active_job: backend required', 0) + end + + local fn = backend[name] + if type(fn) == 'function' then + return fn + end + + if required then + error('update backend missing ' .. name, 0) + end + + return nil +end + +local function perform_backend_op(backend, name, required, ...) + local fn = backend_method(backend, name, required) + if not fn then + return nil, nil + end + + local result, err = fibers.perform(fn(backend, ...)) + if result == nil then + error(err or (name .. '_failed'), 0) + end + + return result, nil +end + +local function base_ctx(params, phase) + local ctx = {} + for k, v in pairs(params.ctx or {}) do + ctx[k] = v + end + ctx.phase = phase + ctx.lease = params.lease + ctx.deadline = params.deadline + return ctx +end + +local function require_job(params, label) + local job = params and params.job or nil + if type(job) ~= 'table' then + error(label .. ': job required', 0) + end + if type(job.job_id) ~= 'string' or job.job_id == '' then + error(label .. ': job.job_id required', 0) + end + return job +end + +local function stage_context(params) + local backend = assert(params.backend, 'active_job.stage: backend required') + local job = require_job(params, 'active_job.stage') + local ctx = base_ctx(params, 'stage') + return backend, job, ctx +end + +local function perform_backend_op_until(backend, name, required, deadline, ...) + local fn = backend_method(backend, name, required) + if not fn then + return nil, nil + end + + local backend_op = fn(backend, ...) + local result, err + + if deadline ~= nil then + if fibers.now() >= deadline then + return nil, name .. '_timeout' + end + + -- The update phase owns this deadline. The backend Op, including any + -- bus request it opens, has no hidden call timeout. When timeout wins the + -- losing backend Op is aborted, allowing lua-bus to abandon the request and + -- Device to observe caller cancellation. + local which, a, b = fibers.perform(fibers.named_choice { + backend = backend_op, + timeout = sleep.sleep_until_op(deadline), + }) + if which == 'timeout' then + return nil, name .. '_timeout' + end + result, err = a, b + else + result, err = fibers.perform(backend_op) + end + + if result == nil then + return nil, err or (name .. '_failed') + end + + return result, nil +end + +function M.stage(_scope, params) + params = params or {} + local backend, job, ctx = stage_context(params) + local staged, err = perform_backend_op_until(backend, 'stage_op', true, params.deadline, job, ctx) + if staged == nil then error(err or 'stage_op_failed', 0) end + + return { + tag = 'staged', + job_id = job.job_id, + component = job.component, + staged = staged, + } +end + + +local COMMIT_POLICIES = { + idempotent_by_token = true, + no_duplicate = true, +} + +local function backend_commit_policy(backend) + local policy + if type(backend.commit_capabilities) == 'function' then + local caps = backend:commit_capabilities() + policy = type(caps) == 'table' and caps.policy or caps + elseif type(backend.commit_policy) == 'function' then + local caps = backend:commit_policy() + policy = type(caps) == 'table' and caps.policy or caps + elseif type(backend.commit_policy) == 'string' then + policy = backend.commit_policy + elseif type(backend.commit) == 'table' then + policy = backend.commit.policy + end + + if not COMMIT_POLICIES[policy] then + error('update backend commit policy must be idempotent_by_token or no_duplicate', 0) + end + + return policy +end + +local function perform_transition(jobs, cmd, unavailable_reason) + if type(jobs) ~= 'table' or type(jobs.admit_transition) ~= 'function' then + error(unavailable_reason or 'job_runtime_unavailable', 0) + end + + local handle, admit_err = jobs:admit_transition(cmd) + if not handle then + error(admit_err or 'job_transition_admission_failed', 0) + end + + local result, err = fibers.perform(handle:outcome_op()) + if not result or result.status ~= 'persisted' then + error(err or (result and result.reason) or 'job_transition_persist_failed', 0) + end + + return result +end + +local function begin_commit_attempt(params, job, ctx, policy) + local jobs = params.jobs or (params.ctx and params.ctx.jobs) + local lease = params.lease + local active_token = lease and lease.token or job.active_token or (job.active_intent and job.active_intent.token) + local commit_token = (job.commit_attempt and job.commit_attempt.token) + or (job.active_intent and job.active_intent.commit_token) + or active_token + + local result = perform_transition(jobs, { + kind = 'begin_commit_attempt', + generation = (lease and lease.generation) or job.generation, + job_id = job.job_id, + phase = 'commit', + token = active_token, + commit_token = commit_token, + commit_policy = policy, + pre_commit = ctx.pre_commit, + reason = 'active_commit_begin_attempt', + }, 'active_job.commit: job_runtime required before backend commit') + + ctx.commit_token = result.commit_token or commit_token + ctx.commit_policy = result.commit_policy or policy + return result +end + +local function persist_commit_accepted(params, job, ctx, policy, accepted) + local jobs = params.jobs or (params.ctx and params.ctx.jobs) + local lease = params.lease + local active_token = lease and lease.token or job.active_token or (job.active_intent and job.active_intent.token) + local ok, result = safe.pcall(function () + return perform_transition(jobs, { + kind = 'commit_accepted', + generation = (lease and lease.generation) or job.generation, + job_id = job.job_id, + phase = 'commit', + token = active_token, + commit_token = ctx.commit_token, + commit_policy = ctx.commit_policy or policy, + accepted = accepted, + reason = 'active_commit_accepted', + }, 'active_job.commit: job_runtime required after backend commit') + end) + if ok then return result end + local reason = tostring(result or 'commit_accepted_persist_failed') + error('critical_inconsistent_commit_acceptance: ' .. reason, 0) +end + +local function persist_commit_failed(params, job, ctx, policy, reason) + local jobs = params.jobs or (params.ctx and params.ctx.jobs) + local lease = params.lease + local active_token = lease and lease.token or job.active_token or (job.active_intent and job.active_intent.token) + local ok, result = safe.pcall(function () + return perform_transition(jobs, { + kind = 'commit_failed', + generation = (lease and lease.generation) or job.generation, + job_id = job.job_id, + phase = 'commit', + token = active_token, + commit_token = ctx.commit_token, + commit_policy = ctx.commit_policy or policy, + error = reason, + reason = reason or 'active_commit_failed', + }, 'active_job.commit: job_runtime required after backend commit failure') + end) + if ok then return result end + error(tostring(result or 'commit_failed_persist_failed'), 0) +end + +local function perform_commit_backend_op(backend, job, ctx) + local fn = backend_method(backend, 'commit_op', true) + local result, err = fibers.perform(fn(backend, job, ctx)) + if result == nil then return nil, err or 'commit_op_failed' end + return result, nil +end + +function M.commit(_scope, params) + params = params or {} + local backend = assert(params.backend, 'active_job.commit: backend required') + local job = require_job(params, 'active_job.commit') + local ctx = base_ctx(params, 'commit') + local policy = backend_commit_policy(backend) + local pre_commit = perform_backend_op(backend, 'pre_commit_record_op', false, job, ctx) + if pre_commit ~= nil then ctx.pre_commit = pre_commit end + local attempt = begin_commit_attempt(params, job, ctx, policy) + + local accepted, commit_err = perform_commit_backend_op(backend, job, ctx) + if accepted == nil then + persist_commit_failed(params, job, ctx, policy, commit_err or 'component_commit_update_failed') + error(commit_err or 'component_commit_update_failed', 0) + end + local persisted = persist_commit_accepted(params, job, ctx, policy, accepted) + + return { + tag = 'commit_started', + job_id = job.job_id, + component = job.component, + commit = accepted, + accepted = true, + commit_token = ctx.commit_token or attempt.commit_token, + commit_policy = ctx.commit_policy or policy, + persisted = persisted, + } +end + +local function reconcile_eval_fn(backend) + local fn = backend_method(backend, 'evaluate_reconcile', false) + if fn then return fn end + fn = backend_method(backend, 'evaluate', false) + if fn then return fn end + error('update backend missing evaluate_reconcile', 0) +end + +local function evaluate_reconcile(backend, job, snapshot, ctx) + local result = reconcile_eval_fn(backend)(backend, job, snapshot, ctx) + if type(result) ~= 'table' then + return { done = false } + end + return result +end + +local function deadline_reached(deadline) + return deadline ~= nil and fibers.now() >= deadline +end + +local function timeout_result(job, deadline) + return { + tag = 'reconcile_timeout', + job_id = job.job_id, + deadline = deadline, + } +end + +local function observer_closed_result(job, reason) + return { + tag = 'reconcile_observer_closed', + job_id = job.job_id, + reason = reason or 'observer_closed', + } +end + +local function normalise_reconcile_done(job, result) + result.job_id = result.job_id or job.job_id + + if result.tag == nil then + if result.ok == false then + result.tag = 'reconciled_failure' + else + result.tag = 'reconciled_success' + end + end + + return result +end + +local function wait_for_reconcile_progress(observer, seen, deadline, poll_s) + if observer and type(observer.changed_op) == 'function' then + local which, version, snapshot, reason = fibers.perform(fibers.named_choice { + changed = observer:changed_op(seen), + timeout = deadline and sleep.sleep_until_op(deadline) or fibers.never(), + }) + + if which == 'timeout' then + return 'timeout' + end + + if version == nil then + return 'observer_closed', reason + end + + return 'changed', version, snapshot + end + + fibers.perform(sleep.sleep_op(poll_s or 0.05)) + return 'polled' +end + +function M.reconcile(_scope, params) + params = params or {} + local backend = assert(params.backend, 'active_job.reconcile: backend required') + local job = require_job(params, 'active_job.reconcile') + local observer = params.observer + local deadline = params.deadline + local seen = observer and observer.version and observer:version() or 0 + local ctx = base_ctx(params, 'reconcile') + + while true do + local snapshot = observer and observer.snapshot and observer:snapshot() or nil + local result = evaluate_reconcile(backend, job, snapshot, ctx) + + if result.done then + return normalise_reconcile_done(job, result) + end + + if deadline_reached(deadline) then + return timeout_result(job, deadline) + end + + local status, a, b = wait_for_reconcile_progress(observer, seen, deadline, params.poll_s) + if status == 'timeout' then + return timeout_result(job, deadline) + end + if status == 'observer_closed' then + return observer_closed_result(job, a) + end + if status == 'changed' then + seen = a or seen + ctx.last_observed = b + end + end +end + +function M.run(scope, params) + params = params or {} + local phase = params.phase or 'stage' + + if phase == 'stage' then + return M.stage(scope, params) + end + + if phase == 'commit' then + return M.commit(scope, params) + end + + if phase == 'reconcile' then + return M.reconcile(scope, params) + end + + error('unknown active job phase: ' .. tostring(phase), 0) +end + +return M diff --git a/src/services/update/active_policy.lua b/src/services/update/active_policy.lua new file mode 100644 index 00000000..62deae2d --- /dev/null +++ b/src/services/update/active_policy.lua @@ -0,0 +1,102 @@ +-- services/update/active_policy.lua +-- +-- Generation-local policy for admitting and applying active update work. +-- +-- The active slot itself is service-owned. This module decides whether a job is +-- eligible for active work and translates active-work completion facts into job +-- state transitions. + +local repo_mod = require 'services.update.job_repository' + +local M = {} + +local function is_artifact_missing_stage_resume(job, ev) + if not job or not ev then return false end + if ev.phase ~= 'stage' then return false end + local reason = ev.primary or ev.error or ev.reason + if reason ~= 'not_found' and reason ~= 'artifact_not_found' and reason ~= 'artifact_source_open_failed' then + return false + end + local adoption = type(job.adoption) == 'table' and job.adoption or {} + if adoption.action ~= 'resume_active_intent' then return false end + if job.component ~= 'mcu' then return false end + return type(job.expected_image_id) == 'string' and job.expected_image_id ~= '' +end + +function M.phase_for(job, payload) + payload = type(payload) == 'table' and payload or {} + return payload.phase or (job and job.state == 'awaiting_commit' and 'commit' or 'stage') +end + +function M.can_start_phase(job, phase) + if not job then return nil, 'not_found' end + phase = phase or M.phase_for(job) + if type(job.runtime) == 'table' and job.runtime.persistence_pending then + return nil, 'job_persistence_pending' + end + if phase == 'commit' then + if job.state ~= 'awaiting_commit' then return nil, 'job_not_committable' end + return true, nil + end + if phase == 'reconcile' then + if job.state ~= 'awaiting_return' then return nil, 'job_not_reconcilable' end + return true, nil + end + if job.state ~= 'created' then + return nil, 'job_not_startable' + end + return true, nil +end + +function M.can_start(job) + return M.can_start_phase(job, M.phase_for(job)) +end + +function M.mark_starting(job, phase, seq) + local updated = assert(repo_mod.normalise_job(job)) + if phase == 'commit' then + repo_mod.mark_committing(updated, { seq = seq, reason = 'start_commit' }) + else + repo_mod.mark_staging(updated, { seq = seq, reason = 'start_stage' }) + end + return updated +end + +function M.apply_completion(job, ev, seq) + if not job then return false, 'not_found' end + if not ev or ev.kind ~= 'active_job_done' then return false, 'not_active_completion' end + + if ev.status == 'ok' then + local result = ev.result or {} + if result.tag == 'staged' then + repo_mod.mark_awaiting_commit(job, result.staged or result, { seq = seq, reason = 'stage_complete' }) + elseif result.tag == 'commit_started' then + repo_mod.mark_awaiting_return(job, result.commit or result, { seq = seq, reason = 'commit_complete' }) + elseif result.tag == 'reconciled_success' then + repo_mod.mark_terminal(job, 'succeeded', nil, result, { seq = seq, reason = 'reconcile_success' }) + elseif result.tag == 'reconcile_timeout' then + repo_mod.mark_terminal(job, 'timed_out', 'timeout', result, { seq = seq, reason = 'reconcile_timeout' }) + elseif result.tag == 'reconcile_observer_closed' then + repo_mod.mark_terminal(job, 'failed', result.reason or 'observer_closed', result, { seq = seq, reason = 'reconcile_observer_closed' }) + elseif result.tag == 'reconciled_failure' then + repo_mod.mark_terminal(job, 'failed', result.reason or result.error or 'reconcile_failed', result, { seq = seq, reason = 'reconcile_failed' }) + else + repo_mod.mark_terminal(job, 'succeeded', nil, result, { seq = seq, reason = 'active_complete' }) + end + elseif ev.status == 'cancelled' then + repo_mod.mark_terminal(job, 'cancelled', ev.primary or 'cancelled', nil, { seq = seq, reason = 'active_cancelled' }) + elseif is_artifact_missing_stage_resume(job, ev) then + repo_mod.mark_awaiting_return(job, { + tag = 'artifact_missing_reconcile', + reason = ev.primary or ev.error or ev.reason or 'not_found', + expected_image_id = job.expected_image_id, + previous_state = 'staging', + }, { seq = seq, reason = 'artifact_missing_reconcile' }) + else + repo_mod.mark_terminal(job, 'failed', ev.primary or 'failed', nil, { seq = seq, reason = 'active_failed' }) + end + + return true, nil +end + +return M diff --git a/src/services/update/active_runtime.lua b/src/services/update/active_runtime.lua new file mode 100644 index 00000000..25abc9a7 --- /dev/null +++ b/src/services/update/active_runtime.lua @@ -0,0 +1,951 @@ +-- services/update/active_runtime.lua +-- +-- Service-owned active-slot reducer plus lease/start helpers. +-- +-- The active slot is coordinator state, not a lifetime boundary. A lease grants +-- immediate authority to start active work. Once work is successfully started, +-- the active work scope owns the running operation. The component stores the +-- completion, applies it through job_runtime, and releases the slot only after +-- durable apply. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local model = require 'services.update.model' +local active_job = require 'services.update.active_job' + +local M = {} + +local State = {} +State.__index = State + +local Lease = {} +Lease.__index = Lease + +local function copy(v) + return model.deep_copy(v) +end + +local function token_for(job_id, generation, phase, seq) + return table.concat({ + tostring(generation or 0), + tostring(job_id), + tostring(phase or 'stage'), + tostring(seq or 0), + }, ':') +end + +local function new_stats() + return { + accepted = 0, + rejected_busy = 0, + completed = 0, + stale = 0, + failed = 0, + cancelled = 0, + released = 0, + } +end + +local function active_matches(state, token) + return state and state.active and state.active.token == token +end + +local function active_public(active) + if not active then return nil end + return { + service_id = active.service_id, + job_id = active.job_id, + generation = active.generation, + phase = active.phase, + token = active.token, + status = active.status, + job = active.job and copy(active.job) or nil, + } +end + +function M.new_state(_opts) + return setmetatable({ + active = nil, + next_token = 1, + stats = new_stats(), + }, State) +end + +function M.snapshot(state) + return { + active = active_public(state and state.active or nil), + stats = copy((state and state.stats) or {}), + last_completion = state and state.last_completion and copy(state.last_completion) or nil, + } +end + +function M.is_idle(state) + return state == nil or state.active == nil +end + +local function make_lease(state, rec, token) + return setmetatable({ + _state = state, + _transferred = false, + _released = false, + + service_id = rec.service_id, + job_id = rec.job_id, + generation = rec.generation, + phase = rec.phase or 'stage', + token = token, + }, Lease) +end + +function M.claim(state, rec) + assert(state, 'active_runtime.claim: state required') + + if state.active ~= nil then + state.stats.rejected_busy = state.stats.rejected_busy + 1 + return nil, 'slot_busy' + end + + rec = rec or {} + local seq = state.next_token or 1 + state.next_token = seq + 1 + + local token = rec.token or token_for(rec.job_id, rec.generation, rec.phase or 'stage', seq) + local lease = make_lease(state, rec, token) + + state.active = { + service_id = lease.service_id, + job_id = lease.job_id, + generation = lease.generation, + phase = lease.phase, + token = lease.token, + status = 'leased', + handle = nil, + } + state.stats.accepted = state.stats.accepted + 1 + + return lease, nil +end + +function Lease:release(reason) + local state = self._state + if self._released then + return false, 'already_released' + end + if self._transferred then + return false, 'transferred' + end + if not active_matches(state, self.token) then + return false, 'not_owner' + end + + state.active = nil + state.stats.released = state.stats.released + 1 + self._released = true + return true, reason +end + +function Lease:handoff() + if not active_matches(self._state, self.token) then + return nil, 'stale' + end + self._transferred = true + self._state.active.status = 'running' + return true, nil +end + +function Lease:terminate(reason) + return self:release(reason or 'active_lease_terminated') +end + +function Lease:attach_handle(handle, job) + if not active_matches(self._state, self.token) then + return nil, 'stale' + end + self._state.active.handle = handle + self._state.active.status = 'running' + self._state.active.job = job and copy(job) or nil + return true, nil +end + +local function default_runner(spec, lease) + return function (scope) + return active_job.run(scope, { + phase = lease.phase, + job = spec.job, + backend = spec.backend, + observer = spec.observer, + deadline = spec.deadline, + ctx = spec.ctx, + lease = lease, + jobs = spec.jobs, + }) + end +end + +function Lease:start_work(lifetime_scope, spec) + spec = spec or {} + + if self._released then + return nil, 'active lease already released' + end + if self._transferred then + return nil, 'active lease already transferred' + end + if not active_matches(self._state, self.token) then + return nil, 'active lease is stale' + end + if spec.done_tx == nil then + return nil, 'done_tx required' + end + + local local_tx, local_rx + if spec.local_observer_scope ~= nil then + local_tx, local_rx = mailbox.new(1, { full = 'reject_newest' }) + spec.local_observer_scope:finally(function (_, status, primary) + local_tx:close(primary or status or 'active job observer closed') + end) + end + + local identity = { + kind = 'active_job_done', + service_id = self.service_id, + generation = self.generation, + job_id = self.job_id, + phase = self.phase, + token = self.token, + } + + local runner = default_runner(spec, self) + local handle, err = scoped_work.start { + lifetime_scope = lifetime_scope, + reaper_scope = spec.reaper_scope or lifetime_scope, + report_scope = spec.report_scope or lifetime_scope, + + identity = identity, + + run = function (scope) + local result = runner(scope, spec) + return result + end, + + report = function (ev) + local ok, report_err = queue.try_admit_required( + spec.done_tx, + ev, + 'update_active_job_completion_report_failed' + ) + if ok ~= true then + return nil, report_err + end + + -- Local observers are request-local convenience only. The authoritative + -- coordinator completion has already been admitted above. + if local_tx ~= nil then + queue.try_send_now(local_tx, ev) + end + + return true, nil + end, + } + + if not handle then + self:release(err or 'active_start_failed') + return nil, err + end + + local ok_attach, attach_err = self:attach_handle(handle, spec.job) + if ok_attach ~= true then + handle:cancel(attach_err or 'active_start_stale') + self:release(attach_err or 'active_start_stale') + return nil, attach_err or 'active_start_stale' + end + + self._transferred = true + + if local_rx ~= nil then + local raw_handle = handle + local service_id = self.service_id + local generation = self.generation + local job_id = self.job_id + local phase = self.phase + local token = self.token + handle = {} + + function handle:cancel(reason) + return raw_handle:cancel(reason) + end + + function handle:outcome_op() + return local_rx:recv_op():wrap(function (ev, recv_err) + if ev == nil then + return { + kind = 'active_job_done', + service_id = service_id, + generation = generation, + job_id = job_id, + phase = phase, + token = token, + status = 'failed', + primary = recv_err or 'active job observer closed', + } + end + return ev + end) + end + + function handle:outcome() + return raw_handle:outcome() + end + + function handle:identity() + return raw_handle:identity() + end + end + + return handle, nil +end + +function M.start_work(lifetime_scope, state, spec) + spec = spec or {} + local lease = spec.lease + if type(lease) ~= 'table' or type(lease.start_work) ~= 'function' then + return nil, 'active lease required' + end + if lease._state ~= state then + return nil, 'active lease belongs to another state' + end + return lease:start_work(lifetime_scope, spec) +end + +function M.cancel_active(state, reason) + local active = state and state.active or nil + local handle = active and active.handle or nil + if handle and handle.cancel then + return handle:cancel(reason or 'active_cancelled') + end + return false, 'no_active' +end + +function M.apply_completion(state, ev) + if not ev or ev.kind ~= 'active_job_done' then + return false, 'not_active_completion' + end + + local active = state and state.active or nil + if not active or active.token ~= ev.token then + if state and state.stats then + state.stats.stale = state.stats.stale + 1 + end + return false, 'stale' + end + + local stored = copy(ev) + state.last_completion = stored + + active.status = 'completed_pending_persist' + active.handle = nil + active.completion = stored + state.stats.completed = state.stats.completed + 1 + if ev.status == 'failed' then + state.stats.failed = state.stats.failed + 1 + elseif ev.status == 'cancelled' then + state.stats.cancelled = state.stats.cancelled + 1 + end + + return true, nil +end + +function M.release_completed(state, token, reason) + local active = state and state.active or nil + if not active or active.token ~= token then + return false, 'stale' + end + if active.status ~= 'completed_pending_persist' then + return false, 'not_completed' + end + state.active = nil + state.stats.released = state.stats.released + 1 + return true, reason +end + +function M.last_completion(state) + return state and state.last_completion and copy(state.last_completion) or nil +end + +function M.completion(state, token) + if not state or token == nil then return nil end + local active = state.active + if active and active.token == token and active.completion then + return copy(active.completion) + end + local last = state.last_completion + if last and last.token == token then return copy(last) end + return nil +end + + +---------------------------------------------------------------------- +-- Service-owned component wrapper +---------------------------------------------------------------------- + +local Component = {} +Component.__index = Component + +local function component_identity(service_id) + return { + kind = 'component_done', + service_id = service_id, + component = 'active_runtime', + } +end + +local function is_stale_apply_reason(reason) + return reason == 'not_active' + or reason == 'stale_active_token' + or reason == 'not_found' +end + +local function active_intent_for(job) + return job and (job.active_intent or job.active) or nil +end + +local function positive_number(v) + local n = tonumber(v) + if type(n) ~= 'number' or n <= 0 then return nil end + return n +end + +local function configured_phase_timeout_s(self, component, phase) + local cfg = self and self._config or nil + local components = cfg and cfg.components or nil + local rec = type(components) == 'table' and component and components[component] or nil + if type(rec) ~= 'table' then return nil end + phase = phase or 'stage' + return positive_number(rec[phase .. '_timeout_s']) + or positive_number(rec.timeout_s) +end + +local function phase_deadline(self, job, phase) + local timeout_s = configured_phase_timeout_s(self, job and job.component, phase) + if timeout_s == nil then return nil end + return fibers.now() + timeout_s +end + +local function reconcile_token_for(job, generation) + return table.concat({ + tostring(generation or 0), + tostring(job and job.job_id or ''), + 'reconcile', + tostring(job and job.updated_seq or 0), + }, ':') +end + +function Component:state() + return self._state +end + +function Component:snapshot() + return M.snapshot(self._state) +end + +function Component:claim(rec) + return M.claim(self._state, rec) +end + +function Component:_report_to_service(ev, label) + local ok, err = queue.try_admit_required( + self._service_done_tx, + ev, + label or 'update_active_runtime_report_failed' + ) + if ok ~= true then return nil, err end + return true, nil +end + +function Component:_report_changed(reason, extra) + local ev = { + kind = 'active_runtime_changed', + service_id = self._service_id, + reason = reason, + snapshot = M.snapshot(self._state), + } + for k, v in pairs(extra or {}) do + ev[k] = v + end + return self:_report_to_service(ev, 'update_active_runtime_changed_report_failed') +end + +function Component:_start_apply(ev) + if not (self._jobs and type(self._jobs.admit_transition) == 'function') then + return nil, 'job_runtime_unavailable' + end + + local token = ev.token or ev.job_id + if self._active_applies[token] ~= nil then + return false, 'already_applying' + end + + local handle, err = scoped_work.start { + lifetime_scope = self._work_scope, + reaper_scope = self._work_scope, + report_scope = self._work_scope, + + identity = { + kind = 'active_job_apply_done', + service_id = self._service_id, + generation = ev.generation, + job_id = ev.job_id, + phase = ev.phase, + token = ev.token, + }, + + run = function () + local handle, admit_err = self._jobs:admit_transition { + kind = 'apply_active_result', + generation = ev.generation, + job_id = ev.job_id, + phase = ev.phase, + token = ev.token, + event = ev, + } + if not handle then error(admit_err or 'active_job_apply_admission_failed', 0) end + local result, jerr = fibers.perform(handle:outcome_op()) + if not result or result.status ~= 'persisted' then + error(jerr or (result and result.reason) or 'active_job_apply_failed', 0) + end + return result + end, + + report = function (done_ev) + local ok, report_err = queue.try_admit_required( + self._local_tx, + done_ev, + 'update_active_job_apply_completion_report_failed' + ) + if ok == true then return true, nil end + + -- During active-runtime cancellation, apply workers may be cancelled after + -- the local coordinator queue has already been closed. That is expected + -- cleanup, not a lost healthy completion. + if self._closing_reason ~= nil then + return true, nil + end + + return nil, report_err + end, + } + + if not handle then return nil, err end + self._active_applies[token] = handle + return handle, nil +end + +local function job_policy(job) + return type(job) == 'table' and type(job.policy) == 'table' and job.policy or {} +end + +local function policy_auto_commit(job) + return job_policy(job).commit == 'auto' +end + +function Component:_start_policy_commit(job) + if not (job and job.job_id) then return nil, 'not_ready' end + if job.state ~= 'awaiting_commit' then return false, 'not_awaiting_commit' end + if not policy_auto_commit(job) then return false, 'manual_commit' end + if not (self._jobs and type(self._jobs.admit_transition) == 'function') then return nil, 'job_runtime_unavailable' end + local handle, admit_err = self._jobs:admit_transition { + kind = 'start_job', + generation = job.generation or self._current_generation, + job_id = job.job_id, + phase = 'commit', + reason = 'policy_auto_commit', + } + if not handle then return nil, admit_err or 'policy_auto_commit_admission_failed' end + local result, jerr = fibers.perform(handle:outcome_op()) + if not result or result.status ~= 'persisted' then + return nil, jerr or (result and result.reason) or 'policy_auto_commit_persist_failed' + end + local ok_report, report_err = self:_report_changed('policy_auto_commit_started', { + job_id = job.job_id, + phase = 'commit', + token = result.token, + }) + if ok_report ~= true then return nil, report_err end + return true, nil +end + +function Component:_start_reconcile(job) + if not (job and job.job_id) then return nil, 'not_ready' end + if self._state.active ~= nil then return nil, 'slot_busy' end + + self._adoption_reconcile_started = self._adoption_reconcile_started or {} + if self._adoption_reconcile_started[job.job_id] then + return false, 'already_started' + end + + local generation = job.generation or self._current_generation + local token = reconcile_token_for(job, generation) + self._adoption_reconcile_started[job.job_id] = true + + local handle, err = self:start_intent({ + service_id = self._service_id, + job_id = job.job_id, + generation = generation, + phase = 'reconcile', + token = token, + }, job, { + backend = self._backend, + phase = 'reconcile', + }) + + if not handle then + self._adoption_reconcile_started[job.job_id] = nil + return nil, err + end + + return handle, nil +end + +function Component:_start_restart_adoption() + local adoption = self._adoption or {} + for _, rec in ipairs(adoption.awaiting_return or {}) do + local job = self._jobs and self._jobs:get(rec.job_id) or nil + if job and job.state == 'awaiting_return' then + local ok, err = self:_start_reconcile(job) + if ok == nil and err ~= 'slot_busy' and err ~= 'reconcile_backend_unavailable' then + return nil, err or 'restart_reconcile_start_failed' + end + if ok ~= nil then + return ok, err + end + end + end + return false, 'no_reconcile_adoption' +end + +function Component:_launch_active_intent(job) + if not (job and job.job_id) then return nil, 'not_ready' end + local intent = active_intent_for(job) + if type(intent) ~= 'table' or intent.token == nil or intent.phase == nil then + return false, 'no_active_intent' + end + if self._state.active ~= nil then return nil, 'slot_busy' end + self._active_launched = self._active_launched or {} + if self._active_launched[intent.token] then + return false, 'already_started' + end + + self._active_launched[intent.token] = true + local handle, err = self:start_intent({ + service_id = self._service_id, + job_id = job.job_id, + generation = intent.generation or job.generation, + phase = intent.phase, + token = intent.token, + }, job, { + backend = self._backend, + phase = intent.phase, + deadline = phase_deadline(self, job, intent.phase), + }) + + if not handle then + self._active_launched[intent.token] = nil + return nil, err + end + + local ok_report, report_err = self:_report_changed('active_intent_started', { + job_id = job.job_id, + token = intent.token, + phase = intent.phase, + }) + if ok_report ~= true then return nil, report_err end + return handle, nil +end + +function Component:update_adoption(adoption) + self._adoption = copy(adoption or {}) + return true, nil +end + +function Component:update_config(config) + self._config = copy(config or {}) + return true, nil +end + +function Component:consider_jobs() + if not self._jobs then return false, 'not_ready' end + if self._state.active ~= nil then return false, 'slot_busy' end + + for _, job in ipairs(self._jobs:list()) do + local intent = active_intent_for(job) + if type(intent) == 'table' and intent.token ~= nil and intent.phase ~= nil then + local ok, err = self:_launch_active_intent(job) + if ok ~= nil then return ok, err end + if err == 'slot_busy' then return nil, err end + return nil, err or 'active_intent_launch_failed' + end + end + + local ok, err = self:_start_restart_adoption() + if ok ~= nil then return ok, err end + if err == 'slot_busy' or err == 'no_reconcile_adoption' or err == 'not_ready' then + return false, 'no_active_intent' + end + return nil, err +end + +function Component:start_intent(intent, job, spec) + intent = intent or {} + spec = spec or {} + if self._state.active ~= nil then + return nil, 'slot_busy' + end + local lease, lerr = M.claim(self._state, { + service_id = intent.service_id or self._service_id, + job_id = intent.job_id or (job and job.job_id), + generation = intent.generation or (job and job.generation), + phase = intent.phase or 'stage', + token = intent.token, + }) + if not lease then return nil, lerr end + + spec.lease = lease + spec.job = job + spec.phase = intent.phase or spec.phase or lease.phase + spec.jobs = spec.jobs or self._jobs + spec.done_tx = self._local_tx + local handle, herr = M.start_work(self._work_scope, self._state, spec) + if not handle then + lease:release(herr or 'active_intent_start_failed') + return nil, herr + end + return handle, nil +end + +function Component:start_work(spec) + spec = spec or {} + spec.done_tx = self._local_tx + return M.start_work(self._work_scope, self._state, spec) +end + +function Component:cancel_active(reason) + return M.cancel_active(self._state, reason) +end + +function Component:release_completed(token, reason) + return M.release_completed(self._state, token, reason) +end + +function Component:terminate(reason) + return self:cancel(reason or 'active_runtime_terminated') +end + +function Component:cancel(reason) + reason = reason or 'active_runtime_cancelled' + self._closing_reason = self._closing_reason or reason + + -- The component owns both the coordinator fibre and any active/apply work it + -- has started. Cancelling only the coordinator handle can leave scoped work + -- running under the component's work scope, which is especially visible when + -- an apply worker is blocked waiting for job_runtime. + local active = self._state and self._state.active or nil + if active and active.handle and active.handle.cancel then + active.handle:cancel(reason) + end + + for token, handle in pairs(self._active_applies or {}) do + if handle and handle.cancel then + handle:cancel(reason) + end + self._active_applies[token] = nil + end + + if self._handle and self._handle.cancel then + self._handle:cancel(reason) + else + self._local_tx:close(reason) + end + + return true +end + +function Component:_handle_active_done(ev) + local applied, aerr = M.apply_completion(self._state, ev) + if not applied then + if aerr == 'stale' then + return true, nil + end + return nil, aerr or 'active_runtime_apply_failed' + end + + local ok_change, cerr = self:_report_changed('active_job_completed', { + job_id = ev.job_id, + phase = ev.phase, + token = ev.token, + }) + if ok_change ~= true then return nil, cerr end + + local handle, err = self:_start_apply(ev) + if handle == nil and err ~= nil then + return nil, err + end + return true, nil +end + +function Component:_handle_apply_done(ev) + local token = ev.token or ev.job_id + self._active_applies[token] = nil + + if ev.status ~= 'ok' then + local reason = ev.primary or 'active_job_apply_failed' + M.release_completed(self._state, ev.token, reason) + local ok_report, report_err = self:_report_changed('active_job_apply_failed', { + job_id = ev.job_id, + phase = ev.phase, + token = ev.token, + error = reason, + }) + if ok_report ~= true then return nil, report_err end + + if is_stale_apply_reason(reason) then + return true, nil + end + return nil, reason + end + + local result = ev.result or {} + local active_ev = result.active or { + kind = 'active_job_done', + service_id = ev.service_id, + generation = ev.generation, + job_id = ev.job_id, + phase = ev.phase, + token = ev.token, + status = ev.status, + } + active_ev.persisted_job = result.job + active_ev.persistence_owner = 'job_runtime' + + M.release_completed(self._state, ev.token, 'durable_apply_complete') + + local ok_change, cerr = self:_report_changed('active_job_applied', { + job_id = ev.job_id, + phase = ev.phase, + token = ev.token, + active = active_ev, + }) + if ok_change ~= true then return nil, cerr end + + local job = result.job + if job and job.state == 'awaiting_commit' then + local ok_commit, commit_err = self:_start_policy_commit(job) + if ok_commit == nil and commit_err ~= 'manual_commit' then + return nil, commit_err or 'policy_auto_commit_failed' + end + end + if job and job.state == 'awaiting_return' then + local ok_rec, rerr = self:_start_reconcile(job) + if ok_rec == nil and rerr ~= 'slot_busy' and rerr ~= 'reconcile_backend_unavailable' then + return nil, rerr or 'reconcile_start_failed' + end + end + + local ok_launch, launch_err = self:consider_jobs() + if ok_launch == nil and launch_err ~= 'slot_busy' and launch_err ~= 'no_active_intent' and launch_err ~= 'not_ready' then + return nil, launch_err or 'active_intent_launch_failed' + end + + return true, nil +end + +function Component:_handle_local_event(ev) + if ev.kind == 'active_job_done' then + return self:_handle_active_done(ev) + end + if ev.kind == 'active_job_apply_done' then + return self:_handle_apply_done(ev) + end + return nil, 'unknown_active_runtime_event: ' .. tostring(ev.kind) +end + +function Component:run(scope) + scope:finally(function (_, status, primary) + self._local_tx:close(primary or status or 'active_runtime_closed') + end) + + while true do + local ev = fibers.perform(self._local_rx:recv_op()) + if ev == nil then + return { + tag = 'active_runtime_stopped', + snapshot = M.snapshot(self._state), + } + end + + local ok, err = self:_handle_local_event(ev) + if ok ~= true then + error(err or 'active_runtime_event_failed', 0) + end + end +end + +--- Start the service-owned active-runtime component. +--- +--- The component stores active-job completion before reporting any state change +--- to the service coordinator, applies the completion through job_runtime, and +--- releases the active slot only after durable apply. +function M.start_component(scope, params) + params = params or {} + local service_done_tx = assert(params.done_tx, 'active_runtime.start_component: done_tx required') + local work_scope = assert(params.work_scope or scope, 'active_runtime.start_component: work_scope required') + local local_tx, local_rx = mailbox.new(params.queue_len or 32, { full = 'reject_newest' }) + local component = setmetatable({ + _state = params.state or M.new_state(), + _local_tx = local_tx, + _local_rx = local_rx, + _service_done_tx = service_done_tx, + _service_id = params.service_id or 'update', + _work_scope = work_scope, + _jobs = params.jobs, + _backend = params.backend, + _observer = params.observer, + _config = copy(params.config or {}), + _adoption = params.adoption or {}, + _active_applies = {}, + _active_launched = {}, + _adoption_reconcile_started = {}, + }, Component) + + local handle, err = scoped_work.start { + lifetime_scope = scope, + reaper_scope = scope, + report_scope = scope, + identity = component_identity(component._service_id), + run = function (component_scope) + return component:run(component_scope) + end, + report = function (ev) + return queue.try_admit_required( + service_done_tx, + ev, + 'update_active_runtime_component_completion_report_failed' + ) + end, + } + if not handle then + local_tx:close(err or 'active_runtime_start_failed') + return nil, err + end + + component._handle = handle + return component, nil +end + +M.Component = Component + +M.State = State +M.Lease = Lease + +return M diff --git a/src/services/update/artifacts/dcmcu.lua b/src/services/update/artifacts/dcmcu.lua new file mode 100644 index 00000000..de4d6845 --- /dev/null +++ b/src/services/update/artifacts/dcmcu.lua @@ -0,0 +1,143 @@ +-- services/update/artifacts/dcmcu.lua +-- +-- Strict .dcmcu v1 identity parser used by the Lua update manager before +-- admitting an MCU update job. The MCU remains the authority for full image +-- verification; this parser extracts only the canonical job identity. + +local fibers = require 'fibers' +local cjson = require 'cjson.safe' +local resource = require 'devicecode.support.resource' + +local M = {} + +local MAGIC = 'DCMCUIMG' +local FORMAT_VERSION = 1 +local MIN_HEADER_LEN = 16 +local MAX_HEADER_LEN = 4096 +local MAX_MANIFEST_LEN = 1024 * 1024 + +local function u16le(s, i) + local b1, b2 = s:byte(i, i + 1) + if b1 == nil or b2 == nil then return nil end + return b1 + b2 * 0x100 +end + +local function u32le(s, i) + local b1, b2, b3, b4 = s:byte(i, i + 3) + if b1 == nil or b2 == nil or b3 == nil or b4 == nil then return nil end + return b1 + b2 * 0x100 + b3 * 0x10000 + b4 * 0x1000000 +end + +local function non_empty_string(v) + return type(v) == 'string' and v ~= '' +end + +local function is_lower_hex_64(v) + return type(v) == 'string' and #v == 64 and v:match('^[0-9a-f]+$') ~= nil +end + +local function read_exact(source, n) + local parts = {} + local have = 0 + while have < n do + local chunk, err = fibers.perform(source:read_chunk_op(n - have)) + if err ~= nil then return nil, err end + if chunk == nil then return nil, 'dcmcu_unexpected_eof' end + if type(chunk) ~= 'string' then return nil, 'dcmcu_source_chunk_not_string' end + if chunk == '' then return nil, 'dcmcu_empty_source_chunk' end + parts[#parts + 1] = chunk + have = have + #chunk + end + return table.concat(parts), nil +end + +local function manifest_identity(manifest) + if type(manifest) ~= 'table' then return nil, 'dcmcu_manifest_not_object' end + if manifest.schema ~= 1 then return nil, 'dcmcu_manifest_schema_unsupported' end + if manifest.component ~= 'mcu' then return nil, 'dcmcu_component_not_mcu' end + + local build = manifest.build + if type(build) ~= 'table' then return nil, 'dcmcu_build_required' end + if not non_empty_string(build.image_id) then return nil, 'dcmcu_image_id_required' end + + local payload = manifest.payload + if type(payload) ~= 'table' then return nil, 'dcmcu_payload_required' end + if type(payload.length) ~= 'number' or payload.length < 0 or payload.length ~= math.floor(payload.length) then + return nil, 'dcmcu_payload_length_invalid' + end + if not is_lower_hex_64(payload.sha256) then return nil, 'dcmcu_payload_sha256_invalid' end + + return { + format = 'dcmcu-v1', + component = 'mcu', + image_id = build.image_id, + version = non_empty_string(build.version) and build.version or nil, + build_id = non_empty_string(build.build_id) and build.build_id or nil, + payload_length = payload.length, + payload_sha256 = payload.sha256, + }, nil +end + +function M.identity_from_header_bytes(raw) + if type(raw) ~= 'string' then return nil, 'dcmcu_bytes_required' end + if #raw < MIN_HEADER_LEN then return nil, 'dcmcu_header_too_short' end + if raw:sub(1, #MAGIC) ~= MAGIC then return nil, 'dcmcu_bad_magic' end + + local version = u16le(raw, 9) + local header_len = u16le(raw, 11) + local manifest_len = u32le(raw, 13) + if version ~= FORMAT_VERSION then return nil, 'dcmcu_version_unsupported' end + if type(header_len) ~= 'number' or header_len < MIN_HEADER_LEN or header_len > MAX_HEADER_LEN then + return nil, 'dcmcu_header_len_invalid' + end + if type(manifest_len) ~= 'number' or manifest_len <= 0 or manifest_len > MAX_MANIFEST_LEN then + return nil, 'dcmcu_manifest_len_invalid' + end + if #raw < header_len + manifest_len then return nil, 'dcmcu_manifest_incomplete' end + + local manifest_raw = raw:sub(header_len + 1, header_len + manifest_len) + local manifest, err = cjson.decode(manifest_raw) + if manifest == nil then return nil, 'dcmcu_manifest_json_invalid:' .. tostring(err or 'decode_failed') end + return manifest_identity(manifest) +end + +function M.identity_from_source_op(source) + return fibers.run_scope_op(function (scope) + if type(source) ~= 'table' or type(source.read_chunk_op) ~= 'function' then + return nil, 'dcmcu_source_required' + end + scope:finally(function (_, status, primary) + resource.terminate_checked(source, primary or status or 'dcmcu source closed', 'dcmcu source cleanup failed') + end) + + local header, herr = read_exact(source, MIN_HEADER_LEN) + if not header then return nil, herr end + local header_len = u16le(header, 11) + local manifest_len = u32le(header, 13) + if type(header_len) ~= 'number' or type(manifest_len) ~= 'number' then + return nil, 'dcmcu_header_invalid' + end + if header_len < MIN_HEADER_LEN or header_len > MAX_HEADER_LEN then + return nil, 'dcmcu_header_len_invalid' + end + if manifest_len <= 0 or manifest_len > MAX_MANIFEST_LEN then + return nil, 'dcmcu_manifest_len_invalid' + end + + local rest_len = header_len + manifest_len - #header + local rest, rerr = read_exact(source, rest_len) + if not rest then return nil, rerr end + return M.identity_from_header_bytes(header .. rest) + end):wrap(function (st, report, identity, err) + if st == 'ok' then return identity, err end + return nil, identity or err or report or st or 'dcmcu_identity_failed' + end) +end + +M._test = { + u16le = u16le, + u32le = u32le, + manifest_identity = manifest_identity, +} + +return M diff --git a/src/services/update/artifacts/lifetime.lua b/src/services/update/artifacts/lifetime.lua new file mode 100644 index 00000000..d54ee970 --- /dev/null +++ b/src/services/update/artifacts/lifetime.lua @@ -0,0 +1,99 @@ +-- services/update/artifacts/lifetime.lua +-- +-- Artifact-specific finaliser-safe ownership helper. +-- +-- Scope-owned artifact and sink resources must expose: +-- terminate(reason) immediate, idempotent, non-yielding cleanup +-- +-- Graceful Op-based close paths belong in workers before finalisation. Legacy cleanup names are adapted at capability/test boundaries, not in +-- this ownership helper. + +local resource = require 'devicecode.support.resource' + +local M = {} + +local Owned = {} +Owned.__index = Owned + +local function has_terminate(obj) + return type(obj) == 'table' and type(obj.terminate) == 'function' +end + +function M.has_immediate_cleanup(obj) + return has_terminate(obj) +end + +function M.own(scope, obj, opts) + opts = opts or {} + if type(scope) ~= 'table' or type(scope.finally) ~= 'function' then + error('artifact lifetime requires a scope', 2) + end + if type(obj) ~= 'table' then + return nil, 'resource required' + end + if not has_terminate(obj) then + if type(obj['close' .. '_op']) == 'function' then + return nil, 'resource exposes close' .. '_op but no terminate(reason) method' + end + return nil, 'resource has no terminate(reason) method' + end + + local owner = resource.owned(obj, { + label = opts.label or 'artifact cleanup', + }) + + local self = setmetatable({ + _owner = owner, + _reason = opts.reason or 'artifact scope terminated', + _label = opts.label or 'artifact cleanup', + }, Owned) + + scope:finally(function (_, status, primary) + owner:terminate_checked(primary or status or self._reason, self._label) + end) + + return self, nil +end + +function Owned:resource() + return self._owner:value() +end + +function Owned:is_owned() + return self._owner:is_owned() +end + +function Owned:terminate(reason) + return self._owner:terminate(reason or self._reason) +end + +function Owned:terminate_checked(reason) + return self._owner:terminate_checked(reason or self._reason, self._label) +end + +function Owned:detach() + return self._owner:detach() +end + +function Owned:handoff(receiver_install) + return self._owner:handoff(receiver_install) +end + +function Owned:append_op(chunk) + local sink = self:resource() + if type(sink) ~= 'table' or type(sink.append_op) ~= 'function' then + return nil, 'append_op not supported' + end + return sink:append_op(chunk) +end + +function Owned:commit_op(...) + local sink = self:resource() + if type(sink) ~= 'table' or type(sink.commit_op) ~= 'function' then + return nil, 'commit_op not supported' + end + return sink:commit_op(...) +end + +M.Owned = Owned +return M diff --git a/src/services/update/artifacts/store_bus.lua b/src/services/update/artifacts/store_bus.lua new file mode 100644 index 00000000..238b1b56 --- /dev/null +++ b/src/services/update/artifacts/store_bus.lua @@ -0,0 +1,148 @@ +-- services/update/artifacts/store_bus.lua +-- +-- Bus-backed artifact store client. This is the Update-side adapter for the +-- curated artifact-store capability surface; callers see only Ops and artifact +-- handles/sources. + +local fibers = require 'fibers' +local op = require 'fibers.op' + +local cap_args = require 'services.hal.types.capability_args' + +local M = {} + +local Store = {} +Store.__index = Store + +local function copy(v) + if type(v) ~= 'table' then return v end + local out = {} + for k, val in pairs(v) do out[k] = copy(val) end + return out +end + +local function rpc_topic(id, method) + return { 'cap', 'artifact-store', id or 'main', 'rpc', method } +end + +local VOID_SUCCESS = { + ['delete'] = true, +} + +local function unwrap_reply_for(method) + return function (reply, err) + if reply == nil then + return nil, err + end + if type(reply) ~= 'table' or type(reply.ok) ~= 'boolean' then + return nil, 'invalid_artifact_store_reply' + end + if reply.ok then + if reply.reason == nil and VOID_SUCCESS[method] then + return true, nil + end + return reply.reason, nil + end + return nil, tostring(reply.reason or err or 'artifact_store_call_failed') + end +end + +local function call_op(self, method, payload, opts) + if type(self._conn) ~= 'table' or type(self._conn.call_op) ~= 'function' then + return op.always(nil, 'artifact_store_bus_connection_required') + end + return self._conn:call_op( + rpc_topic(self._id, method), + payload or {}, + opts or self._call_opts + ):wrap(function (reply, err) + return unwrap_reply_for(method)(reply, err) + end) +end + +local function unwrap_scope_value(label) + return function (st, report, value, err) + if st == 'ok' then + if value == nil then return nil, err or (label and (label .. '_failed')) or 'operation_failed' end + return value, err + end + return nil, value or err or report or st or (label and (label .. '_failed')) or 'operation_failed' + end +end + +function Store:create_sink_op(spec, opts) + spec = copy(spec or {}) + opts = opts or {} + local payload, perr = cap_args.new.ArtifactStoreCreateSinkOpts( + spec.meta or spec.metadata or spec, + spec.policy or opts.policy + ) + if payload == nil then return op.always(nil, perr or 'invalid create sink opts') end + return call_op(self, 'create-sink', payload, opts) +end + +function Store:import_path_op(path, meta, opts) + if type(path) == 'table' then + opts = meta + meta = path.meta or path.metadata + path = path.path + end + opts = opts or {} + local payload, perr = cap_args.new.ArtifactStoreImportPathOpts(path, meta or {}, opts.policy) + if payload == nil then return op.always(nil, perr or 'invalid import path opts') end + return call_op(self, 'import-path', payload, opts) +end + +function Store:import_source_op(source, meta, opts) + opts = opts or {} + local payload, perr = cap_args.new.ArtifactStoreImportSourceOpts(source, meta or {}, opts.policy) + if payload == nil then return op.always(nil, perr or 'invalid import source opts') end + return call_op(self, 'import-source', payload, opts) +end + +function Store:open_op(ref) + if type(ref) == 'table' then ref = ref.ref or ref.artifact_ref or ref.id end + local payload, perr = cap_args.new.ArtifactStoreOpenOpts(ref) + if payload == nil then return op.always(nil, perr or 'invalid artifact ref') end + return call_op(self, 'open', payload) +end + +function Store:open_source_op(ref) + return fibers.run_scope_op(function () + local artifact, err = fibers.perform(self:open_op(ref)) + if artifact == nil then return nil, err or 'artifact_open_failed' end + if type(artifact.open_source_op) ~= 'function' then + return nil, 'artifact_handle_has_no_open_source_op' + end + local ok_source, source_or_err = fibers.perform(artifact:open_source_op()) + if ok_source ~= true then + return nil, source_or_err or 'artifact_source_open_failed' + end + return source_or_err, nil + end):wrap(unwrap_scope_value('artifact_source_open')) +end + +function Store:delete_op(ref, opts) + if type(ref) == 'table' then ref = ref.ref or ref.artifact_ref or ref.id end + local payload, perr = cap_args.new.ArtifactStoreDeleteOpts(ref) + if payload == nil then return op.always(nil, perr or 'invalid artifact ref') end + return call_op(self, 'delete', payload, opts) +end + +function Store:status_op(opts) + local payload, perr = cap_args.new.ArtifactStoreStatusOpts() + if payload == nil then return op.always(nil, perr or 'invalid status opts') end + return call_op(self, 'status', payload, opts) +end + +function M.new(conn, opts) + opts = opts or {} + return setmetatable({ + _conn = conn, + _id = opts.id or 'main', + _call_opts = opts.call_opts, + }, Store) +end + +M.Store = Store +return M diff --git a/src/services/update/backends/component.lua b/src/services/update/backends/component.lua new file mode 100644 index 00000000..8781edc3 --- /dev/null +++ b/src/services/update/backends/component.lua @@ -0,0 +1,443 @@ +-- services/update/backends/component.lua +-- +-- Canonical component update backend. It knows Device's curated component +-- state/capability surface only; it has no Fabric, UART or raw-member knowledge. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' + +local model = require 'services.update.model' +local topics = require 'services.update.topics' + +local M = {} +local Backend = {} +Backend.__index = Backend + +local function copy(v) return model.deep_copy(v) end + +local function component_of(self, job) + return (type(job) == 'table' and job.component) or self._component +end + +local function metadata_of(job) + return type(job) == 'table' and type(job.metadata) == 'table' and job.metadata or {} +end + +local function required_string(v, field) + if type(v) ~= 'string' or v == '' then + return nil, field .. '_required' + end + return v, nil +end + +local function job_expected_image_id(job) + return required_string(type(job) == 'table' and job.expected_image_id or nil, 'expected_image_id') +end + +local function artifact_record(job) + if type(job) ~= 'table' then return nil end + return job.artifact or job.artifact_snapshot or job.artifact_meta +end + +local function artifact_ref(job) + if type(job) ~= 'table' then return nil end + local art = artifact_record(job) + return job.artifact_ref + or job.ref + or (type(art) == 'table' and (art.artifact_ref or art.ref or art.id)) +end + +local function transfer_from(job, ctx) + local stage = type(job) == 'table' and (job.stage_result or job.staged or job.stage) or nil + local pf = ctx and ctx.preflight or nil + local art = artifact_record(job) + local t = type(stage) == 'table' and (stage.transfer or stage) or nil + return { + digest_alg = (t and t.digest_alg) or (pf and pf.digest_alg) or (art and art.digest_alg) or 'xxhash32', + digest = (t and t.digest) or (pf and pf.digest) or (art and (art.digest or art.checksum)), + size = (t and t.size) or (pf and pf.size) or (art and art.size), + } +end + +local NO_BUS_TIMEOUT = { timeout = false } + +local function call_component_op(self, component, method, payload, opts) + if type(self._conn) ~= 'table' or type(self._conn.call_op) ~= 'function' then + return op.always(nil, 'component_backend_connection_required') + end + -- Component update calls may legitimately take as long as the update phase + -- owns them for. Do not use lua-bus' default one-second call timeout here; + -- callers that own a budget must compose this Op with a sleep/deadline Op. + -- If that outer choice loses, lua-bus observes the abort and abandons the + -- request through the Request owner path. + return self._conn:call_op(topics.component_rpc(component, method), payload, opts or self._call_opts or NO_BUS_TIMEOUT):wrap(function (reply, err) + if reply == false then return nil, err or 'component_call_failed' end + return reply, err + end) +end + +local function retry_budget(self, ctx, method) + local cfg = type(self._rpc_retry) == 'table' and self._rpc_retry or {} + local attempts = tonumber(cfg.attempts) or tonumber(cfg.max_attempts) or 3 + if method == 'stage-update' then + attempts = tonumber(cfg.stage_attempts) or attempts + elseif method == 'commit-update' then + attempts = tonumber(cfg.commit_attempts) or attempts + elseif method == 'prepare-update' then + attempts = tonumber(cfg.prepare_attempts) or attempts + end + if attempts < 1 then attempts = 1 end + return math.floor(attempts) +end + +local function retry_delay_s(self, attempt) + local cfg = type(self._rpc_retry) == 'table' and self._rpc_retry or {} + local base = tonumber(cfg.delay_s) or 0.20 + local max = tonumber(cfg.max_delay_s) or 1.00 + local n = base * (2 ^ math.max(0, attempt - 1)) + if n > max then n = max end + return n +end + +local function deadline_remaining(ctx) + local deadline = ctx and ctx.deadline + if type(deadline) ~= 'number' then return nil end + local rem = deadline - fibers.now() + if rem <= 0 then return 0 end + return rem +end + +local function call_component_retry_op(self, component, method, payload, ctx, opts) + return fibers.run_scope_op(function () + local attempts = retry_budget(self, ctx, method) + local last_err + for attempt = 1, attempts do + local rem = deadline_remaining(ctx) + if rem ~= nil and rem <= 0 then + return nil, (method .. '_timeout') + end + + local reply, err + if rem ~= nil then + local which, a, b = fibers.perform(fibers.named_choice { + call = call_component_op(self, component, method, payload, opts), + timeout = sleep.sleep_op(rem), + }) + if which == 'timeout' then + return nil, (method .. '_timeout') + end + reply, err = a, b + else + reply, err = fibers.perform(call_component_op(self, component, method, payload, opts)) + end + + if reply ~= nil then + return reply, nil, attempt + end + last_err = err or (method .. '_failed') + if attempt >= attempts then break end + + local delay = retry_delay_s(self, attempt) + local rem2 = deadline_remaining(ctx) + if rem2 ~= nil then + if rem2 <= 0 then return nil, (method .. '_timeout') end + if delay > rem2 then delay = rem2 end + end + if delay > 0 then fibers.perform(sleep.sleep_op(delay)) end + end + return nil, last_err or (method .. '_failed') + end):wrap(function (st, report, value, err) + if st == 'ok' then + return value, err + end + return nil, value or err or report or st or (method .. '_failed') + end) +end + +local function call_component_once_op(self, component, method, payload, ctx, opts) + return fibers.run_scope_op(function () + local rem = deadline_remaining(ctx) + if rem ~= nil and rem <= 0 then return nil, method .. '_timeout' end + if rem ~= nil then + local which, a, b = fibers.perform(fibers.named_choice { + call = call_component_op(self, component, method, payload, opts), + timeout = sleep.sleep_op(rem), + }) + if which == 'timeout' then return nil, method .. '_timeout' end + return a, b + end + return fibers.perform(call_component_op(self, component, method, payload, opts)) + end):wrap(function (st, report, value, err) + if st == 'ok' then return value, err end + return nil, value or err or report or st or (method .. '_failed') + end) +end + + +local function commit_response_may_be_missing(err) + -- Only timeouts/closed response paths are ambiguous enough to treat as + -- "commit may have reached the MCU; response may have been lost". + -- Definite admission/routing failures such as link_not_ready, no_route or + -- no_session must remain ordinary failures and must not advance the job to + -- awaiting_return. + if err == nil or err == '' then return true end + if err == 'timeout' or err == 'commit-update_timeout' then return true end + if err == 'bus_call_closed' or err == 'local_call_closed' or err == 'reply_closed' then return true end + return false +end + +local function commit_reply_ok(reply) + if type(reply) ~= 'table' then return nil, 'invalid_commit_reply' end + if reply.ok == false then return nil, reply.err or reply.error or reply.reason or 'component_commit_update_failed' end + if reply.accepted == false then return nil, reply.err or reply.error or reply.reason or 'commit_not_accepted' end + if reply.accepted == true or reply.reboot_required ~= nil then return true, nil end + return nil, 'invalid_commit_reply' +end + + +local function unwrap_scope_value(scope_op, label) + return scope_op:wrap(function (st, report, value, err) + if st == 'ok' then + if value == nil then return nil, err or (label and (label .. '_failed')) or 'operation_failed' end + return value, err + end + return nil, value or err or report or st or (label and (label .. '_failed')) or 'operation_failed' + end) +end + +local function validate_stage_reply(reply) + if type(reply) ~= 'table' then + return nil, 'invalid_stage_reply' + end + if reply.ok == false then + return nil, reply.err or reply.error or reply.reason or 'component_stage_update_failed' + end + if reply.public_status ~= nil and reply.public_status ~= 'succeeded' then + return nil, reply.err or reply.error or reply.reason or reply.public_status + end + return true, nil +end + +local function describe_artifact(artifact) + if type(artifact) == 'table' and type(artifact.describe) == 'function' then + local ok, rec = pcall(function () return artifact:describe() end) + if ok and type(rec) == 'table' then return rec end + end + return type(artifact) == 'table' and artifact or nil +end + +function Backend:stage_op(job, ctx) + return unwrap_scope_value(fibers.run_scope_op(function () + local component = component_of(self, job) + local ref = artifact_ref(job) + if not ref then return nil, 'artifact_ref_required' end + if not self._artifact_store or type(self._artifact_store.open_op) ~= 'function' then + return nil, 'artifact_store_unavailable' + end + if type(self._artifact_store.open_source_op) ~= 'function' then + return nil, 'artifact_source_unavailable' + end + + local artifact, aerr = fibers.perform(self._artifact_store:open_op(ref)) + if artifact == nil then return nil, aerr or 'artifact_open_failed' end + local desc = describe_artifact(artifact) or {} + local meta = type(desc.meta) == 'table' and desc.meta or desc.metadata or {} + local image_id, iid_err = job_expected_image_id(job) + if not image_id then return nil, iid_err end + + local prepare_payload = { + job_id = job.job_id, + target = component, + expected_image_id = image_id, + metadata = metadata_of(job), + } + local prepared, perr = fibers.perform(call_component_retry_op(self, component, 'prepare-update', prepare_payload, ctx)) + if prepared == nil then return nil, perr or 'component_prepare_update_failed' end + + local stage_attempts = retry_budget(self, ctx, 'stage-update') + local reply, err + for attempt = 1, stage_attempts do + local source, serr = fibers.perform(self._artifact_store:open_source_op(ref)) + if source == nil then return nil, serr or 'artifact_source_open_failed' end + local payload = { + job_id = job.job_id, + expected_image_id = image_id, + source = source, + size = desc.size, + digest_alg = desc.digest_alg or 'xxhash32', + digest = desc.digest or desc.checksum, + chunk_size = self._chunk_size or 2048, + format = meta.format or desc.format or 'dcmcu-v1', + metadata = metadata_of(job), + } + reply, err = fibers.perform(call_component_once_op(self, component, 'stage-update', payload, ctx)) + if reply ~= nil then break end + if attempt >= stage_attempts then break end + local delay = retry_delay_s(self, attempt) + local rem = deadline_remaining(ctx) + if rem ~= nil then + if rem <= 0 then return nil, 'stage-update_timeout' end + if delay > rem then delay = rem end + end + if delay > 0 then fibers.perform(sleep.sleep_op(delay)) end + end + if reply == nil then return nil, err or 'component_stage_update_failed' end + local ok_reply, rerr = validate_stage_reply(reply) + if ok_reply ~= true then return nil, rerr end + local payload = { + digest_alg = desc.digest_alg or 'xxhash32', + digest = desc.digest or desc.checksum, + size = desc.size, + } + return { + staged = true, + component = component, + expected_image_id = image_id, + preflight = { + component = component, + artifact_ref = desc.artifact_ref or desc.ref or ref, + format = meta.format or desc.format or 'dcmcu-v1', + expected_image_id = image_id, + image_id = image_id, + size = desc.size, + digest_alg = desc.digest_alg or 'xxhash32', + digest = desc.digest or desc.checksum, + payload_sha256 = meta.payload_sha256 or desc.payload_sha256, + metadata = copy(meta), + }, + prepared = prepared, + transfer = { + digest_alg = payload.digest_alg, + digest = payload.digest, + size = payload.size, + }, + reply = reply, + }, nil + end), 'component_stage') +end + +local function component_snapshot(snapshot, component) + if type(snapshot) ~= 'table' then return nil end + local by_id = snapshot.by_id or snapshot.components + local rec = type(by_id) == 'table' and by_id[component] or nil + if type(rec) == 'table' and type(rec.state) == 'table' then return rec.state end + if type(rec) == 'table' then return rec end + if snapshot.component == component then return snapshot.state or snapshot end + return nil +end + +local function latest_component_state(self, component) + local obs = self._observer + if obs and type(obs.snapshot) == 'function' then + return component_snapshot(obs:snapshot(), component) + end + return nil +end + +local function updater_state(state) + if type(state) ~= 'table' then return nil end + return state.update or state.updater +end + +function Backend:pre_commit_record_op(job, ctx) + local component = component_of(self, job) + local state = latest_component_state(self, component) + local sw = state and state.software or nil + if type(sw) ~= 'table' then return op.always(nil, 'component_software_state_unavailable') end + if sw.boot_id == nil or sw.boot_id == '' then return op.always(nil, 'pre_commit_boot_id_required') end + local image_id, iid_err = job_expected_image_id(job) + if not image_id then return op.always(nil, iid_err) end + local transfer = transfer_from(job, ctx) + return op.always({ + component = component, + expected_image_id = image_id, + pre_commit_image_id = sw.image_id, + pre_commit_boot_id = sw.boot_id, + transfer = transfer, + }, nil) +end + +function Backend:commit_op(job, ctx) + local component = component_of(self, job) + local job_id, jid_err = required_string(type(job) == 'table' and job.job_id or nil, 'job_id') + if not job_id then return op.always(nil, jid_err) end + local image_id, iid_err = job_expected_image_id(job) + if not image_id then return op.always(nil, iid_err) end + local payload = { + job_id = job_id, + expected_image_id = image_id, + commit_token = ctx and ctx.commit_token or nil, + } + return call_component_retry_op(self, component, 'commit-update', payload, ctx):wrap(function (reply, err) + if reply == nil then + if not commit_response_may_be_missing(err) then + return nil, err or 'component_commit_update_failed' + end + -- Once commit has plausibly been submitted and the response path is + -- uncertain, the safe update-state transition is + -- awaiting_return/reconcile. A missing reply may mean the MCU accepted + -- and is rebooting. Reconcile will decide whether the image actually + -- took. + return { accepted = true, uncertain = true, reason = err or 'commit_response_missing' } + end + local ok_reply, rerr = commit_reply_ok(reply) + if ok_reply ~= true then return nil, rerr end + return { accepted = true, reply = reply } + end) +end + +function Backend:evaluate_reconcile(job, snapshot, ctx) + local component = component_of(self, job) + local state = component_snapshot(snapshot, component) or latest_component_state(self, component) + local sw = state and state.software or nil + local upd = updater_state(state) or {} + local pre = (job.commit_attempt and job.commit_attempt.pre_commit) + or (ctx and ctx.pre_commit) + local expected, iid_err = job_expected_image_id(job) + if not expected then + return { done = true, ok = false, reason = iid_err, state = copy(state) } + end + local pre_boot = pre and pre.pre_commit_boot_id + + if type(upd) == 'table' and (upd.state == 'failed' or upd.state == 'rollback_detected') then + return { done = true, ok = false, reason = upd.state, state = copy(state) } + end + + if type(sw) == 'table' and expected and sw.image_id == expected and pre_boot and sw.boot_id ~= pre_boot then + return { done = true, ok = true, state = copy(state) } + end + + local commit_result = type(job) == 'table' and type(job.commit_result) == 'table' and job.commit_result or nil + if type(sw) == 'table' and expected and sw.image_id == expected and not pre_boot + and commit_result and commit_result.tag == 'artifact_missing_reconcile' then + return { done = true, ok = true, state = copy(state), reason = 'artifact_missing_reconciled_by_image' } + end + + if type(sw) == 'table' and expected and pre_boot and sw.boot_id ~= nil and sw.boot_id ~= pre_boot and sw.image_id ~= expected then + return { done = true, ok = false, reason = 'wrong_image_after_reboot', state = copy(state) } + end + + return { done = false, reason = 'waiting_for_component_state', state = copy(state) } +end + +function Backend:commit_capabilities() + return { policy = self._commit_policy or 'no_duplicate' } +end + +function M.new(opts) + opts = opts or {} + return setmetatable({ + _conn = opts.conn, + _artifact_store = opts.artifact_store, + _observer = opts.observer, + _component = opts.component or 'mcu', + _chunk_size = opts.chunk_size or 2048, + _commit_policy = opts.commit_policy, + _call_opts = opts.call_opts, + _rpc_retry = opts.rpc_retry, + }, Backend) +end + +M.Backend = Backend +return M diff --git a/src/services/update/backends/router.lua b/src/services/update/backends/router.lua new file mode 100644 index 00000000..a7e443f1 --- /dev/null +++ b/src/services/update/backends/router.lua @@ -0,0 +1,69 @@ +-- services/update/backends/router.lua +-- Component backend router for active update work. + +local op = require 'fibers.op' + +local M = {} +local Router = {} +Router.__index = Router + +local function component_of(job) + return type(job) == 'table' and job.component or nil +end + +local function backend_for(self, job) + local component = component_of(job) + local b = component and self._components[component] or nil + return b or self._default +end + +local function method_op(self, name, required, job, ctx) + local b = backend_for(self, job) + if type(b) ~= 'table' then + if required then return op.always(nil, 'update_backend_unavailable') end + return op.always(nil, nil) + end + local fn = b[name] + if type(fn) ~= 'function' then + if required then return op.always(nil, 'update_backend_missing_' .. name) end + return op.always(nil, nil) + end + return fn(b, job, ctx) +end + + +function Router:stage_op(job, ctx) + return method_op(self, 'stage_op', true, job, ctx) +end + +function Router:pre_commit_record_op(job, ctx) + return method_op(self, 'pre_commit_record_op', false, job, ctx) +end + +function Router:commit_op(job, ctx) + return method_op(self, 'commit_op', true, job, ctx) +end + +function Router:evaluate_reconcile(job, snapshot, ctx) + local b = backend_for(self, job) + if type(b) == 'table' and type(b.evaluate_reconcile) == 'function' then + return b:evaluate_reconcile(job, snapshot, ctx) + end + return { done = false, reason = 'reconcile_backend_unavailable' } +end + +function Router:commit_capabilities() + return { policy = self._commit_policy or 'no_duplicate' } +end + +function M.new(opts) + opts = opts or {} + return setmetatable({ + _components = opts.components or {}, + _default = opts.default, + _commit_policy = opts.commit_policy, + }, Router) +end + +M.Router = Router +return M diff --git a/src/services/update/bundled.lua b/src/services/update/bundled.lua new file mode 100644 index 00000000..2263564c --- /dev/null +++ b/src/services/update/bundled.lua @@ -0,0 +1,276 @@ +-- services/update/bundled.lua +-- +-- Bundled update policy coordinator shape. +-- +-- This module performs no blocking artifact or job work inline. It records +-- desired/current policy state, starts scoped probe/import workers, and then +-- starts scoped apply workers when policy asks for durable job creation. + +local bundled_probe = require 'services.update.bundled_probe' +local bundled_apply = require 'services.update.bundled_apply' +local model = require 'services.update.model' + +local M = {} + +local Coordinator = {} +Coordinator.__index = Coordinator + +local function copy(v) return model.deep_copy(v) end + +local function component_config(cfg, component) + cfg = cfg or {} + local by_component = cfg.components or cfg.by_component or {} + return by_component[component] or cfg[component] +end + +local function component_order(cfg) + local out = {} + local by_component = (cfg and (cfg.components or cfg.by_component)) or {} + for component, item in pairs(by_component) do + if type(component) == 'string' and component ~= '' and type(item) == 'table' then + out[#out + 1] = component + end + end + table.sort(out) + return out +end + +local function job_policy(item) + item = item or {} + local job = copy(item.job or {}) + if job.create_if == nil then + if item.auto_create ~= nil then + job.create_if = item.auto_create == true and 'always' or 'never' + else + job.create_if = 'image_differs' + end + end + if job.start == nil then job.start = item.auto_start == true and 'auto' or 'manual' end + if job.commit == nil then job.commit = 'manual' end + return job +end + +local function component_state(snapshot, component) + if type(snapshot) ~= 'table' then return nil end + local by_id = snapshot.by_id or snapshot.components + local rec = type(by_id) == 'table' and by_id[component] or nil + if type(rec) == 'table' and type(rec.state) == 'table' then return rec.state end + if type(rec) == 'table' then return rec end + if snapshot.component == component then return snapshot.state or snapshot end + return nil +end + +local function current_image_id(snapshot, component) + local state = component_state(snapshot, component) + local software = type(state) == 'table' and (state.software or (type(state.raw_facts) == 'table' and state.raw_facts.software)) or nil + local image_id = type(software) == 'table' and software.image_id or nil + if type(image_id) == 'string' and image_id ~= '' then return image_id end + return nil +end + +local function expected_image_id(desired) + if type(desired) ~= 'table' then return nil end + local artifact = desired.artifact or desired.desired or desired + local image_id = desired.expected_image_id or (type(artifact) == 'table' and artifact.expected_image_id) + if type(image_id) == 'string' and image_id ~= '' then return image_id end + return nil +end + +function M.new(params) + params = params or {} + return setmetatable({ + service_id = params.service_id or 'update', + generation = params.generation, + config = copy(params.config or {}), + probes = {}, + applies = {}, + desired = {}, + state = {}, + last_apply = {}, + }, Coordinator) +end + +function Coordinator:enabled() + return self.config and self.config.enabled == true +end + +function Coordinator:components() + return component_order(self.config) +end + +function Coordinator:spec(component) + return component_config(self.config, component) +end + +function Coordinator:needs_probe(component) + if not self:enabled() then return false end + local item = self:spec(component) + return type(item) == 'table' + and item.source ~= nil + and self.desired[component] == nil + and self.probes[component] == nil + and self.applies[component] == nil +end + +function Coordinator:start_probe(spec) + spec = spec or {} + local component = assert(spec.component, 'component required') + if self.probes[component] then return nil, 'probe already running' end + local handle, err = bundled_probe.start { + lifetime_scope = assert(spec.lifetime_scope, 'lifetime_scope required'), + reaper_scope = spec.reaper_scope or spec.lifetime_scope, + report_scope = spec.report_scope or spec.lifetime_scope, + service_id = self.service_id, + generation = self.generation, + component = component, + artifact_store = assert(spec.artifact_store, 'artifact_store required'), + source = assert(spec.source, 'source required'), + done_tx = spec.done_tx, + } + if not handle then return nil, err end + self.probes[component] = handle + self.state[component] = 'probe_running' + return handle, nil +end + +function Coordinator:start_missing_probes(spec) + spec = spec or {} + local started = {} + if not self:enabled() then return started, nil end + for _, component in ipairs(self:components()) do + if self:needs_probe(component) then + local item = assert(self:spec(component)) + local handle, err = self:start_probe { + lifetime_scope = spec.lifetime_scope, + reaper_scope = spec.reaper_scope, + report_scope = spec.report_scope, + component = component, + artifact_store = spec.artifact_store, + source = item.source, + done_tx = spec.done_tx, + } + if not handle then return nil, err end + started[#started + 1] = component + end + end + return started, nil +end + +function Coordinator:handle_probe_done(ev) + if not ev or ev.kind ~= 'bundled_probe_done' then return false, 'not_probe_done' end + if ev.generation ~= self.generation then return false, 'stale_generation' end + self.probes[ev.component] = nil + if ev.status == 'ok' then + self.desired[ev.component] = copy(ev.result and ev.result.desired or ev.result) + self.state[ev.component] = 'desired_known' + else + self.state[ev.component] = 'probe_failed' + end + return true, nil +end + +function Coordinator:needs_apply(component, opts) + opts = opts or {} + if not self:enabled() then return false end + local item = self:spec(component) + local policy = job_policy(item) + local desired = self.desired[component] + if type(item) ~= 'table' or desired == nil or self.applies[component] ~= nil or self.last_apply[component] ~= nil then + return false + end + if policy.create_if == 'never' then + self.state[component] = 'create_disabled' + return false + end + if policy.create_if == 'image_differs' then + local expected = expected_image_id(desired) + local current = current_image_id(opts.current or opts.components or opts.observer_snapshot, component) + if expected == nil then + self.state[component] = 'pending_expected_image' + return false + end + if current == nil then + self.state[component] = 'pending_current_image' + return false + end + if current == expected then + self.state[component] = 'already_current' + return false + end + end + return true +end + +function Coordinator:start_apply(spec) + spec = spec or {} + local component = assert(spec.component, 'component required') + if self.applies[component] then return nil, 'apply already running' end + local item = spec.config or self:spec(component) or {} + local desired = spec.desired or self.desired[component] + if desired == nil then return nil, 'desired artifact missing' end + local apply_spec = copy(item) + apply_spec.component = apply_spec.component or component + local handle, err = bundled_apply.start { + lifetime_scope = assert(spec.lifetime_scope, 'lifetime_scope required'), + reaper_scope = spec.reaper_scope or spec.lifetime_scope, + report_scope = spec.report_scope or spec.lifetime_scope, + service_id = self.service_id, + generation = self.generation, + component = component, + jobs = assert(spec.jobs, 'jobs required'), + spec = apply_spec, + desired = desired, + done_tx = spec.done_tx, + } + if not handle then return nil, err end + self.applies[component] = handle + self.state[component] = 'apply_running' + return handle, nil +end + +function Coordinator:start_ready_applies(spec) + spec = spec or {} + local started = {} + if not self:enabled() then return started, nil end + for _, component in ipairs(self:components()) do + if self:needs_apply(component, spec) then + local handle, err = self:start_apply { + lifetime_scope = spec.lifetime_scope, + reaper_scope = spec.reaper_scope, + report_scope = spec.report_scope, + component = component, + jobs = spec.jobs, + done_tx = spec.done_tx, + } + if not handle then return nil, err end + started[#started + 1] = component + end + end + return started, nil +end + +function Coordinator:handle_apply_done(ev) + if not ev or ev.kind ~= 'bundled_apply_done' then return false, 'not_apply_done' end + if ev.generation ~= self.generation then return false, 'stale_generation' end + self.applies[ev.component] = nil + self.last_apply[ev.component] = copy(ev) + if ev.status == 'ok' then + self.state[ev.component] = 'applied' + else + self.state[ev.component] = 'apply_failed' + end + return true, nil +end + +function Coordinator:snapshot() + return { + generation = self.generation, + enabled = self:enabled(), + desired = copy(self.desired), + state = copy(self.state), + last_apply = copy(self.last_apply), + } +end + +M.Coordinator = Coordinator +return M diff --git a/src/services/update/bundled_apply.lua b/src/services/update/bundled_apply.lua new file mode 100644 index 00000000..791e3121 --- /dev/null +++ b/src/services/update/bundled_apply.lua @@ -0,0 +1,296 @@ +-- services/update/bundled_apply.lua +-- +-- Scoped bundled-policy application. +-- +-- This worker owns the blocking durable transitions for the fixed-path policy: +-- create a normal update job from the imported artifact and optionally persist +-- a start intent. Active execution remains service-owned by active_runtime. + +local fibers = require 'fibers' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local tablex = require 'shared.table' + +local M = {} + +local function copy(v) return tablex.deep_copy(v) end + +local function merge(a, b) + local out = copy(a or {}) + for k, v in pairs(b or {}) do out[k] = copy(v) end + return out +end + +local function artifact_ref_of(artifact) + if type(artifact) ~= 'table' then return nil end + return artifact.artifact_ref or artifact.ref or artifact.id +end + +local TERMINAL = { succeeded = true, failed = true, cancelled = true, timed_out = true, superseded = true, discarded = true } + +local function optional_number(v) + if type(v) == 'number' then return v end + if type(v) == 'string' then return tonumber(v) end + return nil +end + +local function max_attempts(policy) + local n = optional_number(policy and policy.max_attempts) + if type(n) ~= 'number' or n <= 0 or n ~= math.floor(n) then return nil end + return n +end + +local function max_number(...) + local n = 0 + for i = 1, select('#', ...) do + local candidate = optional_number(select(i, ...)) + if type(candidate) == 'number' and candidate > n then n = candidate end + end + return n +end + +local function job_attempt(job) + if type(job) ~= 'table' then return 0 end + local metadata = type(job.metadata) == 'table' and job.metadata or {} + local policy = type(job.policy) == 'table' and job.policy or {} + local result = type(job.result) == 'table' and job.result or {} + return max_number( + job.attempt, + job.bundled_attempt, + metadata.attempt, + metadata.bundled_attempt, + policy.attempt, + result.attempt, + result.bundled_attempt + ) +end + +local function set_payload_attempt(payload, attempt) + attempt = optional_number(attempt) or 1 + payload.attempt = attempt + payload.metadata = merge(payload.metadata or {}, { bundled_attempt = attempt }) + payload.policy = merge(payload.policy or {}, { attempt = attempt }) + return payload +end + +local function job_policy(spec) + spec = spec or {} + local job = copy(spec.job or {}) + if job.job_id == nil then job.job_id = spec.job_id or (spec.source and spec.source.job_id) end + if job.create_if == nil then + if spec.auto_create ~= nil then + job.create_if = spec.auto_create == true and 'always' or 'never' + else + job.create_if = 'image_differs' + end + end + if job.start == nil then job.start = spec.auto_start == true and 'auto' or 'manual' end + if job.commit == nil then job.commit = 'manual' end + if job.reconcile == nil then job.reconcile = 'required' end + if job.supersede == nil then job.supersede = 'same_job_if_image_changed' end + return job +end + +local function transition(jobs, cmd) + if type(jobs) ~= 'table' or type(jobs.admit_transition) ~= 'function' then + return nil, 'job_runtime_unavailable' + end + local handle, err = jobs:admit_transition(cmd) + if not handle then return nil, err or 'job_transition_admission_failed' end + return fibers.perform(handle:outcome_op()) +end + +local function create_payload(spec, desired) + desired = desired or {} + local artifact = copy(desired.artifact or desired.desired or desired) + local component = assert(spec.component, 'bundled_apply component required') + local source = spec.source or {} + local policy = job_policy(spec) + local job_id = policy.job_id or spec.job_id or source.job_id or ('bundled-' .. component) + local metadata = merge({ + source = 'bundled', + bundled = true, + }, source.metadata or source.meta) + metadata = merge(metadata, spec.metadata) + + local expected_image_id = desired.expected_image_id or (type(artifact) == 'table' and artifact.expected_image_id) + if component == 'mcu' and (type(expected_image_id) ~= 'string' or expected_image_id == '') then + error('expected_image_id_required', 0) + end + return { + job_id = job_id, + component = component, + expected_image_id = expected_image_id, + artifact = artifact, + artifact_ref = desired.artifact_ref or artifact_ref_of(artifact), + metadata = metadata, + policy = policy, + auto_start = policy.start == 'auto', + } +end + +local function discard_existing_job(params, job_id) + local result, err = transition(params.jobs, { + kind = 'discard_job', + generation = params.generation, + job_id = job_id, + reason = 'bundled_supersede_changed_image', + }) + if not result then return nil, err or 'bundled_supersede_discard_failed' end + if result.status ~= 'persisted' then return nil, result.reason or 'bundled_supersede_discard_rejected' end + return result, nil +end + +local function create_job(params, payload, attempt) + set_payload_attempt(payload, attempt) + local result, err = transition(params.jobs, { + kind = 'create_job', + generation = params.generation, + payload = payload, + reason = 'bundled_create_job', + }) + + if not result then return nil, err or 'bundled_create_job_failed' end + if result.status == 'persisted' then return result, nil end + + if result.status == 'rejected' and result.reason == 'job_exists' then + local job = params.jobs:get(payload.job_id) + if job ~= nil then + return { status = 'existing', job_id = payload.job_id, job = job }, nil + end + end + + return nil, result.reason or err or 'bundled_create_job_failed' +end + +local function create_or_reuse_job(params, payload) + local existing = params.jobs:get(payload.job_id) + if existing ~= nil then + local policy = type(payload.policy) == 'table' and payload.policy or {} + local supersede = policy.supersede + if existing.expected_image_id == payload.expected_image_id then + if TERMINAL[existing.state] == true and supersede == 'same_job_if_image_changed' then + -- A previous successful convergence for this image must not consume + -- retry budget if the MCU later runs a different image again + -- (for example after a downgrade/rollback). Treat that as a + -- fresh convergence episode. Failed/timed-out terminal jobs remain + -- bounded by the retry budget to avoid loops on a bad bundle. + local next_attempt + local limit = max_attempts(policy) + if limit == nil then return nil, 'bundled_max_attempts_required' end + if existing.state == 'succeeded' then + next_attempt = 1 + else + local previous_attempt = job_attempt(existing) + next_attempt = previous_attempt + 1 + if next_attempt > limit then + return nil, 'bundled_retry_exhausted:' .. tostring(previous_attempt) .. '/' .. tostring(limit) + end + end + local discarded, derr = discard_existing_job(params, payload.job_id) + if not discarded then return nil, derr end + local created, cerr = create_job(params, payload, next_attempt) + if created then created.superseded = discarded end + return created, cerr + end + return { + status = 'existing', + job_id = payload.job_id, + job = existing, + }, nil + end + if TERMINAL[existing.state] == true and supersede == 'same_job_if_image_changed' then + local discarded, derr = discard_existing_job(params, payload.job_id) + if not discarded then return nil, derr end + local created, cerr = create_job(params, payload, 1) + if created then created.superseded = discarded end + return created, cerr + end + return nil, 'bundled_existing_job_image_mismatch' + end + + return create_job(params, payload, 1) +end + +local function maybe_start_job(params, job, auto_start) + if auto_start ~= true then + return nil, 'auto_start_disabled' + end + if not job or job.state ~= 'created' then + return nil, job and ('job_not_created:' .. tostring(job.state)) or 'job_missing' + end + + local result, err = transition(params.jobs, { + kind = 'start_job', + generation = params.generation, + job_id = job.job_id, + phase = 'stage', + reason = 'bundled_auto_start', + }) + if not result then return nil, err or 'bundled_start_job_failed' end + if result.status ~= 'persisted' then + return nil, result.reason or 'bundled_start_job_rejected' + end + return result, nil +end + +function M.run(scope, params) + params = params or {} + if params.jobs and type(params.jobs.ready_op) == 'function' then + local ok_ready, ready_err = fibers.perform(params.jobs:ready_op()) + if ok_ready ~= true then error(ready_err or 'job_runtime_not_ready', 0) end + end + local spec = assert(params.spec, 'bundled_apply spec required') + local desired = assert(params.desired, 'bundled_apply desired required') + local payload = create_payload(spec, desired) + if type(payload.artifact_ref) ~= 'string' or payload.artifact_ref == '' then + error('bundled_artifact_ref_required', 0) + end + + local created, cerr = create_or_reuse_job(params, payload) + if not created then error(cerr or 'bundled_create_failed', 0) end + + local job = created.job or (created.job_id and params.jobs:get(created.job_id)) + local start_result, start_err = maybe_start_job(params, job, payload.auto_start) + + return { + tag = 'bundled_apply_result', + component = spec.component, + job_id = payload.job_id, + artifact_ref = payload.artifact_ref, + create = created, + start = start_result, + start_skipped = start_result == nil and start_err or nil, + } +end + +function M.start(spec) + spec = spec or {} + return scoped_work.start { + lifetime_scope = assert(spec.lifetime_scope, 'lifetime_scope required'), + reaper_scope = spec.reaper_scope or spec.lifetime_scope, + report_scope = assert(spec.report_scope or spec.lifetime_scope, 'report_scope required'), + + identity = { + kind = 'bundled_apply_done', + service_id = spec.service_id, + generation = spec.generation, + component = spec.component, + }, + + run = function (scope) + return M.run(scope, spec) + end, + + report = function (ev) + if not spec.done_tx then return true, nil end + return queue.try_admit_required( + spec.done_tx, + ev, + 'update_bundled_apply_completion_report_failed' + ) + end, + } +end + +return M diff --git a/src/services/update/bundled_probe.lua b/src/services/update/bundled_probe.lua new file mode 100644 index 00000000..89ebd13e --- /dev/null +++ b/src/services/update/bundled_probe.lua @@ -0,0 +1,119 @@ +-- services/update/bundled_probe.lua +-- +-- Scoped desired-artifact probing/import for bundled update policy. +-- +-- The normal fixed-path path imports the configured file into the artifact +-- store, so downstream Update code always receives the same artifact_ref shape +-- as the browser-ingest path. Pure probe_op backends remain supported for +-- tests and non-importing stores. + +local fibers = require 'fibers' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local tablex = require 'shared.table' +local dcmcu = require 'services.update.artifacts.dcmcu' + +local M = {} + +local function copy(v) return tablex.deep_copy(v) end + +local function artifact_snapshot(v) + if type(v) == 'table' and type(v.describe) == 'function' then + local ok, rec = pcall(function () return v:describe() end) + if ok and type(rec) == 'table' then + rec = copy(rec) + rec.ref = rec.ref or rec.artifact_ref + rec.id = rec.id or rec.artifact_ref or rec.ref + return rec + end + end + local out = copy(v) + if type(out) == 'table' then + out.ref = out.ref or out.artifact_ref + out.id = out.id or out.artifact_ref or out.ref + end + return out +end + +local function import_or_probe_op(store, source, component) + source = source or {} + local meta = copy(source.meta or source.metadata or {}) + meta.component = meta.component or component + meta.source = meta.source or 'bundled' + + local opts = { + policy = source.policy, + copy = source.copy, + } + + if (source.kind == 'file' or source.path ~= nil) and type(store.import_path_op) == 'function' then + return store:import_path_op(source.path, meta, opts) + end + + if type(store.import_source_op) == 'function' and source.source ~= nil then + return store:import_source_op(source.source, meta, opts) + end + + if type(store.probe_op) == 'function' then + return store:probe_op(source) + end + + + return require('fibers.op').always(nil, 'artifact_store_import_or_probe_not_supported') +end + +function M.run(scope, params) + params = params or {} + local store = assert(params.artifact_store, 'bundled_probe: artifact_store required') + local source = assert(params.source, 'bundled_probe: source required') + local result, err = fibers.perform(import_or_probe_op(store, source, params.component)) + if result == nil then error(err or 'bundled_probe_failed', 0) end + local desired = artifact_snapshot(result) + local artifact_ref = type(desired) == 'table' and (desired.artifact_ref or desired.ref or desired.id) or nil + if params.component == 'mcu' then + if type(store.open_source_op) ~= 'function' then error('artifact_source_unavailable', 0) end + local source, serr = fibers.perform(store:open_source_op(artifact_ref)) + if source == nil then error(serr or 'artifact_source_open_failed', 0) end + local identity, ierr = fibers.perform(dcmcu.identity_from_source_op(source)) + if identity == nil then error(ierr or 'dcmcu_identity_unavailable', 0) end + desired.expected_image_id = identity.image_id + end + return { + tag = 'bundled_desired', + component = params.component, + desired = desired, + artifact = desired, + artifact_ref = artifact_ref, + } +end + +function M.start(spec) + spec = spec or {} + return scoped_work.start { + lifetime_scope = assert(spec.lifetime_scope, 'lifetime_scope required'), + reaper_scope = spec.reaper_scope or spec.lifetime_scope, + report_scope = assert(spec.report_scope or spec.lifetime_scope, 'report_scope required'), + + identity = { + kind = 'bundled_probe_done', + service_id = spec.service_id, + generation = spec.generation, + component = spec.component, + }, + + run = function (scope) + return M.run(scope, spec) + end, + + report = function (ev) + if not spec.done_tx then return true, nil end + return queue.try_admit_required( + spec.done_tx, + ev, + 'update_bundled_probe_completion_report_failed' + ) + end, + } +end + +return M diff --git a/src/services/update/component_watch.lua b/src/services/update/component_watch.lua new file mode 100644 index 00000000..2ada84fb --- /dev/null +++ b/src/services/update/component_watch.lua @@ -0,0 +1,96 @@ +-- services/update/component_watch.lua +-- +-- Service-owned watch of Device's canonical component projections. This feeds +-- Update's reconciliation observer without letting active jobs perform ad-hoc +-- bus watches. + +local fibers = require 'fibers' +local scoped_work = require 'devicecode.support.scoped_work' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local topics = require 'services.update.topics' + +local M = {} + +local function component_list(config, explicit) + local seen, out = {}, {} + local function add(c) + if type(c) == 'string' and c ~= '' and not seen[c] then + seen[c] = true + out[#out + 1] = c + end + end + for _, c in ipairs(explicit or {}) do add(c) end + for c in pairs((config and config.components) or {}) do add(c) end + if #out == 0 then add('mcu') end + table.sort(out) + return out +end + +local function open_watches(scope, conn, components, queue_len) + local watches = {} + for _, component in ipairs(components) do + local watch, err = bus_cleanup.watch_retained(conn, topics.device_component(component), { + queue_len = queue_len or 16, + replay = true, + }) + if not watch then + for _, w in pairs(watches) do bus_cleanup.unwatch_retained(conn, w) end + return nil, err + end + watches[component] = watch + end + scope:finally(function () + for _, w in pairs(watches) do bus_cleanup.unwatch_retained(conn, w) end + end) + return watches, nil +end + +local function watch_loop(scope, params) + local observer = assert(params.observer, 'component_watch observer required') + local watches, err = open_watches(scope, assert(params.conn, 'component_watch conn required'), params.components, params.queue_len) + if not watches then error(err or 'component_watch_open_failed', 0) end + + while true do + local arms = {} + for component, watch in pairs(watches) do + arms[component] = watch:recv_op():wrap(function (ev, recv_err) + return component, ev, recv_err + end) + end + local _, component, ev, recv_err = fibers.perform(fibers.named_choice(arms)) + if ev == nil then + return { role = 'update_component_watch', reason = recv_err or 'component_watch_closed' } + end + if ev.op == 'retain' or ev.event == 'retain' or ev.type == 'retain' or ev.kind == 'retain' then + observer:update_component(component, ev.payload, ev.origin) + elseif ev.op == 'unretain' or ev.event == 'unretain' or ev.type == 'unretain' or ev.kind == 'unretain' then + observer:remove_component(component, 'component_unretained') + end + end +end + +function M.start(scope, params) + params = params or {} + local components = component_list(params.config, params.components) + return scoped_work.start { + lifetime_scope = scope, + reaper_scope = scope, + report_scope = scope, + identity = { + kind = 'component_done', + service_id = params.service_id or 'update', + component = 'component_watch', + }, + run = function (work_scope) + return watch_loop(work_scope, { + conn = params.conn, + observer = params.observer, + components = components, + queue_len = params.queue_len, + }) + end, + report = params.report, + } +end + +return M diff --git a/src/services/update/config.lua b/src/services/update/config.lua new file mode 100644 index 00000000..d2a96c1e --- /dev/null +++ b/src/services/update/config.lua @@ -0,0 +1,302 @@ +-- services/update/config.lua +-- +-- Pure update configuration validation and normalisation. + +local model = require 'services.update.model' +local tablex = require 'shared.table' + +local M = {} + +M.SCHEMA = 'devicecode.update/1' + +local function copy(v) + return tablex.deep_copy(v) +end + +local function table_or_empty(v) + if v == nil then return {} end + if type(v) ~= 'table' then return nil, 'expected table' end + return tablex.deep_copy(v), nil +end + +local function sorted_count(t) + local n = 0 + for _ in pairs(t or {}) do n = n + 1 end + return n +end + +local function normalise_optional_positive_number(t, key, where) + local v = t[key] + if v == nil then return true, nil end + local n = tonumber(v) + if type(n) ~= 'number' or n <= 0 then + return false, where .. '.' .. key .. ' must be a positive number' + end + t[key] = n + return true, nil +end + + +local function normalise_non_negative_integer(t, key, where) + local v = t[key] + if v == nil then return true, nil end + local n = tonumber(v) + if type(n) ~= 'number' or n < 0 or n ~= math.floor(n) then + return false, where .. '.' .. key .. ' must be a non-negative integer' + end + t[key] = n + return true, nil +end + +local function normalise_optional_positive_integer(t, key, where) + local v = t[key] + if v == nil then return true, nil end + local n = tonumber(v) + if type(n) ~= 'number' or n <= 0 or n ~= math.floor(n) then + return false, where .. '.' .. key .. ' must be a positive integer' + end + t[key] = n + return true, nil +end + +local function normalise_retention(raw) + local retention, rerr = table_or_empty(raw) + if not retention then return nil, rerr end + -- Update is not an audit archive on-device. This release always uses + -- the single compact current-or-last job policy, regardless of stale + -- legacy config carried by development images. + retention.mode = 'single_job' + retention.scrub_legacy_on_startup = true + retention.prune_on_startup = true + if retention.max_jobs == nil then retention.max_jobs = 1 end + local ok, err = normalise_optional_positive_integer(retention, 'max_jobs', 'retention') + if not ok then return nil, err end + if retention.max_jobs ~= 1 then retention.max_jobs = 1 end + if retention.keep_last_terminal == nil then retention.keep_last_terminal = true end + if retention.keep_last_terminal ~= true and retention.keep_last_terminal ~= false then + return nil, 'retention.keep_last_terminal must be boolean' + end + -- Retained workflow archives are disabled on device targets. + retention.retain_workflows = false + ok, err = normalise_non_negative_integer(retention, 'terminal_max_count', 'retention') + if not ok then return nil, err end + ok, err = normalise_optional_positive_integer(retention, 'terminal_max_age_s', 'retention') + if not ok then return nil, err end + ok, err = normalise_non_negative_integer(retention, 'active_intent_restart_max', 'retention') + if not ok then return nil, err end + if retention.active_intent_restart_max == nil then retention.active_intent_restart_max = 1 end + return retention, nil +end + +local function normalise_component_timeouts(c, where) + for _, key in ipairs({ 'timeout_s', 'stage_timeout_s', 'commit_timeout_s', 'reconcile_timeout_s' }) do + local ok, err = normalise_optional_positive_number(c, key, where) + if not ok then return nil, err end + end + return c, nil +end + +local function normalise_components(raw) + local out = {} + local list = raw or {} + + if model.is_array(list) then + for i = 1, #list do + local item = list[i] + if type(item) ~= 'table' then + return nil, 'components entries must be tables' + end + local id = item.component + if type(id) ~= 'string' or id == '' then + return nil, 'component must be a non-empty string' + end + if item.id ~= nil or item.name ~= nil then + return nil, 'component entries must use component, not id or name' + end + if out[id] ~= nil then + return nil, 'duplicate component: ' .. id + end + local c = copy(item) + c.component = id + local ok, terr = normalise_component_timeouts(c, 'components[' .. tostring(i) .. ']') + if not ok then return nil, terr end + out[id] = c + end + else + for id, item in pairs(list) do + if type(id) ~= 'string' or id == '' then + return nil, 'component map keys must be non-empty strings' + end + if type(item) ~= 'table' then + return nil, 'component entries must be tables' + end + if item.id ~= nil or item.name ~= nil then + return nil, 'component entries must use component, not id or name' + end + if item.component ~= nil and item.component ~= id then + return nil, 'component field must match component map key' + end + local c = copy(item) + c.component = id + local ok, terr = normalise_component_timeouts(c, 'components.' .. id) + if not ok then return nil, terr end + out[id] = c + end + end + + return out, nil +end + + +local CREATE_IF = { always = true, image_differs = true, never = true } +local JOB_START = { auto = true, manual = true } +local JOB_COMMIT = { auto = true, manual = true } +local JOB_RECONCILE = { required = true, manual = true } +local JOB_SUPERSEDE = { same_job_if_image_changed = true, never = true } + +local function normalise_enum(t, key, allowed, default, where) + if t[key] == nil then t[key] = default end + if allowed[t[key]] ~= true then + return nil, where .. '.' .. key .. ' invalid: ' .. tostring(t[key]) + end + return t, nil +end + +local function normalise_bundled_job(raw, component, legacy, inherited_max_attempts, where) + local job, jerr = table_or_empty(raw) + if not job then return nil, where .. ': ' .. tostring(jerr) end + legacy = legacy or {} + if job.job_id == nil then job.job_id = legacy.job_id or ('bundled-' .. component) end + if type(job.job_id) ~= 'string' or job.job_id == '' then return nil, where .. '.job_id must be a non-empty string' end + if job.create_if == nil and legacy.auto_create ~= nil then job.create_if = legacy.auto_create == true and 'always' or 'never' end + if job.start == nil and legacy.auto_start ~= nil then job.start = legacy.auto_start == true and 'auto' or 'manual' end + local ok, err = normalise_enum(job, 'create_if', CREATE_IF, 'image_differs', where) + if not ok then return nil, err end + ok, err = normalise_enum(job, 'start', JOB_START, 'manual', where) + if not ok then return nil, err end + ok, err = normalise_enum(job, 'commit', JOB_COMMIT, 'manual', where) + if not ok then return nil, err end + ok, err = normalise_enum(job, 'reconcile', JOB_RECONCILE, 'required', where) + if not ok then return nil, err end + ok, err = normalise_enum(job, 'supersede', JOB_SUPERSEDE, 'same_job_if_image_changed', where) + if not ok then return nil, err end + ok, err = normalise_optional_positive_integer(job, 'max_attempts', where) + if not ok then return nil, err end + if job.max_attempts == nil then job.max_attempts = inherited_max_attempts end + return job, nil +end + +local function normalise_bundled_component(id, item, inherited_max_attempts, where) + if type(item) ~= 'table' then return nil, where .. ' must be a table' end + local c = copy(item) + if c.component == nil then c.component = id end + if c.component ~= id then return nil, where .. '.component must match component map key' end + if c.source ~= nil and type(c.source) ~= 'table' then return nil, where .. '.source must be a table' end + local job, jerr = normalise_bundled_job(c.job, id, c, inherited_max_attempts, where .. '.job') + if not job then return nil, jerr end + c.job = job + c.auto_create = nil + c.auto_start = nil + return c, nil +end + +local function normalise_bundled(raw) + local bundled, berr = table_or_empty(raw) + if not bundled then return nil, berr end + if bundled.enabled == nil then bundled.enabled = false end + if bundled.enabled ~= true and bundled.enabled ~= false then return nil, 'bundled.enabled must be boolean' end + local comps, cerr = table_or_empty(bundled.components or bundled.by_component) + if not comps then return nil, 'bundled.components: ' .. tostring(cerr) end + local ok, err = normalise_optional_positive_integer(bundled, 'max_attempts', 'bundled') + if not ok then return nil, err end + if bundled.enabled == true and bundled.max_attempts == nil then + return nil, 'bundled.max_attempts must be a positive integer when bundled is enabled' + end + local out = { enabled = bundled.enabled, max_attempts = bundled.max_attempts, components = {} } + for component, item in pairs(comps) do + if type(component) ~= 'string' or component == '' then return nil, 'bundled.components keys must be non-empty strings' end + local normal, nerr = normalise_bundled_component(component, item, bundled.max_attempts, 'bundled.components.' .. component) + if not normal then return nil, nerr end + out.components[component] = normal + end + return out, nil +end + +local function summarise(cfg) + return { + rev = cfg.rev, + schema = cfg.schema, + service_id = cfg.service_id, + namespace = cfg.namespace, + component_count = sorted_count(cfg.components), + bundled_enabled = cfg.bundled and cfg.bundled.enabled == true or false, + } +end + +--- Extract the data table from a config-service retained payload. +function M.extract_payload(v) + local payload = v + if type(payload) == 'table' and payload.payload ~= nil then + payload = payload.payload + end + if type(payload) == 'table' and payload.data ~= nil then + return payload.data, payload.rev + end + return payload, nil +end + +function M.normalise(raw, opts) + opts = opts or {} + raw = raw or {} + if type(raw) ~= 'table' then + return nil, 'update config must be a table' + end + + local components, cerr = normalise_components(raw.components) + if not components then + return nil, cerr + end + + local bundled, berr = normalise_bundled(raw.bundled) + if not bundled then return nil, 'bundled: ' .. tostring(berr) end + + local publish, perr = table_or_empty(raw.publish) + if not publish then return nil, 'publish: ' .. tostring(perr) end + if publish.enabled == nil then publish.enabled = true end + + local retention, rerr = normalise_retention(raw.retention) + if not retention then return nil, 'retention: ' .. tostring(rerr) end + + local out = { + schema = raw.schema or M.SCHEMA, + rev = opts.rev or raw.rev, + service_id = raw.service_id or opts.service_id or 'update', + namespace = raw.namespace or 'default', + components = components, + bundled = bundled, + publish = publish, + retention = retention, + raw = copy(raw), + } + + if out.schema ~= M.SCHEMA then + return nil, 'unsupported update config schema: ' .. tostring(out.schema) + end + + out.summary = summarise(out) + return out, nil +end + +function M.default(opts) + return assert(M.normalise({}, opts)) +end + +function M.material_equal(a, b) + return model.deep_equal(a and a.raw or a, b and b.raw or b) +end + +function M.summary(cfg) + return cfg and copy(cfg.summary or summarise(cfg)) or nil +end + +return M diff --git a/src/services/update/events.lua b/src/services/update/events.lua new file mode 100644 index 00000000..a77ca376 --- /dev/null +++ b/src/services/update/events.lua @@ -0,0 +1,174 @@ +-- services/update/events.lua +-- +-- Semantic event construction for update coordinators. + +local priority_event = require 'devicecode.support.priority_event' +local queue = require 'devicecode.support.queue' + +local M = {} + +local NOT_READY = {} + +local function recv_now(rx, map) + local item = queue.try_now(rx:recv_op(), NOT_READY) + if item == NOT_READY then + return nil + end + return map(item) +end + +local function recv_op(rx, map) + return rx:recv_op():wrap(function (item) + return map(item) + end) +end + +function M.map_generation_done(ev) + if ev == nil then + return { kind = 'generation_done_queue_closed' } + end + return ev +end + +function M.map_config_event(ev) + if ev == nil then + return { kind = 'config_watch_closed' } + end + + -- Shared devicecode.support.config_watch shape. It subscribes to cfg/ + -- and normalises retained config records to config_changed/config_watch_closed + -- events while preserving the original retained record for rev handling. + if type(ev) == 'table' and ev.kind == 'config_watch_closed' then + return { kind = 'config_watch_closed', err = ev.err } + end + if type(ev) == 'table' and ev.kind == 'config_changed' then + return { + kind = 'config_changed', + payload = ev.record or ev.raw, + origin = ev.msg and ev.msg.origin or ev.origin, + rev = ev.rev, + generation = ev.generation, + } + end + + -- Older retained-watch lifecycle shape kept for existing tests and harnesses. + if type(ev) == 'table' and ev.op == 'retain' then + return { + kind = 'config_changed', + payload = ev.payload, + origin = ev.origin, + } + end + if type(ev) == 'table' and ev.op == 'unretain' then + return { + kind = 'config_removed', + origin = ev.origin, + } + end + if type(ev) == 'table' and ev.op == 'replay_done' then + return { kind = 'config_replay_done' } + end + return { + kind = 'config_event_unknown', + event = ev, + } +end + +function M.map_manager_request(item) + if item == nil then + return { kind = 'manager_closed' } + end + if type(item) == 'table' and item.closed then + return { kind = 'manager_closed', method = item.method, reason = item.reason } + end + if type(item) == 'table' and item.request ~= nil then + return { + kind = 'manager_request', + method = item.method, + request = item.request, + } + end + return { + kind = 'manager_request', + request = item, + } +end + +function M.try_generation_done_now(state) + return recv_now(state.done_rx, M.map_generation_done) +end + +function M.try_config_now(state) + if not state.config_rx then return nil end + return recv_now(state.config_rx, M.map_config_event) +end + +function M.try_manager_now(state) + if not state.manager_rx then return nil end + return recv_now(state.manager_rx, M.map_manager_request) +end + + +function M.map_job_runtime_changed(version, snapshot, reason) + if version == nil then + return { kind = 'job_runtime_model_closed', reason = reason } + end + return { + kind = 'job_runtime_changed', + version = version, + snapshot = snapshot, + } +end + +function M.try_job_runtime_now(state) + if not state._jobs or not state._jobs_seen then return nil end + local version, snapshot, reason = queue.try_now(state._jobs:changed_op(state._jobs_seen), NOT_READY) + if version == NOT_READY then return nil end + return M.map_job_runtime_changed(version, snapshot, reason) +end + +function M.next_service_event_op(state) + local sources = { + { + name = 'generation_done', + try_now = function () return M.try_generation_done_now(state) end, + recv_op = function () return recv_op(state.done_rx, M.map_generation_done) end, + }, + { + name = 'job_runtime', + enabled = function () return state._jobs ~= nil and state._jobs_seen ~= nil end, + try_now = function () return M.try_job_runtime_now(state) end, + recv_op = function () + return state._jobs:changed_op(state._jobs_seen):wrap(M.map_job_runtime_changed) + end, + }, + { + name = 'config', + enabled = function () return state.config_rx ~= nil end, + try_now = function () return M.try_config_now(state) end, + recv_op = function () return recv_op(state.config_rx, M.map_config_event) end, + }, + } + + -- Dependency changes are admission facts for the service coordinator. Handle + -- them before externally-triggered manager requests so retained status replay + -- cannot be starved by status polling during startup. + if state._deps and type(state._deps.event_source) == 'function' then + sources[#sources + 1] = state._deps:event_source({ name = 'dependencies' }) + end + + sources[#sources + 1] = { + name = 'manager', + enabled = function () return state.manager_rx ~= nil end, + try_now = function () return M.try_manager_now(state) end, + recv_op = function () return recv_op(state.manager_rx, M.map_manager_request) end, + } + + return priority_event.sources_op { + label = 'update.service.next_event', + pending = state.pending, + sources = sources, + } +end + +return M diff --git a/src/services/update/generation.lua b/src/services/update/generation.lua new file mode 100644 index 00000000..85d4d243 --- /dev/null +++ b/src/services/update/generation.lua @@ -0,0 +1,332 @@ +-- services/update/generation.lua +-- +-- Generation lifetime composition and coordinator. +-- +-- The service starts and replaces generations. This module keeps the +-- generation role narrow: it owns the generation scope, composes +-- generation-local owners, and reduces semantic events. It never calls back +-- into parent service state. Parent-visible changes are reported as events on +-- the supplied service event port. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' + +local model_mod = require 'services.update.model' +local config_mod = require 'services.update.config' +local manager_mod = require 'services.update.manager' +local generation_events = require 'services.update.generation_events' +local observe_mod = require 'services.update.observe' +local ingest_mod = require 'services.update.ingest' +local bundled_mod = require 'services.update.bundled' +local service_events = require 'devicecode.support.service_events' + +local M = {} + +local DEFAULT_DONE_QUEUE = 32 + +local Generation = {} +Generation.__index = Generation + +local function copy(v) + return model_mod.deep_copy(v) +end + +local function emit_generation_snapshot(self, snapshot) + if not self._events then return true, nil end + return self._events:emit_required({ + kind = 'generation_snapshot', + snapshot = snapshot, + }, 'update_generation_snapshot_report_failed') +end + +local function update_generation_model(self) + local jobs_ready = self._jobs == nil or type(self._jobs.ready) ~= 'function' or self._jobs:ready() + local changed, err = self._model:update(function (s) + s.state = jobs_ready and 'running' or 'starting' + s.ready = jobs_ready + s.reason = jobs_ready and nil or 'job_runtime_loading' + s.jobs = self._jobs:snapshot() + s.update_active = copy(self._active_snapshot) + if self._observer then + s.components = self._observer:snapshot() + end + if self._ingest then + s.ingest = self._ingest:snapshot() + end + if self._bundled then + s.bundled = self._bundled:snapshot() + end + return s + end) + if changed == nil then return nil, err or 'generation_model_update_failed' end + if changed then + local ok, eerr = emit_generation_snapshot(self, self._model:snapshot()) + if ok ~= true then return nil, eerr end + end + return true, nil +end + +local function assert_update_generation_model(self) + local ok, err = update_generation_model(self) + if ok ~= true then error(err or 'update_generation_model_failed', 0) end + return true +end + +function M.initial_snapshot(params) + params = params or {} + local cfg = params.config or config_mod.default() + return model_mod.generation_initial { + service_id = params.service_id or 'update', + generation = params.generation or 1, + config = config_mod.summary(cfg), + components = cfg.components or {}, + bundled = cfg.bundled or {}, + } +end + +local function make_manager_context(self) + return { + scope = self._scope, + request_root = self._request_root, + service_id = self._service_id, + generation = self._generation, + config = self._config, + done_tx = self._done_tx, + jobs = self._jobs, + manager_work = self._manager_work, + observer = self._observer, + ingest = self._ingest, + artifact_store = self._artifact_store, + active = copy(self._active_snapshot), + snapshot = self._model:snapshot(), + } +end + +local function handle_manager_request(self, req) + -- Manager requests are cheap coordinator events. Refresh the generation model + -- from local state and service-provided projections before answering or + -- admitting scoped request work. + assert_update_generation_model(self) + return manager_mod.handle_request(make_manager_context(self), req) +end + +local function handle_manager_done(self, ev) + local ok = manager_mod.handle_done(make_manager_context(self), ev) + if ok then assert_update_generation_model(self) end +end + +local function handle_service_active_snapshot(self, ev) + self._active_snapshot = copy(ev.snapshot) + assert_update_generation_model(self) +end + +local function start_bundled_probes(self) + if not self._bundled then return true, nil end + local started, err = self._bundled:start_missing_probes { + lifetime_scope = self._scope, + reaper_scope = self._scope, + report_scope = self._scope, + artifact_store = self._artifact_store, + done_tx = self._done_tx, + } + if not started then return nil, err or 'bundled_probe_start_failed' end + assert_update_generation_model(self) + return true, nil +end + +local function start_bundled_applies(self) + if not self._bundled then return true, nil end + local started, err = self._bundled:start_ready_applies { + lifetime_scope = self._scope, + reaper_scope = self._scope, + report_scope = self._scope, + jobs = self._jobs, + observer_snapshot = self._observer and self._observer:snapshot() or nil, + done_tx = self._done_tx, + } + if not started then return nil, err or 'bundled_apply_start_failed' end + assert_update_generation_model(self) + return true, nil +end + +local function handle_bundled_probe_done(self, ev) + local ok, err = self._bundled:handle_probe_done(ev) + if ok ~= true then return ok, err end + local aok, aerr = start_bundled_applies(self) + if aok ~= true then error(aerr or 'bundled_apply_start_failed', 0) end + assert_update_generation_model(self) + return true, nil +end + +local function handle_bundled_apply_done(self, ev) + local ok, err = self._bundled:handle_apply_done(ev) + if ok ~= true then return ok, err end + assert_update_generation_model(self) + return true, nil +end + +local function reduce_event(self, ev) + if ev.kind == 'manager_closed' then + error('update manager endpoint closed', 0) + end + + if ev.kind == 'service_route_closed' then + error('update generation service route closed', 0) + end + + if ev.kind == 'service_active_snapshot' then + handle_service_active_snapshot(self, ev) + return + end + + if ev.kind == 'component_observer_changed' then + self._observer_seen = ev.version or (self._observer and self._observer:version()) + local aok, aerr = start_bundled_applies(self) + if aok ~= true then error(aerr or 'bundled_apply_start_failed', 0) end + assert_update_generation_model(self) + return + end + + if ev.kind == 'component_observer_closed' then + error(ev.reason or 'component_observer_closed', 0) + end + + if ev.kind == 'manager_request' then + handle_manager_request(self, ev.request) + return + end + + if ev.kind == 'manager_request_done' then + handle_manager_done(self, ev) + return + end + + if ev.kind == 'ingest_request' or ev.kind == 'ingest_closed' then + self._ingest:handle_event(make_manager_context(self), ev) + assert_update_generation_model(self) + return + end + + if ev.kind == 'ingest_request_done' or ev.kind == 'ingest_create_done' then + self._ingest:handle_done(make_manager_context(self), ev) + assert_update_generation_model(self) + return + end + + if ev.kind == 'bundled_probe_done' then + handle_bundled_probe_done(self, ev) + return + end + + if ev.kind == 'bundled_apply_done' then + handle_bundled_apply_done(self, ev) + return + end + + if ev.kind == 'completion_queue_closed' then + error('update generation completion queue closed', 0) + end + + error('update.generation: unknown event kind: ' .. tostring(ev.kind), 0) +end + +local function coordinator_loop(self) + assert_update_generation_model(self) + + while true do + local ev = fibers.perform(generation_events.next_op(self)) + reduce_event(self, ev) + end +end + +function M.run(scope, params) + params = params or {} + local snapshot = M.initial_snapshot(params) + local m = model_mod.new(snapshot, { label = 'update.generation' }) + + scope:finally(function (_, status, primary) + m:terminate(primary or status or 'generation_closed') + end) + + local observer = params.observer or observe_mod.new({ + service_id = params.service_id or 'update', + components = (params.config and params.config.components) or {}, + }) + if params.observer == nil then + scope:finally(function (_, status, primary) + observer:terminate(primary or status or 'generation_closed') + end) + end + + local jobs = params.jobs + if not jobs then + error('update generation requires service-owned job runtime', 0) + end + + local done_tx, done_rx = mailbox.new(params.done_queue_len or DEFAULT_DONE_QUEUE, { + full = 'reject_newest', + }) + scope:finally(function () + done_tx:close('generation_closed') + end) + + local request_root, request_root_err = scope:child() + if not request_root then + error(request_root_err or 'update_generation_request_root_create_failed', 0) + end + + local ingest_state = ingest_mod.new_state(scope, { + queue_len = params.ingest_queue_len, + }) + + local bundled_state = bundled_mod.new({ + service_id = params.service_id or 'update', + generation = params.generation or snapshot.generation, + config = (params.config and params.config.bundled) or {}, + }) + + local parent_events + if params.events_tx ~= nil then + parent_events = service_events.port(params.events_tx, { + service_id = params.service_id or 'update', + source = 'update_generation', + source_id = tostring(params.generation or snapshot.generation), + generation = params.generation or snapshot.generation, + }, { + label = 'update_generation_event_admission_failed', + }) + end + + local self = setmetatable({ + _scope = scope, + _request_root = request_root, + _service_id = params.service_id or 'update', + _generation = params.generation or snapshot.generation, + _config = params.config or config_mod.default(), + _model = m, + _jobs = jobs, + _observer = observer, + _ingest = ingest_state, + _bundled = bundled_state, + _artifact_store = params.artifact_store, + manager_rx = params.manager_rx, + service_rx = params.service_rx, + done_rx = done_rx, + pending = {}, + _done_tx = done_tx, + _done_rx = done_rx, + _observer_seen = observer and observer:version() or nil, + _manager_work = {}, + _active_snapshot = copy(params.active_snapshot), + _events = parent_events, + }, Generation) + + local bok, berr = start_bundled_probes(self) + if bok ~= true then error(berr or 'bundled_probe_start_failed', 0) end + + return coordinator_loop(self) +end + +M.Generation = Generation + +return M diff --git a/src/services/update/generation_events.lua b/src/services/update/generation_events.lua new file mode 100644 index 00000000..115d443b --- /dev/null +++ b/src/services/update/generation_events.lua @@ -0,0 +1,89 @@ +-- services/update/generation_events.lua +-- +-- Semantic event construction for a generation coordinator. + +local priority_event = require 'devicecode.support.priority_event' +local queue = require 'devicecode.support.queue' +local service_events = require 'devicecode.support.service_events' + +local M = {} + +local NOT_READY = {} + +function M.map_done(ev) + if ev == nil then return { kind = 'completion_queue_closed' } end + return ev +end + +function M.map_manager(req) + if req == nil then return { kind = 'manager_closed' } end + if service_events.is_route_event(req) then return req end + return { kind = 'manager_request', request = req } +end + +function M.map_service(ev) + if ev == nil then return { kind = 'service_route_closed' } end + if service_events.is_route_event(ev) then return ev end + return ev +end + +local function try_recv_now(rx, map) + local item = queue.try_now(rx:recv_op(), NOT_READY) + if item == NOT_READY then return nil end + return map(item) +end + +function M.next_op(state) + return priority_event.sources_op { + label = 'update.generation.next_event', + pending = state.pending, + sources = { + { + name = 'observer', + enabled = function () return state._observer ~= nil end, + try_now = function () + local v = state._observer:version() + if v == nil or v == state._observer_seen then return nil end + return { kind = 'component_observer_changed', version = v, snapshot = state._observer:snapshot() } + end, + recv_op = function () + return state._observer:changed_op(state._observer_seen):wrap(function (version, snapshot, reason) + if version == nil then return { kind = 'component_observer_closed', reason = reason } end + return { kind = 'component_observer_changed', version = version, snapshot = snapshot, reason = reason } + end) + end, + }, + { + name = 'done', + try_now = function () return try_recv_now(state.done_rx, M.map_done) end, + recv_op = function () return state.done_rx:recv_op():wrap(M.map_done) end, + }, + { + name = 'ingest_terminal', + enabled = function () return state._ingest ~= nil end, + try_now = function () return state._ingest:try_terminal_now() end, + recv_op = function () return state._ingest:terminal_op() end, + }, + { + name = 'ingest_request', + enabled = function () return state._ingest ~= nil end, + try_now = function () return state._ingest:try_request_now() end, + recv_op = function () return state._ingest:request_op() end, + }, + { + name = 'service', + enabled = function () return state.service_rx ~= nil end, + try_now = function () return try_recv_now(state.service_rx, M.map_service) end, + recv_op = function () return state.service_rx:recv_op():wrap(M.map_service) end, + }, + { + name = 'manager', + enabled = function () return state.manager_rx ~= nil end, + try_now = function () return try_recv_now(state.manager_rx, M.map_manager) end, + recv_op = function () return state.manager_rx:recv_op():wrap(M.map_manager) end, + }, + }, + } +end + +return M diff --git a/src/services/update/ingest.lua b/src/services/update/ingest.lua new file mode 100644 index 00000000..88d0afdb --- /dev/null +++ b/src/services/update/ingest.lua @@ -0,0 +1,683 @@ +-- services/update/ingest.lua +-- +-- Generation-owned ingest coordinator and sink ownership helpers. +-- +-- The coordinator serialises operations for each ingest instance. Append work may +-- block inside request scopes, but at most one append/commit/abort operation may +-- touch a sink at a time. Terminal requests close append admission immediately +-- and are then run after any already-admitted operation for that instance. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local request_owner = require 'devicecode.support.request_owner' +local model = require 'services.update.model' +local lifetime = require 'services.update.artifacts.lifetime' + +local M = {} + +local Instance = {} +Instance.__index = Instance + +local State = {} +State.__index = State + +local function copy(v) return model.deep_copy(v) end + + +local function artifact_snapshot(artifact) + if type(artifact) == 'table' and type(artifact.describe) == 'function' then + local ok, rec = pcall(function () return artifact:describe() end) + if ok and type(rec) == 'table' then + rec = copy(rec) + rec.ref = rec.ref or rec.artifact_ref + rec.id = rec.id or rec.artifact_ref or rec.ref + return rec + end + end + local snap = copy(artifact) + if type(snap) == 'table' then + snap.ref = snap.ref or snap.artifact_ref + snap.id = snap.id or snap.artifact_ref or snap.ref + end + return snap +end + + +local function payload_of(req) + return type(req) == 'table' and type(req.payload) == 'table' and req.payload or {} +end + +local function method_of(req) + local p = payload_of(req) + local method = (req and req._update_method) or 'ingest_append' + method = tostring(method):gsub('-', '_') + if method == 'ingest_create' then method = 'ingest_create' end + if method == 'ingest_append' then method = 'ingest_append' end + if method == 'ingest_commit' then method = 'ingest_commit' end + if method == 'ingest_abort' then method = 'ingest_abort' end + return method, p +end + +local function owner_for(req) + return request_owner.new(req) +end + +local function reply(req, value) + local ok, err = owner_for(req):reply_once(value) + if ok ~= true then error(err or 'ingest_reply_failed', 0) end +end + +local function fail(req, reason) + local ok, err = owner_for(req):fail_once(reason) + if ok ~= true then error(err or tostring(reason or 'ingest_failed'), 0) end +end + +local request_abandoned + +local function fail_owner(owner, req, reason) + owner = owner or owner_for(req) + if owner:done() or request_abandoned(req) then + owner:abandon_unresolved(reason or 'caller_abandoned') + return true + end + local ok, err = owner:fail_once(reason) + if ok ~= true then error(err or tostring(reason or 'ingest_failed'), 0) end + return true +end + +request_abandoned = function(req) + if type(req) == 'table' and type(req.status) == 'function' then + local status = req:status() + return status == 'abandoned' + end + return false +end + +local function entry_abandoned(entry) + if not entry then return false end + if entry.owner and entry.owner:done() then return true end + if request_abandoned(entry.req) then + if entry.owner then entry.owner:abandon_unresolved('caller_abandoned') end + return true + end + return false +end + +local function category_for(method) + if method == 'ingest_append' then return 'append' end + if method == 'ingest_commit' then return 'commit' end + if method == 'ingest_abort' then return 'abort' end + if method == 'ingest_create' then return 'create' end + return 'unknown' +end + +function M.new_instance(scope, params) + params = params or {} + local sink = assert(params.sink, 'ingest sink required') + assert(type(scope) == 'table' and type(scope.child) == 'function', 'ingest instance requires a parent scope') + + local child, child_err = scope:child() + if not child then return nil, child_err or 'ingest_instance_scope_create_failed' end + + local owned, err = lifetime.own(child, sink, { reason = 'ingest scope closed' }) + if not owned then + child:cancel(err or 'ingest_lifetime_own_failed') + return nil, err + end + + local self = setmetatable({ + ingest_id = assert(params.ingest_id, 'ingest_id required'), + component = params.component, + _scope = child, + _owned = owned, + bytes = 0, + state = 'open', + closed = false, + artifact = nil, + active_request_id = nil, + pending = {}, + terminal_requested = false, + _scope_closed = false, + }, Instance) + + child:finally(function (_, status, primary) + local reason = primary or ((status ~= 'ok') and status) or 'ingest_instance_closed' + if self.state == 'open' then self.state = 'closed' end + self.closed = true + while #self.pending > 0 do + local pending = table.remove(self.pending, 1) + local owner = pending.owner or request_owner.new(pending.req) + owner:finalise_unresolved(reason) + end + end) + + return self +end + +function Instance:_close_scope(reason) + if self._scope_closed then return true, nil end + self._scope_closed = true + if self._scope and type(self._scope.close) == 'function' then + self._scope:close(reason or 'ingest_instance_closed') + end + return true, nil +end + +function Instance:cancel(reason) + if self._scope and type(self._scope.cancel) == 'function' then + self._scope:cancel(reason or 'ingest_instance_cancelled') + end + return true, nil +end + + +function Instance:terminate(reason) + return self:cancel(reason or 'ingest_instance_terminated') +end + +function Instance:append_op(chunk) + if self.closed or self.state ~= 'open' then return nil, 'ingest closed' end + local ev, err = self._owned:append_op(chunk) + if not ev then return nil, err end + return ev:wrap(function (ok, aerr) + if ok ~= true then return nil, aerr or 'append failed' end + self.bytes = self.bytes + #(chunk or '') + return { tag = 'ingest_appended', ingest_id = self.ingest_id, bytes = self.bytes }, nil + end) +end + +function Instance:begin_commit() + if self.closed then return nil, 'ingest closed' end + if self.state ~= 'open' and self.state ~= 'committing' then + return nil, 'ingest not committable' + end + self.state = 'committing' + return true, nil +end + +function Instance:commit_worker(_scope, ...) + if self.closed then return nil, 'ingest closed' end + if self.state ~= 'committing' then + return nil, 'ingest not committable' + end + local ev, err = self._owned:commit_op(...) + if not ev then return nil, err end + + local artifact, commit_err = fibers.perform(ev) + if artifact == nil then return nil, commit_err or 'commit failed' end + local snap = artifact_snapshot(artifact) + self.artifact = snap + self.closed = true + self.state = 'committed' + self._owned:handoff(function () return true end) + self:_close_scope('ingest_committed') + return { + tag = 'ingest_committed', + ingest_id = self.ingest_id, + artifact = copy(snap), + artifact_ref = type(snap) == 'table' and (snap.artifact_ref or snap.ref or snap.id) or snap, + bytes = self.bytes, + }, nil +end + +function Instance:abort(reason) + if self.state == 'committed' then return nil, 'ingest already committed' end + if self.closed and self.state == 'aborted' then return true, nil end + self.state = 'aborting' + local ok, err = self._owned:terminate(reason or 'ingest aborted') + if ok ~= true then return nil, err or 'ingest abort failed' end + self.closed = true + self.state = 'aborted' + self:_close_scope(reason or 'ingest_aborted') + return true, nil +end + +function Instance:snapshot() + return { + ingest_id = self.ingest_id, + component = self.component, + bytes = self.bytes, + state = self.state, + closed = self.closed, + active_request_id = self.active_request_id, + queued = #self.pending, + artifact = copy(self.artifact), + } +end + +function M.append(_scope, instance, chunk) + local result, err = fibers.perform(assert(instance:append_op(chunk))) + if not result then error(err or 'ingest_append_failed', 0) end + return result +end + +function M.commit(scope, instance, ...) + local ok_begin, berr = instance:begin_commit() + if ok_begin ~= true then error(berr or 'ingest_commit_begin_failed', 0) end + local result, err = instance:commit_worker(scope, ...) + if not result then error(err or 'ingest_commit_failed', 0) end + return result +end + +function M.new_state(scope, params) + params = params or {} + local request_tx, request_rx = mailbox.new(params.queue_len or 16, { full = 'reject_newest' }) + local terminal_tx, terminal_rx = mailbox.new(params.queue_len or 16, { full = 'reject_newest' }) + scope:finally(function (_, status, primary) + local reason = primary or status or 'ingest_closed' + request_tx:close(reason) + terminal_tx:close(reason) + end) + local self = setmetatable({ + _scope = scope, + _request_tx = request_tx, + _request_rx = request_rx, + _terminal_tx = terminal_tx, + _terminal_rx = terminal_rx, + _instances = {}, + _work = {}, + _next_request_id = 1, + _ctx = nil, + }, State) + + scope:finally(function (_, status, primary) + self:terminate_instances(primary or status or 'ingest_closed') + end) + + return self +end + +function State:snapshot() + local out = {} + for id, inst in pairs(self._instances) do out[id] = inst:snapshot() end + return out +end + +function State:terminate_instances(reason) + for _, inst in pairs(self._instances or {}) do + if inst and type(inst.cancel) == 'function' then + inst:cancel(reason or 'ingest_state_closed') + end + end +end + +function State:submit(req) + local method = method_of(req) + local cat = category_for(method) + local tx = (cat == 'commit' or cat == 'abort') and self._terminal_tx or self._request_tx + return queue.try_admit_required(tx, req, 'update_ingest_request_admission_failed') +end + +function State:try_terminal_now() + local req = queue.try_recv_now(self._terminal_rx) + if req == nil then return nil end + return { kind = 'ingest_request', priority = 'terminal', request = req } +end + +function State:try_request_now() + local req = queue.try_recv_now(self._request_rx) + if req == nil then return nil end + return { kind = 'ingest_request', priority = 'normal', request = req } +end + +function State:terminal_op() + return self._terminal_rx:recv_op():wrap(function (req) + if req == nil then return { kind = 'ingest_closed' } end + return { kind = 'ingest_request', priority = 'terminal', request = req } + end) +end + +function State:request_op() + return self._request_rx:recv_op():wrap(function (req) + if req == nil then return { kind = 'ingest_closed' } end + return { kind = 'ingest_request', priority = 'normal', request = req } + end) +end + +local new_request_id + +local function create_instance_with_sink_now(state, req, payload, sink, owner) + if type(sink) ~= 'table' then + fail_owner(owner, req, 'ingest_sink_required') + return nil, 'ingest_sink_required' + end + local ingest_id = payload.ingest_id + if type(ingest_id) ~= 'string' or ingest_id == '' then + fail_owner(owner, req, 'ingest_id_required') + return nil, 'ingest_id_required' + end + if state._instances[ingest_id] ~= nil then + fail_owner(owner, req, 'ingest_exists') + return nil, 'ingest_exists' + end + local inst, err = M.new_instance(state._scope, { + ingest_id = ingest_id, + component = payload.component, + sink = sink, + }) + if not inst then + fail_owner(owner, req, err or 'ingest_create_failed') + return nil, err or 'ingest_create_failed' + end + state._instances[ingest_id] = inst + owner = owner or owner_for(req) + local ok, rerr = owner:reply_once({ ok = true, ingest = inst:snapshot() }) + if ok ~= true then error(rerr or 'ingest_create_reply_failed', 0) end + return inst, nil +end + +local function start_create_sink_work(ctx, state, req, payload) + if type(ctx) ~= 'table' or type(ctx.artifact_store) ~= 'table' or type(ctx.artifact_store.create_sink_op) ~= 'function' then + fail(req, 'artifact_store_unavailable') + return nil, 'artifact_store_unavailable' + end + local owner = request_owner.new(req) + local ingest_id = payload.ingest_id + local request_id = tostring(payload.request_id or new_request_id(state, 'ingest_create', ingest_id)) + local handle, err = scoped_work.start { + lifetime_scope = ctx.request_root or ctx.scope, + reaper_scope = ctx.request_root or ctx.scope, + report_scope = ctx.scope, + identity = { + kind = 'ingest_create_done', + service_id = ctx.service_id, + generation = ctx.generation, + method = 'ingest_create', + request_id = request_id, + ingest_id = ingest_id, + }, + setup = function (work_scope) + work_scope:finally(function (_, status, primary) + if owner:done() then return end + -- On success the unresolved request owner is intentionally + -- carried by the completion event and resolved by the + -- coordinator after the sink has been admitted to ingest + -- state. Do not fail it merely because the create-sink + -- worker scope has ended successfully. + if status ~= 'ok' then + owner:finalise_unresolved((status == 'failed' and primary) or primary or 'ingest_create_cancelled') + end + end) + return { + request_owner = owner, + cancel_owned_now = function (reason) + owner:abandon_unresolved(reason or 'caller_abandoned') + return true + end, + } + end, + cancel_op = owner:caller_cancel_op(), + run = function (_, setup) + local sink, serr = fibers.perform(ctx.artifact_store:create_sink_op({ + component = payload.component, + meta = payload.metadata or payload.meta or {}, + policy = payload.policy or 'transient_only', + })) + if sink == nil then error(serr or 'artifact_sink_create_failed', 0) end + return { + tag = 'ingest_sink_created', + ingest_id = ingest_id, + payload = copy(payload), + sink = sink, + request_owner = setup and setup.request_owner or owner, + } + end, + report = function (ev) + return queue.try_admit_required(ctx.done_tx, ev, 'update_ingest_create_completion_report_failed') + end, + } + if not handle then + fail_owner(owner, req, err or 'ingest_create_start_failed') + return nil, err or 'ingest_create_start_failed' + end + state._work[request_id] = { handle = handle, ingest_id = ingest_id, category = 'create', request = req, owner = owner } + return handle, nil +end + +local function create_instance_now(ctx, state, req, payload) + if type(payload.sink) == 'table' then + return create_instance_with_sink_now(state, req, payload, payload.sink) + end + return start_create_sink_work(ctx, state, req, payload) +end + +local start_next_for_instance + +local function start_ingest_work(ctx, state, inst, entry) + local work_parent = inst._scope or ctx.request_root or ctx.scope + local handle, err = scoped_work.start { + lifetime_scope = work_parent, + reaper_scope = work_parent, + report_scope = ctx.scope, + identity = { + kind = 'ingest_request_done', + service_id = ctx.service_id, + generation = ctx.generation, + method = entry.method, + request_id = entry.request_id, + ingest_id = inst.ingest_id, + category = entry.category, + }, + setup = function (work_scope) + local owner = entry.owner or request_owner.new(entry.req) + work_scope:finally(function (_, status, primary) + owner:finalise_unresolved((status == 'failed' and primary) or primary or (entry.method .. '_cancelled') or status) + end) + return { + request_owner = owner, + cancel_owned_now = function (reason) + owner:abandon_unresolved(reason or 'caller_abandoned') + return true + end, + } + end, + cancel_op = entry.owner and entry.owner:caller_cancel_op() or nil, + run = function (work_scope, setup) + return entry.run(work_scope, setup and setup.request_owner or entry.owner) + end, + report = function (ev) + return queue.try_admit_required(ctx.done_tx, ev, 'update_ingest_completion_report_failed') + end, + } + if not handle then + inst.active_request_id = nil + fail_owner(entry.owner, entry.req, err or 'ingest_work_start_failed') + return nil, err + end + state._work[entry.request_id] = { handle = handle, ingest_id = inst.ingest_id, category = entry.category } + return handle, nil +end + +start_next_for_instance = function (ctx, state, inst) + if not inst or inst.active_request_id ~= nil then return true end + local entry + repeat + entry = table.remove(inst.pending, 1) + if not entry then return true end + if entry_abandoned(entry) then entry = nil end + until entry ~= nil + inst.active_request_id = entry.request_id + local handle, err = start_ingest_work(ctx, state, inst, entry) + if not handle then + inst.active_request_id = nil + return nil, err + end + return true, nil +end + +new_request_id = function(state, method, ingest_id) + local n = state._next_request_id or 1 + state._next_request_id = n + 1 + return table.concat({ tostring(ingest_id or 'ingest'), tostring(method or 'request'), tostring(n) }, ':') +end + +local function enqueue_instance_work(ctx, state, inst, req, method, payload, run) + local cat = category_for(method) + if cat == 'append' then + if inst.state ~= 'open' or inst.terminal_requested then + fail(req, 'ingest_closing') + return true + end + elseif cat == 'commit' or cat == 'abort' then + if inst.closed then + fail(req, 'ingest_closed') + return true + end + if inst.terminal_requested then + fail(req, 'ingest_terminal_in_progress') + return true + end + inst.terminal_requested = true + -- Close append admission immediately, but do not mutate the + -- instance lifecycle state yet. An already-admitted append may still + -- be active and must be allowed to complete while the sink remains + -- logically open. The terminal worker changes state when it actually + -- owns the lane. + local kept = {} + for _, pending in ipairs(inst.pending) do + if pending.category == 'append' then + fail_owner(pending.owner, pending.req, 'ingest_closing') + else + kept[#kept + 1] = pending + end + end + inst.pending = kept + else + fail(req, 'unsupported_ingest_method: ' .. tostring(method)) + return true + end + + local entry = { + request_id = tostring(payload.request_id or new_request_id(state, method, inst.ingest_id)), + req = req, + owner = request_owner.new(req), + method = method, + payload = payload, + category = cat, + run = run, + } + inst.pending[#inst.pending + 1] = entry + return start_next_for_instance(ctx, state, inst) +end + +function State:handle_event(ctx, ev) + self._ctx = ctx or self._ctx + ctx = ctx or self._ctx + if ev.kind == 'ingest_closed' then error('update ingest queue closed', 0) end + local req = ev.request + local method, payload = method_of(req) + + if method == 'ingest_create' then + create_instance_now(ctx, self, req, payload) + return true + end + + local ingest_id = payload.ingest_id + local inst = ingest_id and self._instances[ingest_id] or nil + if not inst then fail(req, 'ingest_not_found'); return true end + + if method == 'ingest_append' then + enqueue_instance_work(ctx, self, inst, req, method, payload, function (_, owner) + if type(payload.chunk) ~= 'string' then error('ingest_append_chunk_required', 0) end + local result, err = fibers.perform(assert(inst:append_op(payload.chunk))) + if not result then error(err or 'ingest_append_failed', 0) end + local ok, rerr = owner:reply_once({ ok = true, append = result }) + if ok ~= true then error(rerr or 'ingest_append_reply_failed', 0) end + return { tag = 'ingest_appended', ingest_id = ingest_id, result = result } + end) + return true + end + + if method == 'ingest_commit' then + enqueue_instance_work(ctx, self, inst, req, method, payload, function (_, owner) + local ok_begin, berr = inst:begin_commit() + if ok_begin ~= true then error(berr or 'ingest_commit_begin_failed', 0) end + local result, err = inst:commit_worker(nil) + if not result then error(err or 'ingest_commit_failed', 0) end + local ok, rerr = owner:reply_once({ ok = true, commit = result }) + if ok ~= true then error(rerr or 'ingest_commit_reply_failed', 0) end + return { tag = 'ingest_committed', ingest_id = ingest_id, result = result } + end) + return true + end + + if method == 'ingest_abort' then + enqueue_instance_work(ctx, self, inst, req, method, payload, function (_, owner) + local ok_abort, aerr = inst:abort(payload.reason or 'ingest_abort') + if ok_abort ~= true then error(aerr or 'ingest_abort_failed', 0) end + local ok, rerr = owner:reply_once({ ok = true, aborted = true }) + if ok ~= true then error(rerr or 'ingest_abort_reply_failed', 0) end + return { tag = 'ingest_aborted', ingest_id = ingest_id } + end) + return true + end + + fail(req, 'unsupported_ingest_method: ' .. tostring(method)) + return true +end + +function State:handle_done(ctx, ev) + if ev == nil then return true end + if ev.kind ~= 'ingest_request_done' and ev.kind ~= 'ingest_create_done' then return true end + self._ctx = ctx or self._ctx + ctx = ctx or self._ctx + + local rec = self._work[ev.request_id] + self._work[ev.request_id] = nil + local ingest_id = ev.ingest_id or (rec and rec.ingest_id) + + if ev.kind == 'ingest_create_done' then + local result = ev.result or {} + local owner = result.request_owner or (rec and rec.owner) + local req = rec and rec.request or nil + if ev.status ~= 'ok' then + fail_owner(owner, req, ev.primary or 'ingest_create_failed') + return true + end + create_instance_with_sink_now(self, req, result.payload or {}, result.sink, owner) + return true + end + + local inst = ingest_id and self._instances[ingest_id] or nil + if not inst then return true end + + inst.active_request_id = nil + + if ev.status ~= 'ok' then + if ev.category == 'commit' or ev.category == 'abort' then + inst.terminal_requested = false + if not inst.closed then inst.state = 'open' end + end + else + local result = ev.result or {} + if result.tag == 'ingest_committed' then + inst.state = 'committed' + inst.closed = true + elseif result.tag == 'ingest_aborted' then + inst.state = 'aborted' + inst.closed = true + end + end + + if inst.closed then + -- Requests admitted before a terminal completion but not yet run are failed + -- now. Later requests are rejected at admission by state checks. + while #inst.pending > 0 do + local pending = table.remove(inst.pending, 1) + fail_owner(pending.owner, pending.req, 'ingest_closed') + end + inst:_close_scope('ingest_closed') + return true + end + + if ctx then + start_next_for_instance(ctx, self, inst) + end + return true +end + +M.State = State +M.Instance = Instance +return M diff --git a/src/services/update/job_repository.lua b/src/services/update/job_repository.lua new file mode 100644 index 00000000..5e9f3d2f --- /dev/null +++ b/src/services/update/job_repository.lua @@ -0,0 +1,438 @@ +-- services/update/job_repository.lua +-- Pure durable-job model helpers. No Ops, no HAL, no bus. +-- +-- The repository is intentionally compact. Update jobs are operational state, +-- not an audit archive: public/durable forms retain the single current or last +-- job with only the fields needed to continue operation, render UI state, and +-- emit coarse telemetry. + +local model = require 'services.update.model' +local M = {} + +local TERMINAL = { succeeded=true, failed=true, cancelled=true, timed_out=true, superseded=true, discarded=true } + +local function copy(v) return model.deep_copy(v) end +local function count(t) local n=0 for _ in pairs(t or {}) do n=n+1 end return n end + +local function sorted_ids(jobs) + local ids = {} + for id in pairs(jobs or {}) do ids[#ids+1] = id end + table.sort(ids, function(a, b) + local ja, jb = jobs[a] or {}, jobs[b] or {} + local ta = ja.updated_seq or ja.created_seq or ja.created_mono or 0 + local tb = jb.updated_seq or jb.created_seq or jb.created_mono or 0 + if ta == tb then return tostring(a) < tostring(b) end + return ta < tb + end) + return ids +end + +local function first_table(...) + for i = 1, select('#', ...) do + local v = select(i, ...) + if type(v) == 'table' then return v end + end + return nil +end + +local function artifact_ref_of(job) + if type(job) ~= 'table' then return nil end + local art = first_table(job.artifact, job.artifact_snapshot, job.artifact_meta) + return job.artifact_ref + or job.ref + or (art and (art.artifact_ref or art.ref or art.id)) +end + +local function compact_transfer(job) + if type(job) ~= 'table' then return nil end + local stage = type(job.stage_result) == 'table' and job.stage_result or nil + local reply = stage and type(stage.reply) == 'table' and stage.reply or nil + local inner = reply and (type(reply.transfer) == 'table' and reply.transfer or reply) or nil + local result = inner and type(inner.result) == 'table' and inner.result or nil + local out = {} + local function absorb(src) + if type(src) ~= 'table' then return end + for _, key in ipairs({ 'xfer_id', 'request_id', 'link_id', 'target', 'digest_alg', 'digest', 'size', 'sent_bytes', 'retransmits' }) do + if out[key] == nil and src[key] ~= nil then out[key] = src[key] end + end + end + absorb(type(job.transfer) == 'table' and job.transfer or nil) + absorb(stage and type(stage.transfer) == 'table' and stage.transfer or nil) + absorb(result) + absorb(inner) + return next(out) ~= nil and out or nil +end + +local function compact_stage_result(stage, job) + if type(stage) ~= 'table' then return nil end + local pre = type(stage.preflight) == 'table' and stage.preflight or {} + local out = { + staged = stage.staged == true or nil, + component = stage.component or pre.component or (job and job.component), + expected_image_id = stage.expected_image_id or pre.expected_image_id or (job and job.expected_image_id), + transfer = compact_transfer({ stage_result = stage }), + } + local preflight = { + artifact_ref = pre.artifact_ref or pre.ref, + format = pre.format, + image_id = pre.image_id, + expected_image_id = pre.expected_image_id, + size = pre.size, + digest_alg = pre.digest_alg, + digest = pre.digest, + payload_sha256 = pre.payload_sha256, + } + if next(preflight) ~= nil then out.preflight = preflight end + return next(out) ~= nil and out or nil +end + +local function compact_pre_commit(pre) + if type(pre) ~= 'table' then return nil end + local out = { + component = pre.component, + expected_image_id = pre.expected_image_id, + pre_commit_image_id = pre.pre_commit_image_id, + pre_commit_boot_id = pre.pre_commit_boot_id, + transfer = type(pre.transfer) == 'table' and copy(pre.transfer) or nil, + } + return next(out) ~= nil and out or nil +end + +local function compact_commit_attempt(attempt) + if type(attempt) ~= 'table' then return nil end + local out = { + token = attempt.token, + policy = attempt.policy or attempt.commit_policy, + active_token = attempt.active_token, + generation = attempt.generation, + acceptance = attempt.acceptance, + accepted = attempt.accepted, + started_seq = attempt.started_seq, + accepted_seq = attempt.accepted_seq, + failed_seq = attempt.failed_seq, + error = attempt.error, + pre_commit = compact_pre_commit(attempt.pre_commit), + } + return next(out) ~= nil and out or nil +end + +local function compact_result(result) + if type(result) ~= 'table' then return result end + local out = { + tag = result.tag, + ok = result.ok, + reason = result.reason, + error = result.error, + state = result.state and type(result.state) ~= 'table' and result.state or nil, + expected_image_id = result.expected_image_id, + image_id = result.image_id, + commit_token = result.commit_token, + commit_policy = result.commit_policy, + previous_state = result.previous_state, + restart_attempts = result.restart_attempts, + restart_max = result.restart_max, + } + return next(out) ~= nil and out or nil +end + +local function compact_active_intent(intent) + if type(intent) ~= 'table' then return nil end + local out = { + token = intent.token, + phase = intent.phase, + generation = intent.generation, + request_id = intent.request_id, + state = intent.state, + reason = intent.reason, + restart_count = intent.restart_count, + attempt = intent.attempt, + restart_max = intent.restart_max, + commit_token = intent.commit_token, + commit_policy = intent.commit_policy, + } + return next(out) ~= nil and out or nil +end + +local function compact_adoption(adoption) + if type(adoption) ~= 'table' then return nil end + local out = { + action = adoption.action, + reason = adoption.reason, + from_state = adoption.from_state, + restart_attempts = adoption.restart_attempts, + attempt = adoption.attempt, + restart_max = adoption.restart_max, + seq = adoption.seq, + } + return next(out) ~= nil and out or nil +end + +local function compact_policy(policy) + if type(policy) ~= 'table' then return nil end + local out = {} + local keys = { + 'job_id', 'create_if', 'start', 'commit', 'reconcile', 'supersede', + 'attempt', 'max_attempts', + } + for _, key in ipairs(keys) do + out[key] = policy[key] + end + return next(out) ~= nil and out or nil +end + +local function last_event_from(job) + if type(job) ~= 'table' then return nil end + if type(job.last_event) == 'table' then return copy(job.last_event) end + local hist = type(job.history) == 'table' and job.history or nil + local last = hist and hist[#hist] or nil + if type(last) ~= 'table' then return nil end + return { seq = last.seq, state = last.state, reason = last.reason } +end + +local function is_terminal_state(state) + return TERMINAL[state] == true +end + +function M.compact_job(job) + if type(job) ~= 'table' then return nil end + local state = job.state or 'created' + local terminal = is_terminal_state(state) + local out = { + schema = 'devicecode.update.job/2', + job_id = job.job_id, + component = job.component, + state = state, + artifact_ref = artifact_ref_of(job), + expected_image_id = job.expected_image_id, + created_seq = job.created_seq or 0, + updated_seq = job.updated_seq or job.created_seq or 0, + error = job.error, + result = compact_result(job.result), + transfer = compact_transfer(job), + last_event = last_event_from(job), + } + if not terminal then + out.next_step = job.next_step + out.generation = job.generation + end + + if terminal then + -- Keep the compact commit attempt for compatibility with existing update + -- reconciliation/status consumers. This is small and operationally meaningful; + -- the large stage/transfer reply remains collapsed into transfer/result fields. + out.commit_attempt = compact_commit_attempt(job.commit_attempt) + end + if terminal and state == 'failed' then + local adoption = compact_adoption(job.adoption) + if adoption and adoption.action == 'failed_restart_limit' then out.adoption = adoption end + end + + if not terminal then + local metadata = type(job.metadata) == 'table' and copy(job.metadata) or nil + -- Metadata is operational input for component prepare/stage calls. Keep it + -- while a job can still be staged or committed, but terminal compaction + -- deliberately drops it. + out.metadata = metadata and next(metadata) ~= nil and metadata or nil + out.ingest_id = job.ingest_id or (metadata and metadata.ingest_id or nil) + out.attempt = job.attempt + out.active_token = job.active_token + out.active_intent = compact_active_intent(job.active_intent) + out.active = compact_active_intent(job.active) + out.adoption = compact_adoption(job.adoption) + out.stage_result = compact_stage_result(job.stage_result, job) + out.commit_attempt = compact_commit_attempt(job.commit_attempt) + out.commit_result = compact_result(job.commit_result) + out.policy = compact_policy(job.policy) + if out.active == nil and out.active_intent ~= nil then out.active = copy(out.active_intent) end + -- Preserve the one runtime field that can affect admission while a save is pending. + if type(job.runtime) == 'table' and job.runtime.persistence_pending then + out.runtime = { persistence_pending = true } + end + end + + return out +end + +local function strip_runtime(job) + return M.compact_job(job) +end + +local function normalise_job(job) + if type(job) ~= 'table' then return nil, 'job must be a table' end + local id = job.job_id + if type(id) ~= 'string' or id == '' then return nil, 'job_id required' end + local component = job.component + if type(component) ~= 'string' or component == '' then return nil, 'component required' end + if component == 'mcu' and (type(job.expected_image_id) ~= 'string' or job.expected_image_id == '') then + return nil, 'expected_image_id_required' + end + + if job.phase ~= nil or job.stage ~= nil then return nil, 'job lifecycle must use state, not phase or stage' end + + local out = M.compact_job(job) + out.job_id = id + out.component = component + out.state = out.state or 'created' + out.phase = nil + out.stage = nil + out.created_seq = out.created_seq or 0 + out.updated_seq = out.updated_seq or out.created_seq or 0 + return out, nil +end + +function M.new_state(initial) + local jobs = {} + for id, job in pairs((initial and initial.jobs) or {}) do + local normal, err = normalise_job(job) + if not normal then + -- Legacy stores may contain malformed records from development builds. Keep a + -- minimal tombstone long enough for the startup scrub to delete it, rather + -- than failing the service before it can reclaim storage. + jobs[id] = { + schema = 'devicecode.update.job/2', + job_id = type(job) == 'table' and job.job_id or tostring(id), + component = type(job) == 'table' and job.component or 'unknown', + state = 'invalid_legacy', + error = err or 'invalid_legacy_job', + created_seq = type(job) == 'table' and tonumber(job.created_seq) or 0, + updated_seq = type(job) == 'table' and tonumber(job.updated_seq) or 0, + } + else + jobs[id] = normal + end + end + return { jobs = jobs, order = sorted_ids(jobs), next_seq = initial and initial.next_seq or 1, dirty = {} }, nil +end + +function M.is_terminal(state) return TERMINAL[state] == true end +function M.normalise_job(job) return normalise_job(job) end + +function M.snapshot(repo) + local by_id = {} + for id, job in pairs(repo.jobs or {}) do by_id[id] = strip_runtime(job) end + return { count = count(by_id), order = copy(repo.order or sorted_ids(by_id)), by_id = by_id } +end + +function M.list(repo) + local out = {} + for _, id in ipairs(repo.order or sorted_ids(repo.jobs)) do + if repo.jobs[id] then out[#out+1] = strip_runtime(repo.jobs[id]) end + end + return out +end + +function M.get(repo, job_id) + local job = repo.jobs and repo.jobs[job_id] + return job and strip_runtime(job) or nil +end + +function M.upsert(repo, job) + local normal, err = normalise_job(job) + if not normal then return nil, err end + local id = normal.job_id + local existed = repo.jobs[id] ~= nil + repo.jobs[id] = normal + if not existed then repo.order[#repo.order+1] = id end + repo.order = sorted_ids(repo.jobs) + repo.dirty[id] = true + return strip_runtime(normal), nil +end + +function M.replace(repo, job_id, job) + local normal, err = normalise_job(job) + if not normal then return nil, err end + repo.jobs[job_id or normal.job_id] = normal + repo.order = sorted_ids(repo.jobs) + repo.dirty[job_id or normal.job_id] = true + return strip_runtime(normal), nil +end + +function M.remove(repo, job_id) + if not (repo.jobs and repo.jobs[job_id]) then return false, 'not_found' end + repo.jobs[job_id] = nil + repo.dirty[job_id] = true + for i = #repo.order, 1, -1 do + if repo.order[i] == job_id then table.remove(repo.order, i) end + end + return true, nil +end + +function M.next_sequence(repo) + local n = repo.next_seq or 1 + repo.next_seq = n + 1 + return n +end + +function M.new_job(spec, opts) + spec = spec or {}; opts = opts or {} + local id = spec.job_id or opts.job_id + if type(id) ~= 'string' or id == '' then return nil, 'job_id required' end + local component = spec.component + if type(component) ~= 'string' or component == '' then return nil, 'component required' end + local expected_image_id = spec.expected_image_id + if component == 'mcu' and (type(expected_image_id) ~= 'string' or expected_image_id == '') then + return nil, 'expected_image_id_required' + end + local seq = opts.seq or 0 + return M.compact_job({ + job_id = id, + component = component, + expected_image_id = expected_image_id, + state = 'created', + next_step = 'start', + generation = opts.generation, + artifact = copy(spec.artifact or spec.artifact_ref), + artifact_ref = spec.artifact_ref, + attempt = spec.attempt, + metadata = copy(spec.metadata or {}), + policy = copy(spec.policy), + created_seq = seq, + updated_seq = seq, + last_event = { seq = seq, state = 'created', reason = opts.reason or 'create_job' }, + }), nil +end + +local function bump(job, seq, reason) + job.updated_seq = seq or ((job.updated_seq or 0) + 1) + job.last_event = { seq = job.updated_seq, state = job.state, reason = reason } + job.history = nil +end + +function M.patch(job, patch, opts) + if type(job) ~= 'table' then return nil, 'job required' end + patch = patch or {}; opts = opts or {} + if patch.phase ~= nil or patch.stage ~= nil then return nil, 'job lifecycle must use state, not phase or stage' end + for k, v in pairs(patch) do + job[k] = copy(v) + end + bump(job, opts.seq, opts.reason or patch.reason or 'patch') + local compact = M.compact_job(job) + for k in pairs(job) do job[k] = nil end + for k, v in pairs(compact or {}) do job[k] = v end + return strip_runtime(job), nil +end + +function M.mark_staging(job, opts) + return M.patch(job, { state = 'staging', next_step = nil, error = nil }, opts) +end + +function M.mark_awaiting_commit(job, result, opts) + return M.patch(job, { state = 'awaiting_commit', next_step = 'commit', stage_result = copy(result), error = nil }, opts) +end + +function M.mark_committing(job, opts) + return M.patch(job, { state = 'committing', next_step = nil, error = nil }, opts) +end + +function M.mark_awaiting_return(job, result, opts) + return M.patch(job, { state = 'awaiting_return', next_step = 'reconcile', commit_result = copy(result), error = nil }, opts) +end + +function M.mark_terminal(job, state, reason, result, opts) + state = state or 'failed' + return M.patch(job, { state = state, next_step = nil, error = reason, result = copy(result) }, opts) +end + +function M.public_job(job) return strip_runtime(job) end +M.TERMINAL = TERMINAL +return M diff --git a/src/services/update/job_runtime.lua b/src/services/update/job_runtime.lua new file mode 100644 index 00000000..b793e826 --- /dev/null +++ b/src/services/update/job_runtime.lua @@ -0,0 +1,1867 @@ +-- services/update/job_runtime.lua +-- +-- Service-owned durable job runtime. +-- +-- This is the only owner of authoritative durable job state. Callers submit +-- transition requests and receive completion values after the transition has +-- been durably applied. Generations and active workers may observe snapshots +-- and request transitions; they must not mutate durable jobs or call the job +-- store directly. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local cond = require 'fibers.cond' +local mailbox = require 'fibers.mailbox' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' + +local model_mod = require 'services.update.model' +local repo_mod = require 'services.update.job_repository' +local active_policy = require 'services.update.active_policy' + +local M = {} + +local DEFAULT_QUEUE = 32 +local MAX_TRANSITION_RECORDS = 8 + +local Runtime = {} +Runtime.__index = Runtime + +local function copy(v) + return model_mod.deep_copy(v) +end + +local function count_keys(t) + local n = 0 + for _ in pairs(t or {}) do n = n + 1 end + return n +end + +local function new_id(prefix) + local ok_uuid, uuid = pcall(require, 'uuid') + if ok_uuid and uuid and type(uuid.new) == 'function' then + return tostring(uuid.new()) + end + return (prefix or 'job') .. '-' .. tostring(math.floor((fibers.now() or 0) * 1000000)) +end + +local TRANSITION_STATES = { + proposed = true, + admitted = true, + persisting = true, + persisted = true, + rejected = true, + failed = true, + cancelled = true, +} + +local function transition_public(cmd) + cmd = type(cmd) == 'table' and cmd or {} + return { + transition = cmd.kind, + job_id = cmd.job_id or (cmd.payload and cmd.payload.job_id), + generation = cmd.generation, + phase = cmd.phase, + token = cmd.token, + reason = cmd.reason, + } +end + +local function transition_record(req, state, details) + details = details or {} + local cmd = req and req.cmd or {} + local base = transition_public(cmd) + base.transition_id = req and req.id or details.transition_id + base.state = state or 'proposed' + base.status = state or 'proposed' + base.sequence = details.sequence + base.started = details.started + base.finished = details.finished + base.error = details.error + base.plan_kind = details.plan_kind + if state == 'admitted' + or state == 'persisting' + or state == 'persisted' + or details.admitted == true + or (state == 'failed' and details.plan_kind ~= nil) + then + base.admitted = true + end + return base +end + +local function copy_transition_record(rec) + return copy(rec or {}) +end + +local function transition_snapshot(self) + local by_id = {} + local order = {} + for _, id in ipairs(self._transition_order or {}) do + local rec = self._transitions and self._transitions[id] + if rec then + order[#order + 1] = id + by_id[id] = copy_transition_record(rec) + end + end + return { count = #order, order = order, by_id = by_id } +end + +local function transition_set(self, req, state, details) + if not TRANSITION_STATES[state] then + error('invalid transition lifecycle state: ' .. tostring(state), 2) + end + + details = details or {} + self._transitions = self._transitions or {} + self._transition_order = self._transition_order or {} + + local id = assert(req and req.id, 'transition request missing id') + local rec = transition_record(req, state, details) + local existing = self._transitions[id] + + if existing == nil then + self._transitions[id] = rec + self._transition_order[#self._transition_order + 1] = id + else + -- Current lifecycle fields must describe the current state, not a + -- breadcrumb trail. In particular, a rejected-before-admission request must + -- not retain admitted=true from an earlier provisional record. + for k in pairs(existing) do + existing[k] = nil + end + for k, v in pairs(rec) do + existing[k] = v + end + rec = existing + end + + if state == 'rejected' or state == 'failed' or state == 'persisted' or state == 'cancelled' then + rec.finished = true + end + + return rec +end + +local function transition_outcome_from_record(rec, value) + local out = copy(value or {}) + out.transition_id = rec.transition_id + out.transition = out.transition or rec.transition + out.job_id = out.job_id or rec.job_id + out.generation = out.generation or rec.generation + out.phase = out.phase or rec.phase + out.token = out.token or rec.token + out.transition_state = rec.state + out.lifecycle = rec.state + return out +end + +local function load_jobs(params) + params = params or {} + local store = params.store + local loaded, err + + if store and type(store.load_all_op) == 'function' then + loaded, err = fibers.perform(store:load_all_op()) + if not loaded then return nil, err or 'job_load_failed' end + else + loaded = params.initial_jobs or { jobs = {}, order = {} } + end + + local jobs, jerr = repo_mod.new_state(loaded) + if not jobs then return nil, jerr end + return jobs, nil +end + +local function public_job(job) + return job and repo_mod.public_job(job) or nil +end + +local function adoption_empty() + return { + awaiting_commit = {}, + awaiting_return = {}, + active_intent = {}, + failed = {}, + pruned = {}, + decisions = {}, + diagnostics = {}, + } +end + + +local DEFAULT_ACTIVE_INTENT_RESTART_MAX = 1 + +local function optional_number(v) + if type(v) == 'number' then return v end + if type(v) == 'string' then return tonumber(v) end + return nil +end + +local function retention_active_intent_restart_max(params) + local retention = type(params and params.retention) == 'table' and params.retention or {} + local n = optional_number(retention.active_intent_restart_max) + if n == nil and type(params) == 'table' then + n = optional_number(params.active_intent_restart_max) + end + if type(n) ~= 'number' or n < 0 or n ~= math.floor(n) then + return DEFAULT_ACTIVE_INTENT_RESTART_MAX + end + return n +end + +local function max_number(...) + local out = 0 + for i = 1, select('#', ...) do + local n = optional_number(select(i, ...)) + if type(n) == 'number' and n > out then out = n end + end + return out +end + +local function history_active_restart_count(job) + local n = 0 + local hist = type(job) == 'table' and type(job.history) == 'table' and job.history or {} + for _, row in ipairs(hist) do + local reason = type(row) == 'table' and row.reason or nil + if reason == 'restart_adoption_active_intent' + or reason == 'restart_active_intent' + or reason == 'restart_adoption_active_intent_limit_exceeded' + then + n = n + 1 + end + end + return n +end + +local function active_intent_restart_count(job, intent) + intent = type(intent) == 'table' and intent or {} + local adoption = type(job) == 'table' and type(job.adoption) == 'table' and job.adoption or {} + local runtime = type(job) == 'table' and type(job.runtime) == 'table' and job.runtime or {} + return max_number( + intent.restart_count, + intent.restart_attempts, + intent.attempt, + adoption.restart_count, + adoption.restart_attempts, + adoption.attempt, + runtime.active_intent_restart_count, + runtime.active_intent_restart_attempts, + runtime.active_intent_attempt, + type(job) == 'table' and job.active_intent_restart_count or nil, + type(job) == 'table' and job.active_intent_restart_attempts or nil, + type(job) == 'table' and job.active_intent_attempt or nil, + history_active_restart_count(job) + ) +end + +local function fail_active_intent_restart_limit(job, state, intent, next_count, max_count, seq) + repo_mod.mark_terminal(job, 'failed', 'active_intent_restart_limit_exceeded', { + previous_state = state, + restart_attempts = next_count, + restart_max = max_count, + active_intent = copy(intent), + }, { seq = seq, reason = 'restart_adoption_active_intent_limit_exceeded' }) + job.adoption = { + action = 'failed_restart_limit', + reason = 'active_intent_restart_limit_exceeded', + from_state = state, + restart_attempts = next_count, + attempt = next_count, + restart_max = max_count, + seq = seq, + } + job.active = nil + job.active_token = nil + job.active_intent = nil + job.runtime = copy(job.runtime or {}) + job.runtime.active_intent_restart_count = next_count + job.runtime.active_intent_attempt = next_count + job.runtime.active_intent_restart_max = max_count + return job +end + +local function clear_terminal_active_markers(job, state, seq) + repo_mod.patch(job, { + adoption = { + action = 'cleared_terminal_active_markers', + reason = 'restart_terminal_active_markers', + from_state = state, + seq = seq, + }, + }, { seq = seq, reason = 'restart_clear_terminal_active_markers' }) + -- repo.patch cannot clear fields via nil values because pairs() does not carry + -- them. Clear stale ownership markers explicitly so persisted terminal jobs + -- cannot block durable_active_owner after a restart. + job.active = nil + job.active_token = nil + job.active_intent = nil + return job +end + + +local function runtime_snapshot(self_or_jobs, ready, adoption) + local jobs = self_or_jobs + local transitions + if type(self_or_jobs) == 'table' and self_or_jobs._jobs ~= nil then + jobs = self_or_jobs._jobs + transitions = transition_snapshot(self_or_jobs) + end + return { + ready = not not ready, + jobs = repo_mod.snapshot(jobs), + adoption = copy(adoption or {}), + transitions = transitions, + count = count_keys(jobs and jobs.jobs or {}), + } +end + +local function refresh_model(self) + if self and self._model then + self._model:set_snapshot(runtime_snapshot(self, self._ready, self._adoption)) + end +end + +local function resolve_cell(cell, value, err) + if not cell or cell.done then return false end + cell.done = true + cell.value = value + cell.err = err + cell.cond:signal() + return true +end + +local function transition_result_op(cell) + return op.guard(function () + if cell.done then + return op.always(cell.value, cell.err) + end + + return cell.cond:wait_op():wrap(function () + return cell.value, cell.err + end) + end) +end + +local function make_cell() + return { + cond = cond.new(), + done = false, + value = nil, + err = nil, + } +end + +local TransitionHandle = {} +TransitionHandle.__index = TransitionHandle + +function TransitionHandle:transition_id() + return self._id +end + +function TransitionHandle:command() + return copy(self._cmd) +end + +function TransitionHandle:outcome_op() + return transition_result_op(self._cell) +end + +function TransitionHandle:outcome() + return fibers.perform(self:outcome_op()) +end + +local function new_transition_handle(req) + return setmetatable({ + _id = req.id, + _cmd = copy(req.cmd), + _cell = req.cell, + }, TransitionHandle) +end + +local function store_save_op(store, job) + if not store or type(store.save_job_op) ~= 'function' then + return op.always(true, nil) + end + return store:save_job_op(job) +end + +local function store_delete_op(store, job_id) + if not store or type(store.delete_job_op) ~= 'function' then + return op.always(true, nil) + end + return store:delete_job_op(job_id) +end + +local function sorted_job_ids(jobs) + local ids = {} + for id in pairs((jobs and jobs.jobs) or {}) do ids[#ids + 1] = id end + table.sort(ids) + return ids +end + +local function next_adoption_seq(jobs) + local seq = jobs.next_seq or 1 + jobs.next_seq = seq + 1 + return seq +end + +local function save_adopted_job(self, jobs, job) + local saved, uerr = repo_mod.upsert(jobs, job) + if not saved then return nil, uerr end + local ok, serr = fibers.perform(store_save_op(self._store, public_job(job))) + if ok ~= true then return nil, serr or 'adoption_save_failed' end + return true, nil +end + +local function adopt_restart_jobs(self, jobs) + local adoption = adoption_empty() + local active_restart_max = retention_active_intent_restart_max(self._params or {}) + + for _, id in ipairs(sorted_job_ids(jobs)) do + local current = jobs.jobs[id] + local state = current and current.state + if state == 'awaiting_commit' then + -- A staged-but-uncommitted job surviving restart is still visible and + -- committable, but adoption is a run-local observation. Do not patch, + -- bump or resave the durable job merely because the service restarted. + local pub = public_job(current) + adoption.awaiting_commit[#adoption.awaiting_commit + 1] = pub + adoption.decisions[#adoption.decisions + 1] = { + job_id = id, + from_state = state, + action = 'kept_committable', + reason = 'restart_awaiting_commit_run_local', + job = pub, + } + elseif state == 'awaiting_return' then + local job = copy(current) + local seq = next_adoption_seq(jobs) + repo_mod.patch(job, { + adoption = { + action = 'reconcile_required', + reason = 'restart_awaiting_return', + seq = seq, + }, + }, { seq = seq, reason = 'restart_adoption_awaiting_return' }) + local ok, err = save_adopted_job(self, jobs, job) + if ok ~= true then return nil, err end + local pub = public_job(job) + adoption.awaiting_return[#adoption.awaiting_return + 1] = pub + adoption.decisions[#adoption.decisions + 1] = { + job_id = id, + from_state = state, + action = 'reconcile_required', + job = pub, + } + elseif state == 'staging' or state == 'committing' then + local has_intent = current.active_token ~= nil or type(current.active_intent) == 'table' + local job = copy(current) + local seq = next_adoption_seq(jobs) + local intent = type(job.active_intent) == 'table' and job.active_intent or nil + local phase = intent and intent.phase or nil + local attempt = type(job.commit_attempt) == 'table' and job.commit_attempt or nil + local commit_policy = attempt and (attempt.policy or attempt.commit_policy) or (intent and intent.commit_policy) + + if has_intent and state == 'committing' and phase == 'commit' and commit_policy == 'no_duplicate' then + repo_mod.mark_awaiting_return(job, { + tag = 'commit_uncertain_no_duplicate', + commit_token = attempt and attempt.token or intent.commit_token or job.active_token, + commit_policy = commit_policy, + }, { seq = seq, reason = 'restart_commit_no_duplicate_reconcile' }) + job.adoption = { + action = 'reconcile_required', + reason = 'restart_commit_no_duplicate', + from_state = state, + seq = seq, + } + if attempt then + job.commit_attempt = copy(attempt) + job.commit_attempt.acceptance = job.commit_attempt.acceptance or 'unknown' + job.commit_attempt.adoption = 'no_duplicate_reconcile' + end + job.active = nil + job.active_token = nil + job.active_intent = nil + local ok, err = save_adopted_job(self, jobs, job) + if ok ~= true then return nil, err end + local pub = public_job(job) + adoption.awaiting_return[#adoption.awaiting_return + 1] = pub + adoption.decisions[#adoption.decisions + 1] = { + job_id = id, + from_state = state, + action = 'reconcile_required', + reason = 'restart_commit_no_duplicate', + job = pub, + } + elseif has_intent and (state ~= 'committing' or phase ~= 'commit' or commit_policy == 'idempotent_by_token') then + local restart_count = active_intent_restart_count(job, intent) + local next_restart_count = restart_count + 1 + if next_restart_count > active_restart_max then + fail_active_intent_restart_limit(job, state, intent, next_restart_count, active_restart_max, seq) + local ok, err = save_adopted_job(self, jobs, job) + if ok ~= true then return nil, err end + local pub = public_job(job) + adoption.failed[#adoption.failed + 1] = pub + adoption.decisions[#adoption.decisions + 1] = { + job_id = id, + from_state = state, + action = 'failed_restart_limit', + reason = 'active_intent_restart_limit_exceeded', + restart_attempts = next_restart_count, + attempt = next_restart_count, + restart_max = active_restart_max, + job = pub, + } + adoption.diagnostics[#adoption.diagnostics + 1] = { + job_id = id, + message = 'active intent restart limit exceeded; job failed and slot released', + restart_attempts = next_restart_count, + attempt = next_restart_count, + restart_max = active_restart_max, + } + else + job.active_intent = intent + job.active_intent.state = 'adopted' + job.active_intent.reason = 'restart_active_intent' + job.active_intent.restart_count = next_restart_count + job.active_intent.attempt = next_restart_count + job.active_intent.restart_max = active_restart_max + job.active = copy(job.active_intent) + job.runtime = copy(job.runtime or {}) + job.runtime.active_intent_restart_count = next_restart_count + job.runtime.active_intent_attempt = next_restart_count + job.runtime.active_intent_restart_max = active_restart_max + repo_mod.patch(job, { + active_intent = job.active_intent, + active = job.active, + runtime = job.runtime, + adoption = { + action = 'resume_active_intent', + reason = 'restart_active_intent', + from_state = state, + restart_attempts = next_restart_count, + attempt = next_restart_count, + restart_max = active_restart_max, + seq = seq, + }, + }, { seq = seq, reason = 'restart_adoption_active_intent' }) + local ok, err = save_adopted_job(self, jobs, job) + if ok ~= true then return nil, err end + local pub = public_job(job) + adoption.active_intent[#adoption.active_intent + 1] = pub + adoption.decisions[#adoption.decisions + 1] = { + job_id = id, + from_state = state, + action = 'resume_active_intent', + restart_attempts = next_restart_count, + attempt = next_restart_count, + restart_max = active_restart_max, + job = pub, + } + end + else + local reason = (state == 'committing' and phase == 'commit') and 'restart_commit_policy_missing' or ('restart_interrupted_' .. tostring(state)) + repo_mod.mark_terminal(job, 'failed', reason, { + previous_state = state, + }, { seq = seq, reason = 'restart_adoption_failed_' .. tostring(reason) }) + job.adoption = { + action = 'failed_impossible_state', + reason = reason, + from_state = state, + seq = seq, + } + job.active = nil + job.active_token = nil + job.active_intent = nil + local ok, err = save_adopted_job(self, jobs, job) + if ok ~= true then return nil, err end + local pub = public_job(job) + adoption.failed[#adoption.failed + 1] = pub + adoption.decisions[#adoption.decisions + 1] = { + job_id = id, + from_state = state, + action = 'failed_impossible_state', + reason = job.error, + job = pub, + } + adoption.diagnostics[#adoption.diagnostics + 1] = { + job_id = id, + message = 'job was interrupted during ' .. tostring(state) .. ' and was marked failed', + } + end + elseif repo_mod.TERMINAL[state] and (current.active_token ~= nil or type(current.active_intent) == 'table' or type(current.active) == 'table') then + local job = copy(current) + local seq = next_adoption_seq(jobs) + clear_terminal_active_markers(job, state, seq) + local ok, err = save_adopted_job(self, jobs, job) + if ok ~= true then return nil, err end + adoption.decisions[#adoption.decisions + 1] = { + job_id = id, + from_state = state, + action = 'cleared_terminal_active_markers', + } + end + end + + return adoption, nil +end + + +local function normalise_retention(params) + params = params or {} + local retention = type(params.retention) == 'table' and copy(params.retention) or {} + -- The startup scrub is a migration, not ordinary policy. It is intentionally + -- unconditional for this version so stale development records cannot preserve + -- the legacy archive by carrying old config. + retention.mode = 'single_job' + retention.max_jobs = 1 + retention.scrub_legacy_on_startup = true + retention.prune_on_startup = true + retention.keep_last_terminal = retention.keep_last_terminal ~= false + retention.retain_workflows = false + if params.active_intent_restart_max ~= nil then + retention.active_intent_restart_max = tonumber(params.active_intent_restart_max) + end + if retention.active_intent_restart_max == nil then + retention.active_intent_restart_max = DEFAULT_ACTIVE_INTENT_RESTART_MAX + end + return retention +end + +local function job_time(job) + return tonumber(job and (job.updated_seq or job.created_seq or job.created_mono)) or 0 +end + +local function newest_first(jobs, a, b) + local ja, jb = jobs.jobs[a] or {}, jobs.jobs[b] or {} + local ta, tb = job_time(ja), job_time(jb) + if ta == tb then return tostring(a) > tostring(b) end + return ta > tb +end + +local function recoverable_job(job) + if type(job) ~= 'table' then return false end + local state = job.state + if state == 'awaiting_return' then return true end + if state == 'committing' and type(job.commit_attempt) == 'table' then return true end + if (state == 'staging' or state == 'committing') and (job.active_token ~= nil or type(job.active_intent) == 'table') then + return true + end + return false +end + +local function select_single_job_to_keep(jobs) + local recoverable = {} + local terminal = {} + local any = {} + for id, job in pairs((jobs and jobs.jobs) or {}) do + if job and job.state ~= 'invalid_legacy' then any[#any + 1] = id end + if recoverable_job(job) then + recoverable[#recoverable + 1] = id + elseif repo_mod.is_terminal(job and job.state) then + terminal[#terminal + 1] = id + end + end + table.sort(recoverable, function(a, b) return newest_first(jobs, a, b) end) + if recoverable[1] ~= nil then return recoverable[1], 'recoverable' end + table.sort(terminal, function(a, b) return newest_first(jobs, a, b) end) + if terminal[1] ~= nil then return terminal[1], 'last_terminal' end + table.sort(any, function(a, b) return newest_first(jobs, a, b) end) + if any[1] ~= nil then return any[1], 'last_job' end + return nil, 'none' +end + +local function mark_pruned(adoption, job_id, job, reason) + adoption.pruned = adoption.pruned or {} + adoption.pruned[#adoption.pruned + 1] = { + job_id = job_id, + state = job and job.state, + action = 'deleted_legacy_job', + reason = reason or 'single_job_startup_scrub', + } + adoption.decisions[#adoption.decisions + 1] = { + job_id = job_id, + from_state = job and job.state, + action = 'deleted_legacy_job', + reason = reason or 'single_job_startup_scrub', + } +end + +local function job_has_legacy_cruft(job) + if type(job) ~= 'table' then return false end + if job.schema ~= nil and job.schema ~= 'devicecode.update.job/2' then return true end + local terminal = repo_mod.is_terminal(job.state) + for _, key in ipairs({ + 'history', 'stage_result', 'commit_result', 'active', + 'active_token', 'active_intent', 'runtime', 'generation', + }) do + if job[key] ~= nil then return true end + end + if not terminal and job.commit_attempt ~= nil then return true end + if not terminal and job.adoption ~= nil then return true end + return false +end + +local function save_compact_job(self, jobs, id, reason, opts) + local job = jobs.jobs[id] + if not job then return true, nil end + local compact = repo_mod.public_job(job) + local ok_replace, rerr = repo_mod.replace(jobs, id, compact) + if not ok_replace then return nil, rerr end + local force = type(opts) == 'table' and opts.force == true + if not force and not job_has_legacy_cruft(job) then + return true, nil + end + local ok, serr = fibers.perform(store_save_op(self._store, compact)) + if ok ~= true then return nil, serr or (reason or 'single_job_save_failed') end + return true, nil +end + +local function newest_created_waiter(jobs, keep_id) + local ids = {} + for id, job in pairs((jobs and jobs.jobs) or {}) do + if id ~= keep_id and type(job) == 'table' and job.state == 'created' then ids[#ids + 1] = id end + end + table.sort(ids, function(a, b) return newest_first(jobs, a, b) end) + return ids[1] +end + +local function scrub_legacy_jobs(self, jobs, adoption) + local keep_id, keep_reason = select_single_job_to_keep(jobs) + local keep = {} + if keep_id ~= nil then keep[keep_id] = true end + -- Compatibility exception: if a genuinely active staging/committing job owns + -- the slot, a single created job may be waiting behind it. Awaiting-return is + -- recovery state rather than an active stage owner; legacy created waiters are + -- blitzed in that case. + local keep_job = keep_id and jobs.jobs[keep_id] or nil + local keep_owns_active_slot = type(keep_job) == 'table' + and (keep_job.state == 'staging' or keep_job.state == 'committing') + and (keep_job.active_token ~= nil or type(keep_job.active_intent) == 'table') + if keep_reason == 'recoverable' and keep_owns_active_slot then + local waiter_id = newest_created_waiter(jobs, keep_id) + if waiter_id ~= nil then keep[waiter_id] = true end + end + + local ids = sorted_job_ids(jobs) + local deleted = 0 + for _, id in ipairs(ids) do + if not keep[id] then + local job = jobs.jobs[id] + local ok, derr = fibers.perform(store_delete_op(self._store, id)) + if ok ~= true then + adoption.diagnostics[#adoption.diagnostics + 1] = { + job_id = id, + message = 'legacy job delete failed: ' .. tostring(derr or 'delete_failed'), + } + return nil, derr or 'legacy_job_delete_failed' + end + repo_mod.remove(jobs, id) + mark_pruned(adoption, id, job, 'single_job_startup_scrub') + deleted = deleted + 1 + end + end + for id in pairs(keep) do + local job = jobs.jobs[id] + local force_compact = deleted > 0 + or (id == keep_id and keep_reason == 'last_job' and job and job.state == 'awaiting_commit') + local ok, err = save_compact_job(self, jobs, id, 'single_job_compact_save_failed', { force = force_compact }) + if ok ~= true then return nil, err end + end + local kept_aux = {} + for id in pairs(keep) do if id ~= keep_id then kept_aux[#kept_aux + 1] = id end end + table.sort(kept_aux) + adoption.single_job = { + mode = 'single_job', + kept_job_id = keep_id, + kept_aux_ids = kept_aux, + kept_reason = keep_reason, + deleted_count = deleted, + legacy_job_ids = copy(ids), + } + return true, nil +end + +local function plan_delete_others(self, plan) + if not (plan and plan.kind == 'save_job' and plan.job_id ~= nil) then return plan end + local ids = {} + for id, job in pairs((self._jobs and self._jobs.jobs) or {}) do + -- Do not delete another non-terminal job merely because this transition is + -- being persisted. Original update semantics allow a created job to wait while + -- an active job owns the slot. Terminal records are replaceable compact + -- telemetry and may be removed when a new operational record is saved. + if id ~= plan.job_id and (repo_mod.is_terminal(job and job.state) + or (plan.transition == 'create_job' and job and job.state == 'created')) then + ids[#ids + 1] = id + end + end + table.sort(ids) + plan.delete_others = ids + return plan +end + +local function next_sequence_value(jobs) + return jobs and (jobs.next_seq or 1) or 1 +end + +local function validate_generation(cmd, current) + -- Generation-scoped admissions must be checked against the service-owned + -- current generation at the point where job_runtime is about to admit them. + -- Already-admitted durable work is not revalidated later, and active-result + -- application is validated by durable active token rather than by the current + -- generation, because accepted active work may outlive generation replacement. + if cmd.generation ~= nil and current ~= nil and cmd.generation ~= current then + return nil, 'stale_generation' + end + return true, nil +end + +local GENERATION_VALIDATED_TRANSITIONS = { + create_job = true, + start_job = true, + mark_starting = true, + patch_job = true, + mark_job = true, + discard_job = true, + delete_job = true, +} + +local function active_token_for(cmd, job, phase, seq) + if cmd.token ~= nil then return cmd.token end + return table.concat({ + tostring(cmd.generation or job.generation or 0), + tostring(job.job_id), + tostring(phase or 'stage'), + tostring(seq or 0), + }, ':') +end + +local COMMIT_POLICIES = { + idempotent_by_token = true, + no_duplicate = true, +} + +local function normalise_commit_policy(policy) + if type(policy) == 'table' then policy = policy.policy end + if COMMIT_POLICIES[policy] then return policy end + return nil, 'commit_policy_required' +end + +local function commit_token_for(cmd, job) + return cmd.commit_token + or (type(job.commit_attempt) == 'table' and job.commit_attempt.token) + or (type(job.active_intent) == 'table' and job.active_intent.commit_token) + or job.active_token + or cmd.token +end + +local function durable_active_owner(jobs) + for id, job in pairs((jobs and jobs.jobs) or {}) do + if not repo_mod.TERMINAL[job.state] and (job.active_token ~= nil or type(job.active_intent) == 'table') then + return id, job + end + end + return nil, nil +end + +local function compute_create(self, cmd) + -- Compatibility boundary: create-job does not own the operational update slot. + -- The slot is owned by start/commit work. A second job may therefore be + -- created while a first job is still active; start-job will continue to reject + -- with slot_busy until the active owner has completed. The storage policy is + -- enforced by pruning terminal records, not by rejecting create. + local payload = copy(cmd.payload or cmd.job or {}) + payload.job_id = payload.job_id or cmd.job_id or new_id('job') + + local job, err = repo_mod.new_job(payload, { + job_id = payload.job_id, + generation = cmd.generation, + seq = next_sequence_value(self._jobs), + reason = cmd.reason or 'create_job', + }) + if not job then return nil, err end + + if self._jobs.jobs[job.job_id] ~= nil then + return nil, 'job_exists' + end + + return { + kind = 'save_job', + transition = cmd.kind, + job = job, + job_id = job.job_id, + next_seq = next_sequence_value(self._jobs) + 1, + public_result = { + tag = 'job_created', + method = 'create_job', + job_id = job.job_id, + job = public_job(job), + auto_start = payload.auto_start == true, + }, + }, nil +end + +local function compute_start(self, cmd) + local current = cmd.job_id and self._jobs.jobs[cmd.job_id] or nil + if not current then return nil, 'not_found' end + + local active_owner = durable_active_owner(self._jobs) + if active_owner ~= nil then + return nil, 'slot_busy' + end + + local phase = cmd.phase or active_policy.phase_for(current, cmd.payload) + local ok_phase, phase_err = active_policy.can_start_phase(current, phase) + if ok_phase ~= true then return nil, phase_err end + + local seq = next_sequence_value(self._jobs) + local job = active_policy.mark_starting(copy(current), phase, seq) + local token = active_token_for(cmd, job, phase, seq) + + job.active_token = token + job.active_intent = { + token = token, + phase = phase, + generation = cmd.generation, + request_id = cmd.request_id, + state = 'pending', + reason = cmd.reason or (phase == 'commit' and 'start_commit' or 'start_stage'), + } + -- Transitional public field for existing projections/tests. Ownership is the + -- durable active_intent above. + job.active = copy(job.active_intent) + + return { + kind = 'save_job', + transition = cmd.kind, + job = job, + job_id = job.job_id, + phase = phase, + token = token, + next_seq = seq + 1, + public_result = { + tag = 'job_started', + method = phase == 'commit' and 'commit_job' or 'start_job', + job_id = job.job_id, + phase = phase, + token = token, + job = public_job(job), + }, + }, nil +end + +local function compute_begin_commit_attempt(self, cmd) + local current = cmd.job_id and self._jobs.jobs[cmd.job_id] or nil + if not current then return nil, 'not_found' end + if current.state ~= 'committing' then return nil, 'job_not_committing' end + + local expected_token = current.active_token or (current.active and current.active.token) + if expected_token == nil then return nil, 'not_active' end + if cmd.token ~= nil and cmd.token ~= expected_token then return nil, 'stale_active_token' end + + local policy, perr = normalise_commit_policy(cmd.commit_policy or cmd.policy) + if not policy then return nil, perr end + + local token = commit_token_for(cmd, current) + if token == nil or token == '' then return nil, 'commit_token_required' end + + local existing = type(current.commit_attempt) == 'table' and current.commit_attempt or nil + if existing ~= nil then + if existing.token ~= nil and existing.token ~= token then + return nil, 'commit_token_mismatch' + end + local existing_policy = existing.policy or existing.commit_policy + if existing_policy ~= nil and existing_policy ~= policy then + return nil, 'commit_policy_mismatch' + end + end + + local seq = next_sequence_value(self._jobs) + local job = copy(current) + job.commit_attempt = copy(existing or {}) + job.commit_attempt.token = token + job.commit_attempt.policy = policy + job.commit_attempt.active_token = expected_token + job.commit_attempt.generation = cmd.generation or job.generation + job.commit_attempt.acceptance = job.commit_attempt.acceptance or 'unknown' + job.commit_attempt.started_seq = job.commit_attempt.started_seq or seq + job.commit_attempt.reason = cmd.reason or 'begin_commit_attempt' + if cmd.pre_commit ~= nil then + job.commit_attempt.pre_commit = copy(cmd.pre_commit) + end + + local pending = copy(cmd.pending or cmd.commit_result or {}) + pending.tag = pending.tag or 'commit_pending' + pending.commit_token = pending.commit_token or token + pending.commit_policy = pending.commit_policy or policy + if cmd.pre_commit ~= nil and pending.pre_commit == nil then + pending.pre_commit = copy(cmd.pre_commit) + end + + -- The durable boundary for MCU commits is before the commit RPC is sent. + -- After this point restart adoption must reconcile against the component's + -- reported software state and must not try to resume staging from a transient + -- upload artifact. + repo_mod.patch(job, { + state = 'awaiting_return', + next_step = 'reconcile', + commit_result = pending, + commit_attempt = job.commit_attempt, + }, { seq = seq, reason = cmd.reason or 'begin_commit_attempt' }) + job.error = nil + job.active_token = nil + job.active = nil + job.active_intent = nil + + return { + kind = 'save_job', + transition = cmd.kind, + job = job, + job_id = job.job_id, + phase = 'commit', + token = expected_token, + commit_token = token, + commit_policy = policy, + next_seq = seq + 1, + public_result = { + tag = 'commit_attempt_begun', + job_id = job.job_id, + phase = 'commit', + token = expected_token, + commit_token = token, + commit_policy = policy, + job = public_job(job), + }, + }, nil +end + + +local function compute_commit_accepted(self, cmd) + local current = cmd.job_id and self._jobs.jobs[cmd.job_id] or nil + if not current then return nil, 'not_found' end + + local policy, perr = normalise_commit_policy(cmd.commit_policy or cmd.policy) + if not policy then return nil, perr end + + local token = commit_token_for(cmd, current) + if token == nil or token == '' then return nil, 'commit_token_required' end + + local attempt = type(current.commit_attempt) == 'table' and current.commit_attempt or nil + local expected_active_token = current.active_token or (current.active_intent and current.active_intent.token) or (attempt and attempt.active_token) + if cmd.token ~= nil and expected_active_token ~= nil and cmd.token ~= expected_active_token then + return nil, 'stale_active_token' + end + + if current.state == 'awaiting_return' then + if attempt ~= nil then + if attempt.token ~= nil and attempt.token ~= token then return nil, 'commit_token_mismatch' end + local existing_policy = attempt.policy or attempt.commit_policy + if existing_policy ~= nil and existing_policy ~= policy then return nil, 'commit_policy_mismatch' end + end + local seq = next_sequence_value(self._jobs) + local accepted = copy(cmd.accepted or cmd.result or {}) + local job = copy(current) + job.commit_attempt = copy(attempt or {}) + job.commit_attempt.token = token + job.commit_attempt.policy = policy + job.commit_attempt.active_token = job.commit_attempt.active_token or expected_active_token or cmd.token + job.commit_attempt.acceptance = 'accepted' + job.commit_attempt.accepted = true + job.commit_attempt.accepted_result = accepted + job.commit_attempt.accepted_seq = job.commit_attempt.accepted_seq or seq + repo_mod.patch(job, { + commit_attempt = job.commit_attempt, + commit_result = accepted, + }, { seq = seq, reason = cmd.reason or 'commit_accepted_idempotent' }) + job.active_token = nil + job.active = nil + job.active_intent = nil + return { + kind = 'save_job', + transition = cmd.kind, + job = job, + job_id = job.job_id, + phase = 'commit', + token = cmd.token or expected_active_token, + commit_token = token, + commit_policy = policy, + next_seq = seq + 1, + public_result = { + tag = 'commit_accepted', + job_id = job.job_id, + phase = 'commit', + token = cmd.token or expected_active_token, + commit_token = token, + commit_policy = policy, + job = public_job(job), + }, + }, nil + end + + if current.state ~= 'committing' then return nil, 'job_not_committing' end + if expected_active_token == nil then return nil, 'not_active' end + + if attempt ~= nil then + if attempt.token ~= nil and attempt.token ~= token then return nil, 'commit_token_mismatch' end + local existing_policy = attempt.policy or attempt.commit_policy + if existing_policy ~= nil and existing_policy ~= policy then return nil, 'commit_policy_mismatch' end + end + + local seq = next_sequence_value(self._jobs) + local job = copy(current) + job.commit_attempt = copy(attempt or {}) + job.commit_attempt.token = token + job.commit_attempt.policy = policy + job.commit_attempt.active_token = expected_active_token + job.commit_attempt.generation = cmd.generation or job.generation + job.commit_attempt.acceptance = 'accepted' + job.commit_attempt.accepted = true + job.commit_attempt.accepted_result = copy(cmd.accepted or cmd.result or {}) + job.commit_attempt.accepted_seq = seq + job.commit_attempt.reason = cmd.reason or 'commit_accepted' + + repo_mod.mark_awaiting_return(job, cmd.accepted or cmd.result or {}, { seq = seq, reason = cmd.reason or 'commit_accepted' }) + job.active_token = nil + job.active = nil + job.active_intent = nil + + repo_mod.patch(job, { + commit_attempt = job.commit_attempt, + active_token = nil, + active = nil, + active_intent = nil, + }, { seq = seq, reason = cmd.reason or 'commit_accepted' }) + + return { + kind = 'save_job', + transition = cmd.kind, + job = job, + job_id = job.job_id, + phase = 'commit', + token = expected_active_token, + commit_token = token, + commit_policy = policy, + next_seq = seq + 1, + public_result = { + tag = 'commit_accepted', + job_id = job.job_id, + phase = 'commit', + token = expected_active_token, + commit_token = token, + commit_policy = policy, + job = public_job(job), + }, + }, nil +end + + +local function compute_commit_failed(self, cmd) + local current = cmd.job_id and self._jobs.jobs[cmd.job_id] or nil + if not current then return nil, 'not_found' end + + local token = commit_token_for(cmd, current) + if token == nil or token == '' then return nil, 'commit_token_required' end + local attempt = type(current.commit_attempt) == 'table' and current.commit_attempt or nil + if attempt ~= nil and attempt.token ~= nil and attempt.token ~= token then + return nil, 'commit_token_mismatch' + end + + local policy, perr = normalise_commit_policy(cmd.commit_policy or cmd.policy or (attempt and (attempt.policy or attempt.commit_policy))) + if not policy then return nil, perr end + if attempt ~= nil then + local existing_policy = attempt.policy or attempt.commit_policy + if existing_policy ~= nil and existing_policy ~= policy then return nil, 'commit_policy_mismatch' end + end + + if current.state ~= 'awaiting_return' and current.state ~= 'committing' then + return nil, 'job_not_committing' + end + + local seq = next_sequence_value(self._jobs) + local reason = cmd.error or cmd.reason or 'commit_failed' + local job = copy(current) + job.commit_attempt = copy(attempt or {}) + job.commit_attempt.token = token + job.commit_attempt.policy = policy + job.commit_attempt.acceptance = 'failed' + job.commit_attempt.accepted = false + job.commit_attempt.error = reason + job.commit_attempt.failed_seq = seq + repo_mod.mark_terminal(job, 'failed', reason, { + commit_token = token, + commit_policy = policy, + error = reason, + result = copy(cmd.result), + }, { seq = seq, reason = cmd.reason or 'commit_failed' }) + job.active_token = nil + job.active = nil + job.active_intent = nil + + return { + kind = 'save_job', + transition = cmd.kind, + job = job, + job_id = job.job_id, + phase = 'commit', + token = cmd.token, + commit_token = token, + commit_policy = policy, + next_seq = seq + 1, + public_result = { + tag = 'commit_failed', + job_id = job.job_id, + phase = 'commit', + token = cmd.token, + commit_token = token, + commit_policy = policy, + error = reason, + job = public_job(job), + }, + }, nil +end + +local function compute_apply_active(self, cmd) + local ev = assert(cmd.event or cmd.active_event or cmd.completion, 'active completion event required') + local current = ev.job_id and self._jobs.jobs[ev.job_id] or nil + if not current then return nil, 'not_found' end + + local expected_token = current.active_token or (current.active and current.active.token) + if expected_token == nil then + -- Reconcile is idempotent observation of an already-durable + -- awaiting_return job. Commit completion may also arrive after the commit + -- worker has already persisted commit_accepted and cleared active intent; in + -- that case durable apply is a confirmation, not the first accepted boundary. + if ev.phase == 'commit' and current.state == 'awaiting_return' and ev.status == 'ok' then + local job = copy(current) + if type(job.commit_attempt) == 'table' then + job.commit_attempt.accepted_result = copy((ev.result or {}).commit or (ev.result or {})) + job.commit_attempt.acceptance = 'accepted' + job.commit_attempt.accepted = true + end + return { + kind = 'save_job', + transition = cmd.kind, + job = job, + job_id = job.job_id, + phase = 'commit', + token = ev.token, + next_seq = next_sequence_value(self._jobs), + public_result = { + tag = 'active_result_applied', + job_id = job.job_id, + phase = ev.phase, + token = ev.token, + job = public_job(job), + active = copy(ev), + }, + }, nil + end + if not (ev.phase == 'reconcile' and current.state == 'awaiting_return') then + return nil, 'not_active' + end + elseif ev.token ~= expected_token then + return nil, 'stale_active_token' + end + + local job = copy(current) + local ok_apply, aerr = active_policy.apply_completion(job, ev, next_sequence_value(self._jobs)) + if ok_apply ~= true then return nil, aerr or 'active_completion_apply_failed' end + if ev.phase == 'commit' and type(job.commit_attempt) == 'table' then + job.commit_attempt.acceptance = ev.status == 'ok' and 'accepted' or (ev.status or 'failed') + job.commit_attempt.accepted_result = copy((ev.result or {}).commit or (ev.result or {})) + end + job.active_token = nil + job.active = nil + job.active_intent = nil + + return { + kind = 'save_job', + transition = cmd.kind, + job = job, + job_id = job.job_id, + next_seq = next_sequence_value(self._jobs) + 1, + public_result = { + tag = 'active_result_applied', + job_id = job.job_id, + phase = ev.phase, + token = ev.token, + job = public_job(job), + active = copy(ev), + }, + }, nil +end + +local function compute_patch(self, cmd) + local current = cmd.job_id and self._jobs.jobs[cmd.job_id] or nil + if not current then return nil, 'not_found' end + local job = copy(current) + local public, err = repo_mod.patch(job, cmd.patch or {}, { + seq = next_sequence_value(self._jobs), + reason = cmd.reason or cmd.kind or 'patch_job', + }) + if not public then return nil, err end + return { + kind = 'save_job', + transition = cmd.kind, + job = job, + job_id = job.job_id, + next_seq = next_sequence_value(self._jobs) + 1, + public_result = { + tag = 'job_patched', + method = cmd.kind, + job_id = job.job_id, + job = public_job(job), + }, + }, nil +end + +local function compute_discard(self, cmd) + local current = cmd.job_id and self._jobs.jobs[cmd.job_id] or nil + if not current then return nil, 'not_found' end + return { + kind = 'delete_job', + transition = cmd.kind, + job_id = cmd.job_id, + public_result = { + tag = 'job_discarded', + method = cmd.kind, + job_id = cmd.job_id, + job = public_job(current), + }, + }, nil +end + +local function compute_transition(self, cmd) + if type(cmd) ~= 'table' then return nil, 'transition command required' end + if type(cmd.kind) ~= 'string' then return nil, 'transition kind required' end + + if GENERATION_VALIDATED_TRANSITIONS[cmd.kind] then + local ok_gen, gen_err = validate_generation(cmd, self._current_generation) + if ok_gen ~= true then return nil, gen_err end + end + + if cmd.kind == 'create_job' then + return compute_create(self, cmd) + elseif cmd.kind == 'start_job' or cmd.kind == 'mark_starting' then + return compute_start(self, cmd) + elseif cmd.kind == 'begin_commit_attempt' then + return compute_begin_commit_attempt(self, cmd) + elseif cmd.kind == 'commit_accepted' then + return compute_commit_accepted(self, cmd) + elseif cmd.kind == 'commit_failed' then + return compute_commit_failed(self, cmd) + elseif cmd.kind == 'apply_active_result' then + return compute_apply_active(self, cmd) + elseif cmd.kind == 'patch_job' or cmd.kind == 'mark_job' then + return compute_patch(self, cmd) + elseif cmd.kind == 'discard_job' or cmd.kind == 'delete_job' then + return compute_discard(self, cmd) + end + + return nil, 'unsupported_job_transition: ' .. tostring(cmd.kind) +end + +local function apply_plan(self, plan) + if plan.kind == 'save_job' then + local saved, err = repo_mod.upsert(self._jobs, plan.job) + if not saved then return nil, err end + for _, id in ipairs(plan.delete_others or {}) do + if self._jobs.jobs and self._jobs.jobs[id] ~= nil then + repo_mod.remove(self._jobs, id) + end + end + if plan.next_seq and plan.next_seq > (self._jobs.next_seq or 1) then + self._jobs.next_seq = plan.next_seq + end + elseif plan.kind == 'delete_job' then + local ok, err = repo_mod.remove(self._jobs, plan.job_id) + if ok ~= true then return nil, err end + else + return nil, 'unsupported_plan_kind' + end + + self._model:set_snapshot(runtime_snapshot(self, self._ready, self._adoption)) + return true, nil +end + +local function start_transition_worker(self, req, plan) + local identity = { + kind = 'job_transition_done', + service_id = self._service_id, + transition_id = req.id, + transition = plan.transition, + job_id = plan.job_id, + generation = req.cmd and req.cmd.generation, + phase = req.cmd and req.cmd.phase, + token = req.cmd and req.cmd.token, + } + + local handle, err = scoped_work.start { + lifetime_scope = self._scope, + reaper_scope = self._scope, + report_scope = self._scope, + identity = identity, + + run = function () + if plan.kind == 'save_job' then + local ok, serr = fibers.perform(store_save_op(self._store, public_job(plan.job))) + if ok ~= true then error(serr or 'job_save_failed', 0) end + for _, id in ipairs(plan.delete_others or {}) do + local dok, derr = fibers.perform(store_delete_op(self._store, id)) + if dok ~= true then error(derr or 'job_delete_failed', 0) end + end + elseif plan.kind == 'delete_job' then + local ok, derr = fibers.perform(store_delete_op(self._store, plan.job_id)) + if ok ~= true then error(derr or 'job_delete_failed', 0) end + else + error('unsupported_plan_kind', 0) + end + + return { + tag = 'job_transition_persisted', + transition_id = req.id, + transition = plan.transition, + job_id = plan.job_id, + } + end, + + report = function (ev) + return queue.try_admit_required( + self._done_tx, + ev, + 'update_job_transition_completion_report_failed' + ) + end, + } + + if not handle then return nil, err end + + return handle, nil +end + +local record_transition_outcome + +local function start_next(self) + if self._inflight ~= nil then return true end + + while self._inflight == nil and #self._pending > 0 do + local req = table.remove(self._pending, 1) + + local plan, perr = compute_transition(self, req.cmd) + if not plan then + local rec = transition_set(self, req, 'rejected', { + sequence = req.sequence, + error = perr or 'job_transition_rejected', + }) + local outcome = transition_outcome_from_record(rec, { + transition_id = req.id, + status = 'rejected', + transition = req.cmd and req.cmd.kind, + job_id = req.cmd and req.cmd.job_id, + reason = perr or 'job_transition_rejected', + }) + record_transition_outcome(self, outcome) + refresh_model(self) + resolve_cell(req.cell, outcome, nil) + else + plan = plan_delete_others(self, plan) + local rec = transition_set(self, req, 'admitted', { + sequence = req.sequence, + plan_kind = plan.kind, + }) + refresh_model(self) + + rec = transition_set(self, req, 'persisting', { + sequence = req.sequence, + plan_kind = plan.kind, + }) + refresh_model(self) + + local handle, herr = start_transition_worker(self, req, plan) + if not handle then + rec = transition_set(self, req, 'failed', { + sequence = req.sequence, + plan_kind = plan.kind, + error = herr or 'job_transition_start_failed', + }) + local outcome = transition_outcome_from_record(rec, { + transition_id = req.id, + status = 'failed', + transition = req.cmd and req.cmd.kind, + job_id = plan.job_id, + reason = herr or 'job_transition_start_failed', + }) + record_transition_outcome(self, outcome) + refresh_model(self) + resolve_cell(req.cell, outcome, nil) + else + req.admitted = true + self._inflight = { + request = req, + plan = plan, + handle = handle, + } + end + end + end + + return true +end + +local function handle_request(self, req) + if self._closed then + resolve_cell(req.cell, nil, self._closed_reason or 'job_runtime_closed') + return + end + + req.sequence = req.sequence or (#(self._transition_order or {}) + 1) + transition_set(self, req, 'proposed', { sequence = req.sequence }) + refresh_model(self) + self._pending[#self._pending + 1] = req + start_next(self) +end + +local function transition_outcome(req, plan, status, reason) + local out = {} + if status == 'persisted' and plan.public_result then + out = copy(plan.public_result) + end + out.transition_id = req.id + out.status = status + out.transition = plan.transition + out.job_id = plan.job_id + out.phase = plan.phase or (req.cmd and req.cmd.phase) + out.token = plan.token or (req.cmd and req.cmd.token) + out.commit_token = plan.commit_token or (req.cmd and req.cmd.commit_token) or out.commit_token + out.commit_policy = plan.commit_policy or (req.cmd and (req.cmd.commit_policy or req.cmd.policy)) or out.commit_policy + if status ~= 'persisted' then out.reason = reason end + return out +end + +local function trim_ordered_map(map, order, max_count) + max_count = max_count or MAX_TRANSITION_RECORDS + while #order > max_count do + local id = table.remove(order, 1) + if map then map[id] = nil end + end +end + +function record_transition_outcome(self, outcome) + self._transition_outcomes = self._transition_outcomes or {} + self._transition_outcome_order = self._transition_outcome_order or {} + self._transition_outcomes[outcome.transition_id] = copy(outcome) + self._transition_outcome_order[#self._transition_outcome_order + 1] = outcome.transition_id + trim_ordered_map(self._transition_outcomes, self._transition_outcome_order, MAX_TRANSITION_RECORDS) + trim_ordered_map(self._transitions, self._transition_order or {}, MAX_TRANSITION_RECORDS) +end + +local function handle_transition_done(self, ev) + local inflight = self._inflight + if not inflight or ev.transition_id ~= inflight.request.id then + return + end + + self._inflight = nil + + local outcome + local rec + if ev.status == 'ok' then + local ok, err = apply_plan(self, inflight.plan) + if ok == true then + rec = transition_set(self, inflight.request, 'persisted', { + sequence = inflight.request.sequence, + plan_kind = inflight.plan.kind, + }) + outcome = transition_outcome_from_record(rec, transition_outcome(inflight.request, inflight.plan, 'persisted')) + else + rec = transition_set(self, inflight.request, 'failed', { + sequence = inflight.request.sequence, + plan_kind = inflight.plan.kind, + error = err or 'job_transition_apply_failed', + }) + outcome = transition_outcome_from_record(rec, transition_outcome(inflight.request, inflight.plan, 'failed', err or 'job_transition_apply_failed')) + end + else + local reason = ev.primary or ev.status or 'job_transition_failed' + rec = transition_set(self, inflight.request, 'failed', { + sequence = inflight.request.sequence, + plan_kind = inflight.plan.kind, + error = reason, + }) + outcome = transition_outcome_from_record(rec, transition_outcome(inflight.request, inflight.plan, 'failed', reason)) + end + + record_transition_outcome(self, outcome) + refresh_model(self) + resolve_cell(inflight.request.cell, outcome, nil) + + start_next(self) +end + +local function map_request(req) + if req == nil then return { kind = 'job_runtime_requests_closed' } end + return { kind = 'transition_requested', request = req } +end + +local function map_done(ev) + if ev == nil then return { kind = 'job_runtime_done_closed' } end + return ev +end + +function Runtime:_run(scope) + self._scope = scope + scope:finally(function (_, status, primary) + local reason = primary or status or 'job_runtime_closed' + self._request_tx:close(reason) + self._done_tx:close(reason) + self._closed = true + self._closed_reason = reason + if self._inflight and self._inflight.request then + resolve_cell(self._inflight.request.cell, nil, reason) + end + for _, req in ipairs(self._pending) do + resolve_cell(req.cell, nil, reason) + end + end) + + local jobs, load_err = load_jobs(self._params or {}) + if not jobs then + self._ready_err = load_err or 'job_load_failed' + self._ready_cond:signal() + error(self._ready_err, 0) + end + self._jobs = jobs + local adoption = adoption_empty() + local scrubbed, scrub_err = scrub_legacy_jobs(self, jobs, adoption) + if scrubbed ~= true then + self._ready_err = scrub_err or 'legacy_job_scrub_failed' + self._ready_cond:signal() + error(self._ready_err, 0) + end + local adopted, adopt_err = adopt_restart_jobs(self, jobs) + if not adopted then + self._ready_err = adopt_err or 'restart_adoption_failed' + self._ready_cond:signal() + error(self._ready_err, 0) + end + for k, v in pairs(adopted or {}) do + if type(v) == 'table' and type(adoption[k]) == 'table' then + for _, item in ipairs(v) do adoption[k][#adoption[k] + 1] = item end + else + adoption[k] = v + end + end + self._adoption = adoption + self._ready = true + self._model:set_snapshot(runtime_snapshot(self, true, self._adoption)) + self._ready_cond:signal() + + while true do + local which, ev = fibers.perform(fibers.named_choice { + request = self._request_rx:recv_op():wrap(map_request), + done = self._done_rx:recv_op():wrap(map_done), + }) + + if which == 'request' then + if ev.kind == 'job_runtime_requests_closed' then + return { tag = 'job_runtime_stopped', reason = 'request_queue_closed' } + end + handle_request(self, ev.request) + elseif which == 'done' then + if ev.kind == 'job_runtime_done_closed' then + return { tag = 'job_runtime_stopped', reason = 'done_queue_closed' } + end + handle_transition_done(self, ev) + end + end +end + +function Runtime:ready() + return self._ready == true +end + +function Runtime:ready_op() + return op.guard(function () + if self._ready then return op.always(true, nil) end + if self._ready_err then return op.always(nil, self._ready_err) end + return self._ready_cond:wait_op():wrap(function () + if self._ready then return true, nil end + return nil, self._ready_err or self._closed_reason or 'job_runtime_closed' + end) + end) +end + +function Runtime:version() + return self._model:version() +end + +function Runtime:changed_op(seen) + return self._model:changed_op(seen) +end + +function Runtime:snapshot() + return repo_mod.snapshot(self._jobs or { jobs = {}, order = {}, next_seq = 1 }) +end + +function Runtime:model_snapshot() + return self._model:snapshot() +end + +function Runtime:adoption() + return copy(self._adoption or {}) +end + +function Runtime:set_current_generation(generation) + self._current_generation = generation + return true, nil +end + +function Runtime:current_generation() + return self._current_generation +end + +function Runtime:transition_snapshot() + return transition_snapshot(self) +end + +function Runtime:transition_outcome(transition_id) + local out = self._transition_outcomes and self._transition_outcomes[transition_id] or nil + return out and copy(out) or nil +end + +function Runtime:list() + return repo_mod.list(self._jobs or { jobs = {}, order = {} }) +end + +function Runtime:get(job_id) + return repo_mod.get(self._jobs or { jobs = {} }, job_id) +end + +function Runtime:admit_transition(cmd) + local cell = make_cell() + local req = { + id = tostring(self._next_transition_id), + cmd = copy(cmd), + cell = cell, + } + self._next_transition_id = self._next_transition_id + 1 + + if self._closed then + return nil, self._closed_reason or 'job_runtime_closed' + end + + if self._ready ~= true then + return nil, self._ready_err or 'job_runtime_not_ready' + end + + local ok, err = queue.try_admit_now(self._request_tx, req) + if ok ~= true then + return nil, err or 'job_runtime_busy' + end + + return new_transition_handle(req), nil +end + +function Runtime:terminate(reason) + return self:cancel(reason or 'job_runtime_terminated') +end + +function Runtime:cancel(reason) + if self._handle and self._handle.cancel then + return self._handle:cancel(reason or 'job_runtime_cancelled') + end + self._request_tx:close(reason or 'job_runtime_cancelled') + return true +end + +local function empty_jobs() + return { jobs = {}, order = {}, next_seq = 1, dirty = {} } +end + +local function new_runtime(scope, params) + local request_tx, request_rx = mailbox.new(params.queue_len or DEFAULT_QUEUE, { full = 'reject_newest' }) + local done_tx, done_rx = mailbox.new(params.done_queue_len or DEFAULT_QUEUE, { full = 'reject_newest' }) + local initial_jobs = empty_jobs() + local runtime = setmetatable({ + _scope = scope, + _service_id = params.service_id or 'update', + _store = params.store, + _params = params, + _jobs = initial_jobs, + _adoption = {}, + _ready = false, + _ready_err = nil, + _ready_cond = cond.new(), + _request_tx = request_tx, + _request_rx = request_rx, + _done_tx = done_tx, + _done_rx = done_rx, + _pending = {}, + _inflight = nil, + _next_transition_id = 1, + _closed = false, + _closed_reason = nil, + _current_generation = params.current_generation, + _transition_outcomes = {}, + _transition_order = {}, + _model = model_mod.new(runtime_snapshot(initial_jobs, false, {}), { label = 'update.job_runtime' }), + }, Runtime) + + scope:finally(function (_, status, primary) + local reason = primary or status or 'job_runtime_parent_closed' + runtime._model:terminate(reason) + request_tx:close(reason) + done_tx:close(reason) + if runtime._ready ~= true and runtime._ready_err == nil then + runtime._ready_err = reason + runtime._ready_cond:signal() + end + end) + + return runtime +end + +function M.start(scope, params) + params = params or {} + local runtime = new_runtime(scope, params) + + local handle, herr = scoped_work.start { + lifetime_scope = scope, + reaper_scope = scope, + report_scope = scope, + identity = { + kind = 'component_done', + service_id = params.service_id or 'update', + component = 'job_runtime', + }, + run = function (component_scope) + return runtime:_run(component_scope) + end, + report = function (ev) + if params.done_tx == nil then return true, nil end + return queue.try_admit_required( + params.done_tx, + ev, + 'update_job_runtime_component_completion_report_failed' + ) + end, + } + + if not handle then + runtime._model:terminate(herr or 'job_runtime_start_failed') + runtime._request_tx:close(herr or 'job_runtime_start_failed') + runtime._done_tx:close(herr or 'job_runtime_start_failed') + runtime._ready_err = herr or 'job_runtime_start_failed' + runtime._ready_cond:signal() + return nil, herr + end + + runtime._handle = handle + return runtime, nil, nil +end + + +M.Runtime = Runtime + +return M diff --git a/src/services/update/job_store_control_store.lua b/src/services/update/job_store_control_store.lua new file mode 100644 index 00000000..5b5efb89 --- /dev/null +++ b/src/services/update/job_store_control_store.lua @@ -0,0 +1,186 @@ +-- services/update/job_store_control_store.lua +-- +-- Control-store-backed durable Update job store. +-- +-- This is the strict production adapter for the curated HAL control-store +-- capability surface: +-- cap/control-store//rpc/{get,put,delete,list} +-- +-- The store is intentionally flat because the HAL control-store provider +-- accepts flat safe keys. Job ids are encoded into safe key suffixes; the job id +-- inside the JSON payload remains authoritative when loading. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local cjson = require 'cjson.safe' + +local cap_args = require 'services.hal.types.capability_args' +local model = require 'services.update.model' + +local M = {} + +local Store = {} +Store.__index = Store + +local function copy(v) + return model.deep_copy(v) +end + +local function rpc_topic(id, method) + return { 'cap', 'control-store', id or 'update', 'rpc', method } +end + +local VOID_SUCCESS = { + put = true, + delete = true, +} + +local function unwrap_reply_for(method) + return function (reply, err) + if reply == nil then + return nil, err + end + if type(reply) ~= 'table' or type(reply.ok) ~= 'boolean' then + return nil, 'invalid_control_store_reply' + end + if reply.ok then + if reply.reason == nil and VOID_SUCCESS[method] then + return true, nil + end + return reply.reason, nil + end + return nil, tostring(reply.reason or err or 'control_store_call_failed') + end +end + +local function call_op(self, method, payload, opts) + if type(self._conn) ~= 'table' or type(self._conn.call_op) ~= 'function' then + return op.always(nil, 'control_store_connection_required') + end + return self._conn:call_op( + rpc_topic(self._id, method), + payload or {}, + opts or self._call_opts + ):wrap(function (reply, err) + return unwrap_reply_for(method)(reply, err) + end) +end + +local function safe_key_suffix(id) + id = tostring(id or '') + if id == '' then return nil, 'invalid_job_id' end + return (id:gsub('[^%w%._%-]', function (ch) + return ('_%02X'):format(ch:byte()) + end)), nil +end + +local function key_for(self, job_id) + local suffix, err = safe_key_suffix(job_id) + if not suffix then return nil, err end + return self._prefix .. suffix, nil +end + +local function sorted_ids(jobs) + local ids = {} + for id in pairs(jobs or {}) do ids[#ids + 1] = id end + table.sort(ids) + return ids +end + +local function decode_job(key, body) + if type(body) ~= 'string' then + return nil, 'control_store_job_body_not_string:' .. tostring(key) + end + local job, err = cjson.decode(body) + if type(job) ~= 'table' then + return nil, 'control_store_job_json_invalid:' .. tostring(err or key) + end + if type(job.job_id) ~= 'string' or job.job_id == '' then + return nil, 'control_store_job_missing_id:' .. tostring(key) + end + return job, nil +end + +function Store:load_all_op() + return fibers.run_scope_op(function () + local list_opts, lerr = cap_args.new.ControlStoreListOpts(self._prefix) + if not list_opts then return nil, lerr or 'invalid_control_store_list_opts' end + + local keys, err = fibers.perform(call_op(self, 'list', list_opts)) + if keys == nil then return nil, err or 'control_store_list_failed' end + if type(keys) ~= 'table' then return nil, 'control_store_list_returned_non_table' end + + local jobs = {} + for _, key in ipairs(keys) do + if type(key) == 'string' and key:sub(1, #self._prefix) == self._prefix then + local get_opts, gerr = cap_args.new.ControlStoreGetOpts(key) + if not get_opts then return nil, gerr or 'invalid_control_store_get_opts' end + local body, berr = fibers.perform(call_op(self, 'get', get_opts)) + if body == nil then return nil, berr or ('control_store_get_failed:' .. key) end + local job, derr = decode_job(key, body) + if not job then return nil, derr end + jobs[job.job_id] = copy(job) + end + end + + return { jobs = jobs, order = sorted_ids(jobs) }, nil + end):wrap(function (st, rep, snapshot, err) + -- fibers.run_scope_op returns (status, report, result) on success and + -- (status, report, primary) on failure. Preserve the primary failure so + -- callers can classify capability-routing errors such as `no_route`. + if st ~= 'ok' then + return nil, tostring(snapshot or err or rep) + end + return snapshot, err + end) +end + +function Store:save_job_op(job) + return op.guard(function () + if type(job) ~= 'table' or type(job.job_id) ~= 'string' or job.job_id == '' then + return op.always(nil, 'invalid_job') + end + local key, kerr = key_for(self, job.job_id) + if not key then return op.always(nil, kerr or 'invalid_job_id') end + + local body, jerr = cjson.encode(copy(job)) + if type(body) ~= 'string' then + return op.always(nil, 'control_store_job_json_encode_failed:' .. tostring(jerr)) + end + + local put_opts, perr = cap_args.new.ControlStorePutOpts(key, body) + if not put_opts then return op.always(nil, perr or 'invalid_control_store_put_opts') end + return call_op(self, 'put', put_opts):wrap(function (ok, err) + if ok == nil then return nil, err or 'control_store_put_failed' end + return true, nil + end) + end) +end + +function Store:delete_job_op(job_id) + return op.guard(function () + local key, kerr = key_for(self, job_id) + if not key then return op.always(nil, kerr or 'invalid_job_id') end + local delete_opts, derr = cap_args.new.ControlStoreDeleteOpts(key) + if not delete_opts then return op.always(nil, derr or 'invalid_control_store_delete_opts') end + return call_op(self, 'delete', delete_opts):wrap(function (ok, err) + if ok == nil and tostring(err) ~= 'not found' then + return nil, err or 'control_store_delete_failed' + end + return true, nil + end) + end) +end + +function M.new(conn, opts) + opts = opts or {} + return setmetatable({ + _conn = conn, + _id = opts.id or opts.store_id or 'update', + _prefix = opts.prefix or 'update-job-', + _call_opts = opts.call_opts, + }, Store) +end + +M.Store = Store +return M diff --git a/src/services/update/job_store_memory.lua b/src/services/update/job_store_memory.lua new file mode 100644 index 00000000..2a2c5729 --- /dev/null +++ b/src/services/update/job_store_memory.lua @@ -0,0 +1,52 @@ +-- services/update/job_store_memory.lua +-- Strict op-only in-memory Update job store for tests and harnesses. + +local op = require 'fibers.op' +local model = require 'services.update.model' + +local M = {} +local Store = {} +Store.__index = Store + +local function copy(v) + return model.deep_copy(v) +end + +local function sorted_ids(jobs) + local ids = {} + for id in pairs(jobs or {}) do ids[#ids + 1] = id end + table.sort(ids) + return ids +end + +function Store:load_all_op() + local jobs = copy(self._jobs) + return op.always({ jobs = jobs, order = sorted_ids(jobs) }, nil) +end + +function Store:save_job_op(job) + if type(job) ~= 'table' or type(job.job_id) ~= 'string' or job.job_id == '' then + return op.always(nil, 'invalid_job') + end + self._jobs[job.job_id] = copy(job) + return op.always(true, nil) +end + +function Store:delete_job_op(job_id) + if type(job_id) ~= 'string' or job_id == '' then + return op.always(nil, 'invalid_job_id') + end + self._jobs[job_id] = nil + return op.always(true, nil) +end + +function M.new(initial) + local jobs = {} + for id, job in pairs((initial and initial.jobs) or {}) do + jobs[id] = copy(job) + end + return setmetatable({ _jobs = jobs }, Store) +end + +M.Store = Store +return M diff --git a/src/services/update/manager.lua b/src/services/update/manager.lua new file mode 100644 index 00000000..6ada5f86 --- /dev/null +++ b/src/services/update/manager.lua @@ -0,0 +1,318 @@ +-- services/update/manager.lua +-- +-- Generation-local manager request router. +-- +-- This module owns cheap manager request routing and admission into scoped +-- request work. Blocking request bodies live in manager_requests.lua. + +local fibers = require 'fibers' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local request_owner = require 'devicecode.support.request_owner' +local model_mod = require 'services.update.model' +local manager_requests = require 'services.update.manager_requests' +local projection = require 'services.update.projection' + +local M = {} + +local ok_uuid, uuid = pcall(require, 'uuid') + +local function copy(v) + return model_mod.deep_copy(v) +end + +local function new_id(prefix) + if ok_uuid and uuid and type(uuid.new) == 'function' then + return tostring(uuid.new()) + end + return (prefix or 'id') .. '-' .. tostring(math.floor((fibers.now() or 0) * 1000000)) .. '-' .. tostring(math.random(1, 1000000)) +end + +local function fail_request(req, reason) + local owner = request_owner.new(req) + local ok, err = owner:fail_once(reason) + if ok ~= true then error(err or tostring(reason or 'request_failed'), 0) end +end + +local function reply_request(req, value, label) + local owner = request_owner.new(req) + local ok, err = owner:reply_once(value) + if ok ~= true then error(err or label or 'manager_reply_failed', 0) end +end + +local function manager_payload(req) + return req and req.payload or nil +end + +local function normalise_method(method) + method = tostring(method or 'status') + return method:gsub('-', '_') +end + +function M.method(req) + local payload = manager_payload(req) + local method = req and req._update_method or 'status' + return normalise_method(method), payload +end + +local function start_scoped_request(ctx, req, method, runner) + local request_id = new_id('req') + local owner = request_owner.new(req) + local handle, err = scoped_work.start { + lifetime_scope = ctx.request_root, + reaper_scope = ctx.request_root, + report_scope = ctx.scope, + + identity = { + kind = 'manager_request_done', + service_id = ctx.service_id, + generation = ctx.generation, + method = method, + request_id = request_id, + }, + + setup = function (scope) + scope:finally(function (_, status, primary) + if not owner:done() then + if status == 'ok' then + owner:finalise_unresolved(primary or 'request_scope_closed') + else + owner:finalise_unresolved(primary or status or 'request_cancelled') + end + end + end) + return { + request_owner = owner, + cancel_owned_now = function (reason) + owner:abandon_unresolved(reason or 'caller_abandoned') + return true + end, + } + end, + + cancel_op = owner:caller_cancel_op(), + + run = function (work_scope, setup) + return runner(work_scope, setup and setup.request_owner or owner) + end, + + report = function (ev) + return queue.try_admit_required( + ctx.done_tx, + ev, + 'update_manager_request_completion_report_failed' + ) + end, + } + + if not handle then + owner:fail_once(err or 'request_start_failed') + return nil, err + end + + ctx.manager_work[request_id] = handle + return handle, nil +end + +local function handle_status(ctx, req) + reply_request(req, projection.manager_status(ctx.snapshot or {}), 'status_reply_failed') +end + +local function handle_list(ctx, req) + reply_request(req, { ok = true, jobs = ctx.jobs:list() }, 'list_reply_failed') +end + +local function handle_get(ctx, req, payload) + local owner = request_owner.new(req) + local job_id = type(payload) == 'table' and payload.job_id or nil + local job = job_id and ctx.jobs:get(job_id) or nil + if not job then + local ok, err = owner:fail_once('not_found') + if ok ~= true then error(err or 'get_failure_reply_failed', 0) end + return + end + local ok, err = owner:reply_once({ ok = true, job = job }) + if ok ~= true then error(err or 'get_reply_failed', 0) end +end + + +local function jobs_ready(ctx) + return ctx.jobs == nil or type(ctx.jobs.ready) ~= 'function' or ctx.jobs:ready() == true +end + +local function reject_if_jobs_not_ready(ctx, req) + if jobs_ready(ctx) then return false end + fail_request(req, 'job_runtime_not_ready') + return true +end + +local function is_ingest_method(method) + return method == 'ingest_create' + or method == 'ingest_append' + or method == 'ingest_commit' + or method == 'ingest_abort' +end + +local function handle_create(ctx, req) + if reject_if_jobs_not_ready(ctx, req) then return end + return start_scoped_request(ctx, req, 'create_job', function (work_scope, owner) + return manager_requests.create_job(work_scope, { + request_owner = owner, + request = req, + jobs = ctx.jobs, + config = ctx.config, + generation = ctx.generation, + artifact_store = ctx.artifact_store, + }) + end) +end + +local function handle_start(ctx, req, payload) + if reject_if_jobs_not_ready(ctx, req) then return end + payload = type(payload) == 'table' and payload or {} + local job_id = payload.job_id + local job = ctx.jobs:get(job_id) + local phase = payload._forced_phase or payload.phase or (job and job.state == 'awaiting_commit' and 'commit' or 'stage') + + -- Cheap rejection is allowed, but durable admission is owned by job_runtime. + -- The request scope persists active intent; the service-owned active launcher + -- later starts work from that durable intent. + if not job then + fail_request(req, 'not_found') + return + end + if ctx.active ~= nil then + local active = ctx.active + local same_job_commit_release_lag = active + and active.job_id == job_id + and phase == 'commit' + and job + and job.state == 'awaiting_commit' + if active ~= nil and not same_job_commit_release_lag then + fail_request(req, 'slot_busy') + return + end + end + + return start_scoped_request(ctx, req, phase == 'commit' and 'commit_job' or 'start_job', function (work_scope, owner) + return manager_requests.start_job(work_scope, { + request_owner = owner, + request = req, + jobs = ctx.jobs, + job_id = job_id, + generation = ctx.generation, + phase = phase, + }) + end) +end + +local function handle_patch(ctx, req, payload, method, patch) + if reject_if_jobs_not_ready(ctx, req) then return end + payload = type(payload) == 'table' and payload or {} + local job_id = payload.job_id + if type(job_id) ~= 'string' or job_id == '' then + fail_request(req, 'job_id_required') + return + end + return start_scoped_request(ctx, req, method, function (work_scope, owner) + return manager_requests.persist_job_state(work_scope, { + request_owner = owner, + request = req, + jobs = ctx.jobs, + job_id = job_id, + generation = ctx.generation, + method = method, + patch = patch(payload), + }) + end) +end + +local function handle_cancel(ctx, req, payload) + return handle_patch(ctx, req, payload, 'cancel_job', function (p) + return { + state = 'cancelled', + next_step = nil, + error = p.reason or 'cancelled', + } + end) +end + +local function handle_retry(ctx, req, payload) + return handle_patch(ctx, req, payload, 'retry_job', function (_) + return { + state = 'created', + next_step = 'start', + error = nil, + active = nil, + active_token = nil, + active_intent = nil, + } + end) +end + +local function handle_discard(ctx, req, payload) + if reject_if_jobs_not_ready(ctx, req) then return end + payload = type(payload) == 'table' and payload or {} + local job_id = payload.job_id + if type(job_id) ~= 'string' or job_id == '' then + fail_request(req, 'job_id_required') + return + end + return start_scoped_request(ctx, req, 'discard_job', function (work_scope, owner) + return manager_requests.discard_job(work_scope, { + request_owner = owner, + request = req, + jobs = ctx.jobs, + job_id = job_id, + generation = ctx.generation, + }) + end) +end + +function M.handle_request(ctx, req) + local method, payload = M.method(req) + + if is_ingest_method(method) then + if ctx.ingest and type(ctx.ingest.submit) == 'function' then + local ok, err = ctx.ingest:submit(req) + if ok ~= true then fail_request(req, err or 'ingest_admission_failed') end + return + end + fail_request(req, 'ingest_unavailable') + return + end + + if method == 'status' then handle_status(ctx, req); return end + if method == 'list_jobs' then handle_list(ctx, req); return end + if method == 'get_job' then handle_get(ctx, req, payload); return end + if method == 'create_job' then handle_create(ctx, req); return end + if method == 'start_job' then handle_start(ctx, req, payload); return end + if method == 'commit_job' then + payload = type(payload) == 'table' and payload or {} + payload._forced_phase = 'commit' + handle_start(ctx, req, payload) + return + end + if method == 'cancel_job' then handle_cancel(ctx, req, payload); return end + if method == 'retry_job' then handle_retry(ctx, req, payload); return end + if method == 'discard_job' then handle_discard(ctx, req, payload); return end + + fail_request(req, 'unsupported_update_method: ' .. tostring(method)) +end + +function M.handle_done(ctx, ev) + if ev.service_id ~= ctx.service_id then return false, 'stale_service' end + if ev.generation ~= ctx.generation then return false, 'stale_generation' end + + ctx.manager_work[ev.request_id] = nil + + -- Durable job state is owned by job_runtime. Manager completions are + -- observations only; update the generation projection from the runtime + -- snapshot if the request completed successfully or failed after a state + -- transition attempt. + return true, nil +end + +M.fail_request = fail_request + +return M diff --git a/src/services/update/manager_requests.lua b/src/services/update/manager_requests.lua new file mode 100644 index 00000000..c0e113e4 --- /dev/null +++ b/src/services/update/manager_requests.lua @@ -0,0 +1,262 @@ +-- services/update/manager_requests.lua +-- +-- Scoped manager-request worker bodies. +-- +-- These functions own caller-visible request resolution. Durable job mutation +-- is requested from the service-owned job_runtime; this module does not call the +-- job store directly and does not mutate the authoritative repository. + +local fibers = require 'fibers' +local request_owner = require 'devicecode.support.request_owner' +local model_mod = require 'services.update.model' +local dcmcu = require 'services.update.artifacts.dcmcu' + +local M = {} + + +local function copy(v) + return model_mod.deep_copy(v) +end + +local function payload_of(req) + return type(req) == 'table' and type(req.payload) == 'table' and req.payload or {} +end + +local function install_owner(scope, req, reason, owner) + owner = owner or request_owner.new(req) + scope:finally(function (_, status, primary) + if status == 'failed' then + owner:finalise_unresolved(primary or reason or 'request_failed') + elseif status == 'cancelled' then + owner:finalise_unresolved(primary or reason or 'request_cancelled') + else + owner:finalise_unresolved(reason or primary or status or 'request_scope_closed') + end + end) + return owner +end + +local function reply_or_fail(owner, value, err, label) + local ok, rerr + if err == nil then + ok, rerr = owner:reply_once(value) + else + ok, rerr = owner:fail_once(err) + end + if ok ~= true then + error(rerr or (label or 'manager_request_reply_failed'), 0) + end +end + +local function assert_known_component(cfg, component) + local comps = cfg and cfg.components or nil + if comps and next(comps) ~= nil and comps[component] == nil then + return nil, 'unknown_component' + end + return true, nil +end + +local function require_artifact_ref(payload) + local ref = payload.artifact_ref or payload.artifact_id + if type(ref) ~= 'string' or ref == '' then + return nil, 'artifact_ref_required' + end + payload.artifact_ref = ref + payload.artifact_id = payload.artifact_id or ref + payload.artifact = payload.artifact or ref + return ref, nil +end + +local function transition(jobs, cmd) + if not jobs or type(jobs.admit_transition) ~= 'function' then + return nil, 'job_runtime_unavailable' + end + local handle, admit_err = jobs:admit_transition(cmd) + if not handle then + return nil, admit_err or 'job_transition_admission_failed' + end + return fibers.perform(handle:outcome_op()) +end + +function M.create_job(scope, params) + params = params or {} + local req = assert(params.request, 'create_job: request required') + local owner = install_owner(scope, req, 'create_job_cancelled', params.request_owner) + local payload = payload_of(req) + local component = payload.component + + if type(component) ~= 'string' or component == '' then + reply_or_fail(owner, nil, 'component_required', 'component_required_reply_failed') + return { + tag = 'manager_request_rejected', + method = 'create_job', + reason = 'component_required', + } + end + + local ok_component, component_err = assert_known_component(params.config, component) + if ok_component ~= true then + reply_or_fail(owner, nil, component_err, 'unknown_component_reply_failed') + return { + tag = 'manager_request_rejected', + method = 'create_job', + reason = component_err, + } + end + + local artifact_ref, artifact_err = require_artifact_ref(payload) + if artifact_err ~= nil then + reply_or_fail(owner, nil, artifact_err, 'artifact_ref_reply_failed') + return { + tag = 'manager_request_rejected', + method = 'create_job', + reason = artifact_err, + } + end + + if component == 'mcu' then + if payload.expected_image_id ~= nil then + reply_or_fail(owner, nil, 'expected_image_id_must_be_resolved_from_artifact', 'expected_image_id_reply_failed') + return { + tag = 'manager_request_rejected', + method = 'create_job', + reason = 'expected_image_id_must_be_resolved_from_artifact', + } + end + local store = params.artifact_store + if not store or type(store.open_source_op) ~= 'function' then + reply_or_fail(owner, nil, 'artifact_store_unavailable', 'artifact_store_reply_failed') + return { tag = 'manager_request_rejected', method = 'create_job', reason = 'artifact_store_unavailable' } + end + local source, serr = fibers.perform(store:open_source_op(artifact_ref)) + if source == nil then + reply_or_fail(owner, nil, serr or 'artifact_source_open_failed', 'artifact_source_reply_failed') + return { tag = 'manager_request_rejected', method = 'create_job', reason = serr or 'artifact_source_open_failed' } + end + local identity, ierr = fibers.perform(dcmcu.identity_from_source_op(source)) + if identity == nil then + reply_or_fail(owner, nil, ierr or 'dcmcu_identity_unavailable', 'dcmcu_identity_reply_failed') + return { tag = 'manager_request_rejected', method = 'create_job', reason = ierr or 'dcmcu_identity_unavailable' } + end + payload.expected_image_id = identity.image_id + end + + local result, err = transition(params.jobs, { + kind = 'create_job', + generation = params.generation, + payload = payload, + reason = 'create_job', + }) + if not result or result.status ~= 'persisted' then + local reason = err or (result and result.reason) or 'create_job_failed' + reply_or_fail(owner, nil, reason, 'create_job_reply_failed') + if result and result.status == 'rejected' then + return { tag = 'manager_request_rejected', method = 'create_job', reason = reason } + end + error(reason, 0) + end + + reply_or_fail(owner, { ok = true, job = result.job }, nil, 'create_job_reply_failed') + return result +end + +function M.start_job(scope, params) + params = params or {} + local req = assert(params.request, 'start_job: request required') + local owner = install_owner(scope, req, 'start_job_cancelled', params.request_owner) + local phase = params.phase or 'stage' + local job_id = params.job_id or (params.job and params.job.job_id) + + local result, err = transition(params.jobs, { + kind = 'start_job', + generation = params.generation, + job_id = job_id, + phase = phase, + request_id = params.request_id, + reason = phase == 'commit' and 'start_commit' or 'start_stage', + }) + + if not result or result.status ~= 'persisted' then + local reason = err or (result and result.reason) or 'job_transition_failed' + reply_or_fail(owner, nil, reason, 'start_job_persist_reply_failed') + if result and result.status == 'rejected' then + return { tag = 'manager_request_rejected', method = phase == 'commit' and 'commit_job' or 'start_job', reason = reason } + end + error(reason, 0) + end + + -- Durable active intent now exists. The service-owned active launcher, not + -- this request scope, will start active execution from job_runtime state. + reply_or_fail(owner, { + ok = true, + accepted = true, + job = result.job, + phase = result.phase, + token = result.token, + }, nil, 'start_job_reply_failed') + + return result +end + +function M.persist_job_state(scope, params) + params = params or {} + local req = assert(params.request, 'persist_job_state: request required') + local owner = install_owner(scope, req, tostring(params.method or 'request') .. '_cancelled', params.request_owner) + + local result, err = transition(params.jobs, { + kind = 'patch_job', + generation = params.generation, + job_id = params.job_id or (params.job and params.job.job_id), + patch = params.patch or params.job or {}, + reason = params.method or 'job_request', + }) + if not result or result.status ~= 'persisted' then + local reason = err or (result and result.reason) or 'job_persist_failed' + reply_or_fail(owner, nil, reason, 'persist_job_state_reply_failed') + if result and result.status == 'rejected' then + return { tag = 'manager_request_rejected', method = params.method or 'job_request', reason = reason } + end + error(reason, 0) + end + + reply_or_fail(owner, { + ok = true, + job = result.job, + accepted = true, + }, nil, 'persist_job_state_reply_failed') + + return result +end + + +function M.discard_job(scope, params) + params = params or {} + local req = assert(params.request, 'discard_job: request required') + local owner = install_owner(scope, req, 'discard_job_cancelled', params.request_owner) + + local result, err = transition(params.jobs, { + kind = 'discard_job', + generation = params.generation, + job_id = params.job_id or (params.job and params.job.job_id), + reason = params.method or 'discard_job', + }) + if not result or result.status ~= 'persisted' then + local reason = err or (result and result.reason) or 'discard_job_failed' + reply_or_fail(owner, nil, reason, 'discard_job_reply_failed') + if result and result.status == 'rejected' then + return { tag = 'manager_request_rejected', method = 'discard_job', reason = reason } + end + error(reason, 0) + end + + reply_or_fail(owner, { + ok = true, + job = result.job, + accepted = true, + discarded = true, + }, nil, 'discard_job_reply_failed') + + return result +end + +return M diff --git a/src/services/update/model.lua b/src/services/update/model.lua new file mode 100644 index 00000000..c999ff4d --- /dev/null +++ b/src/services/update/model.lua @@ -0,0 +1,72 @@ +-- services/update/model.lua +-- +-- Update service snapshot helpers and observable model factory. + +local base_model = require 'devicecode.support.model' +local tablex = require 'shared.table' + +local M = {} +local BaseModel = base_model.Model +local Model = {} + +Model.__index = function (_, key) + return Model[key] or BaseModel[key] +end + +local is_array = tablex.is_array +local deep_copy = tablex.deep_copy +local deep_equal = tablex.deep_equal + +function M.service_initial(service_id, generation) + return { + service = service_id or 'update', + state = 'starting', + ready = false, + generation = generation or 0, + reason = nil, + config = nil, + active = nil, + update_active = nil, + jobs = { count = 0, by_id = {} }, + ingest = { count = 0, by_id = {} }, + pending = {}, + publisher = { state = 'starting' }, + dependencies = {}, + } +end + +function M.generation_initial(params) + params = params or {} + return { + service = params.service_id or 'update', + generation = params.generation or 1, + state = 'starting', + ready = false, + config = deep_copy(params.config or {}), + jobs = { count = 0, by_id = {} }, + ingest = { count = 0, by_id = {} }, + components = deep_copy(params.components or {}), + bundled = deep_copy(params.bundled or {}), + } +end + +function Model:terminate(reason) + return BaseModel.terminate(self, reason) +end + +function M.new(initial, opts) + opts = opts or {} + local instance = base_model.new(initial or {}, { + copy = opts.copy or deep_copy, + equals = opts.equals or deep_equal, + label = opts.label or 'update.model', + }) + return setmetatable(instance, Model) +end + +M.deep_copy = deep_copy +M.deep_equal = deep_equal +M.is_array = is_array +M.Model = Model + +return M diff --git a/src/services/update/observe.lua b/src/services/update/observe.lua new file mode 100644 index 00000000..061b1369 --- /dev/null +++ b/src/services/update/observe.lua @@ -0,0 +1,106 @@ +-- services/update/observe.lua +-- +-- Generation-owned component observer model. +-- +-- This is deliberately only an observation owner. It stores the latest +-- component snapshots, exposes versioned changes, and may be consumed by +-- active reconcile workers. It does not decide update policy and does not +-- perform Ops. + +local model_mod = require 'services.update.model' + +local M = {} + +local Observer = {} +Observer.__index = Observer + +local function copy(v) + return model_mod.deep_copy(v) +end + +local function initial_components(components) + local by_id = {} + for id, cfg in pairs(components or {}) do + by_id[id] = { + id = id, + config = copy(cfg), + state = nil, + origin = nil, + } + end + return by_id +end + +local function make_snapshot(service_id, components) + local by_id = initial_components(components) + local count = 0 + for _ in pairs(by_id) do count = count + 1 end + return { + service = service_id or 'update', + count = count, + by_id = by_id, + } +end + +function M.new(opts) + opts = opts or {} + return setmetatable({ + _model = model_mod.new(make_snapshot(opts.service_id, opts.components), { + label = opts.label or 'update.observe', + }), + }, Observer) +end + +function Observer:version() + return self._model:version() +end + +function Observer:snapshot() + return self._model:snapshot() +end + +function Observer:changed_op(seen) + return self._model:changed_op(seen) +end + +function Observer:terminate(reason) + return self._model:terminate(reason or 'observer_closed') +end + +function Observer:update_component(component, snapshot, origin) + if type(component) ~= 'string' or component == '' then + return nil, 'component required' + end + + return self._model:update(function (s) + s.by_id = s.by_id or {} + local cur = s.by_id[component] or { id = component } + cur.state = copy(snapshot) + cur.origin = copy(origin) + s.by_id[component] = cur + local count = 0 + for _ in pairs(s.by_id) do count = count + 1 end + s.count = count + return s + end) +end + +function Observer:remove_component(component, reason) + if type(component) ~= 'string' or component == '' then + return nil, 'component required' + end + + return self._model:update(function (s) + if s.by_id then + s.by_id[component] = nil + end + local count = 0 + for _ in pairs(s.by_id or {}) do count = count + 1 end + s.count = count + s.reason = reason + return s + end) +end + +M.Observer = Observer +return M diff --git a/src/services/update/projection.lua b/src/services/update/projection.lua new file mode 100644 index 00000000..8f84232c --- /dev/null +++ b/src/services/update/projection.lua @@ -0,0 +1,287 @@ +-- services/update/projection.lua +-- +-- Pure conversion from internal update snapshots to retained/public payloads. +-- Retained update state is intentionally compact: it describes the current +-- appliance update state, not an audit archive of all historic jobs. + +local model = require 'services.update.model' +local repo = require 'services.update.job_repository' + +local M = {} + +local function copy(v) + return model.deep_copy(v) +end + +local function jobs_by_id(snapshot) + local jobs = snapshot and snapshot.jobs or nil + local src = type(jobs) == 'table' and type(jobs.by_id) == 'table' and jobs.by_id or {} + local out = {} + for id, job in pairs(src) do + if type(job) == 'table' then + if job.job_id == nil and type(id) == 'string' then + -- Some recovery paths and legacy compact records may be keyed by durable + -- job id while carrying a minimal payload. Public component/update + -- projections must still expose the durable identity expected by UI and + -- integration callers. Do this at the projection boundary rather than + -- re-inflating retained/durable job records. + local copy_job = copy(job) + copy_job.job_id = id + out[id] = copy_job + else + out[id] = job + end + end + end + return out +end + +local function newest_job(snapshot, component) + local best + for _, job in pairs(jobs_by_id(snapshot)) do + if type(job) == 'table' and (component == nil or job.component == component) then + local bt = best and (best.updated_seq or best.created_seq or 0) or -1 + local jt = job.updated_seq or job.created_seq or 0 + if best == nil or jt > bt then best = job end + end + end + return best +end + +local function dependency_summary(deps) + local out = {} + for k, dep in pairs(type(deps) == 'table' and deps or {}) do + if type(dep) == 'table' then + out[k] = { + key = dep.key or k, + class = dep.class, + id = dep.id, + available = dep.available == true, + status = dep.status or dep.observed_status, + required = dep.required == true, + route_missing = dep.route_missing == true, + updated_at = dep.updated_at, + } + end + end + return out +end + +local function job_brief_fields(job) + job = repo.public_job(job) or {} + local result = type(job.result) == 'table' and job.result or nil + return { + job_id = job.job_id, + component = job.component, + state = job.state, + next_step = job.next_step, + error = job.error, + expected_image_id = job.expected_image_id, + artifact_ref = job.artifact_ref, + created_seq = job.created_seq, + updated_seq = job.updated_seq, + commit_attempt = copy(job.commit_attempt), + commit_result = copy(job.commit_result), + result = result and { + ok = result.ok, + tag = result.tag, + reason = result.reason, + error = result.error, + } or nil, + last_event = copy(job.last_event), + } +end + +function M.job_brief(job) + if type(job) ~= 'table' then return nil end + return job_brief_fields(job) +end + +local function action_set(job) + local state = type(job) == 'table' and job.state or nil + return { + commit = state == 'awaiting_commit', + discard = state == 'awaiting_commit' or state == 'created' or state == 'failed', + retry = state == 'failed' or state == 'timed_out', + } +end + +function M.component_summary(component, snapshot) + local job = newest_job(snapshot, component) + local brief = M.job_brief(job) + local active = type(job) == 'table' and not repo.is_terminal(job.state) or false + local out = { + schema = 'devicecode.update.component/1', + kind = 'update.component', + component = component, + state = (brief and brief.state) or 'idle', + ready = snapshot and snapshot.ready == true or nil, + current_job = active and brief or nil, + last_job = (not active) and brief or nil, + actions = action_set(job), + } + if brief then + -- Compatibility: existing UI/devhost callers treat state/update/component/ + -- itself as the visible job record and read fields such as job_id and + -- commit_attempt from the top level. Keep that compact surface while avoiding + -- the previous full job/history/stage_result payloads. + out.job_id = brief.job_id + out.expected_image_id = brief.expected_image_id + out.artifact_ref = brief.artifact_ref + out.next_step = brief.next_step + out.error = brief.error + out.created_seq = brief.created_seq + out.updated_seq = brief.updated_seq + out.commit_attempt = copy(brief.commit_attempt) + out.commit_result = copy(brief.commit_result) + out.result = copy(brief.result) + out.last_event = copy(brief.last_event) + end + return out +end + +local function component_ids(snapshot) + local ids = {} + local seen = {} + local cfg = snapshot and snapshot.config or nil + if type(cfg) == 'table' and type(cfg.components) == 'table' then + for id in pairs(cfg.components) do seen[id] = true end + end + for _, job in pairs(jobs_by_id(snapshot)) do + if type(job) == 'table' and type(job.component) == 'string' and job.component ~= '' then + seen[job.component] = true + end + end + for id in pairs(seen) do ids[#ids + 1] = id end + table.sort(ids) + return ids +end + +function M.service_summary(snapshot) + snapshot = snapshot or {} + local latest = newest_job(snapshot) + local latest_brief = M.job_brief(latest) + local comps = {} + for _, component in ipairs(component_ids(snapshot)) do + comps[component] = M.component_summary(component, snapshot) + end + return { + schema = 'devicecode.update.summary/1', + service = snapshot.service or 'update', + state = snapshot.state or 'unknown', + ready = snapshot.ready == true, + reason = snapshot.reason, + generation = snapshot.generation, + config = snapshot.config and { + rev = snapshot.config.rev, + schema = snapshot.config.schema, + service_id = snapshot.config.service_id, + namespace = snapshot.config.namespace, + component_count = snapshot.config.component_count, + bundled_enabled = snapshot.config.bundled_enabled, + } or nil, + active = snapshot.active and { + generation = snapshot.active.generation, + state = snapshot.active.state, + } or nil, + job = latest_brief and { + present = true, + job_id = latest_brief.job_id, + component = latest_brief.component, + state = latest_brief.state, + next_step = latest_brief.next_step, + expected_image_id = latest_brief.expected_image_id, + updated_seq = latest_brief.updated_seq, + result = latest_brief.result, + } or { present = false }, + jobs = { + count = latest_brief and 1 or 0, + last = latest_brief, + }, + components = comps, + dependencies = dependency_summary(snapshot.dependencies), + pending = copy(snapshot.pending), + publisher = snapshot.publisher and { state = snapshot.publisher.state } or nil, + } +end + +-- Backwards-compatible name for callers that still ask for service_state. +function M.service_state(snapshot) + return M.service_summary(snapshot) +end + +function M.capability(snapshot) + snapshot = snapshot or {} + return { + kind = 'update.service', + service = snapshot.service or 'update', + generation = snapshot.generation, + ready = snapshot.ready == true, + methods = { + 'status', + 'list-jobs', + 'get-job', + 'create-job', + 'start-job', + 'commit-job', + 'cancel-job', + 'retry-job', + 'discard-job', + }, + } +end + +function M.jobs(snapshot) + local list = {} + for _, job in pairs(jobs_by_id(snapshot)) do + local brief = M.job_brief(job) + if brief then list[#list + 1] = brief end + end + table.sort(list, function(a, b) + local ta = a.updated_seq or a.created_seq or 0 + local tb = b.updated_seq or b.created_seq or 0 + if ta == tb then return tostring(a.job_id) < tostring(b.job_id) end + return ta < tb + end) + local by_id = {} + local order = {} + for _, job in ipairs(list) do + order[#order + 1] = job.job_id + by_id[job.job_id] = job + end + return { count = #list, order = order, by_id = by_id } +end + +function M.job(job) + return repo.public_job(job) +end + +function M.job_timeline(job) + job = repo.public_job(job) or {} + local events = {} + if type(job.last_event) == 'table' then + events[1] = copy(job.last_event) + end + return { + kind = 'update.job.timeline', + job_id = job.job_id, + component = job.component, + state = job.state, + updated_seq = job.updated_seq, + events = events, + } +end + +function M.ingest(record) + return copy(record) +end + +function M.manager_status(snapshot) + local summary = M.service_summary(snapshot) + -- Manager RPC status keeps a compact jobs map briefly for older callers. + -- Retained state/update/summary deliberately does not carry this map. + summary.jobs = M.jobs(snapshot) + return { ok = true, snapshot = summary } +end + +return M diff --git a/src/services/update/publisher.lua b/src/services/update/publisher.lua new file mode 100644 index 00000000..ebaab598 --- /dev/null +++ b/src/services/update/publisher.lua @@ -0,0 +1,312 @@ +-- services/update/publisher.lua +-- +-- Retained publication owner for update models. +-- +-- Publication is separated from update policy. The service supervises this as a +-- normal scoped component; publisher.start is retained as a small convenience for +-- tests and direct embedding. + +local fibers = require 'fibers' +local scoped_work = require 'devicecode.support.scoped_work' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local topics = require 'services.update.topics' +local projection = require 'services.update.projection' +local model_mod = require 'services.update.model' + +local M = {} + +local Publisher = {} +Publisher.__index = Publisher + +local function copy(v) + return model_mod.deep_copy(v) +end + +local function publish_snapshot(conn, topic, project, snapshot) + local ok, err = bus_cleanup.retain(conn, topic, project(snapshot)) + if ok ~= true then + return nil, err or 'update publisher retain failed' + end + return true, nil +end + +local function retain_payload(conn, topic, payload) + local ok, err = bus_cleanup.retain(conn, topic, payload) + if ok ~= true then return nil, err end + return true, nil +end + +local function unretain(conn, topic) + local ok, err = bus_cleanup.unretain(conn, topic) + if ok ~= true then return nil, err end + return true, nil +end + +local function require_params(scope, params, name) + if type(scope) ~= 'table' then + error(name .. ': scope required', 2) + end + if type(params) ~= 'table' then + error(name .. ': params table required', 2) + end + if params.conn == nil then + error(name .. ': conn required', 2) + end + if params.model == nil then + error(name .. ': model required', 2) + end +end + +local function jobs_by_id(snapshot) + local jobs = snapshot and snapshot.jobs or nil + return type(jobs) == 'table' and type(jobs.by_id) == 'table' and jobs.by_id or {} +end + +local function ingest_by_id(snapshot) + local ingest = snapshot and snapshot.ingest or nil + if type(ingest) ~= 'table' then return {} end + if type(ingest.by_id) == 'table' then return ingest.by_id end + + -- Generation-owned ingest state snapshots are keyed directly by ingest id. + -- The publisher accepts both that internal shape and the public { by_id = ... } + -- shape so workflow publication is not coupled to one model owner. + local out = {} + for id, rec in pairs(ingest) do + if type(id) == 'string' and type(rec) == 'table' then + out[id] = rec + end + end + return out +end + +local function component_summaries(snapshot) + local out = {} + local seen = {} + local cfg = snapshot and snapshot.config or nil + if type(cfg) == 'table' and type(cfg.components) == 'table' then + for id in pairs(cfg.components) do seen[id] = true end + end + for _, job in pairs(jobs_by_id(snapshot)) do + if type(job) == 'table' and type(job.component) == 'string' and job.component ~= '' then + seen[job.component] = true + end + end + for id in pairs(seen) do + out[id] = projection.component_summary(id, snapshot) + end + return out +end + +local function retained_set_clear(conn, retained, make_topic) + for id in pairs(retained) do + unretain(conn, make_topic(id)) + retained[id] = nil + end +end + +local function unretain_workflow_job(conn, id) + local ok, err = unretain(conn, topics.workflow_update_job(id)) + if ok ~= true then return nil, err or 'update workflow job unretain failed' end + ok, err = unretain(conn, topics.workflow_update_job_timeline(id)) + if ok ~= true then return nil, err or 'update workflow job timeline unretain failed' end + return true, nil +end + +local function sync_workflows(conn, retained, snapshot) + -- Workflow job/timeline retained topics were useful during early development but + -- are not device state. Normal publication no longer creates them. This cleanup + -- removes anything retained by this publisher and any legacy job IDs reported by + -- the destructive startup scrub. + for id in pairs(retained.jobs or {}) do + local ok, err = unretain_workflow_job(conn, id) + if ok ~= true then return nil, err end + retained.jobs[id] = nil + end + local adoption = type(snapshot) == 'table' and type(snapshot.adoption) == 'table' and snapshot.adoption or {} + local seen_legacy = {} + local single = type(adoption.single_job) == 'table' and adoption.single_job or {} + for _, id in ipairs(single.legacy_job_ids or {}) do + if id ~= nil and not seen_legacy[id] then + seen_legacy[id] = true + local ok, err = unretain_workflow_job(conn, id) + if ok ~= true then return nil, err end + end + end + for _, row in ipairs(adoption.pruned or {}) do + local id = type(row) == 'table' and row.job_id or nil + if id ~= nil and not seen_legacy[id] then + seen_legacy[id] = true + local ok, err = unretain_workflow_job(conn, id) + if ok ~= true then return nil, err end + end + end + for id in pairs(retained.ingest or {}) do + local ok, err = unretain(conn, topics.workflow_artifact_ingest(id)) + if ok ~= true then return nil, err or 'artifact ingest workflow unretain failed' end + retained.ingest[id] = nil + end + return true, nil +end + +local function sync_components(conn, retained, snapshot) + local current = component_summaries(snapshot) + for id, payload in pairs(current) do + local ok, err = retain_payload(conn, topics.update_component(id), payload) + if ok ~= true then return nil, err or 'update component retain failed' end + retained.components[id] = true + end + for id in pairs(retained.components) do + if current[id] == nil then + local ok, err = unretain(conn, topics.update_component(id)) + if ok ~= true then return nil, err or 'update component unretain failed' end + retained.components[id] = nil + end + end + return true, nil +end + +local function publish_capabilities(conn, snapshot) + local mgr_methods = topics.manager_methods() + local ingest_methods = topics.ingest_methods() + local ok, err = retain_payload(conn, topics.update_manager_meta(), { + kind = 'cap.update-manager', + class = 'update-manager', + id = 'main', + owner = 'update', + methods = mgr_methods, + canonical_state = topics.update_summary(), + }) + if ok ~= true then return nil, err end + ok, err = retain_payload(conn, topics.update_manager_status(), { + state = snapshot.ready and 'available' or 'unavailable', + available = snapshot.ready == true, + ready = snapshot.ready == true, + reason = snapshot.reason, + }) + if ok ~= true then return nil, err end + + ok, err = retain_payload(conn, topics.artifact_ingest_meta(), { + kind = 'cap.artifact-ingest', + class = 'artifact-ingest', + id = 'main', + owner = 'update', + methods = ingest_methods, + }) + if ok ~= true then return nil, err end + return retain_payload(conn, topics.artifact_ingest_status(), { + state = snapshot.ready and 'available' or 'unavailable', + available = snapshot.ready == true, + ready = snapshot.ready == true, + reason = snapshot.reason, + }) +end + +local function publish_all(conn, retained, snapshot) + local ok, err = publish_snapshot(conn, topics.update_summary(), projection.service_summary, snapshot) + if ok ~= true then return nil, err end + ok, err = publish_capabilities(conn, snapshot) + if ok ~= true then return nil, err end + ok, err = sync_workflows(conn, retained, snapshot) + if ok ~= true then return nil, err end + return sync_components(conn, retained, snapshot) +end + +--- Publisher worker body. +function M.run(scope, params) + require_params(scope, params, 'publisher.run') + + local conn = params.conn + local model = params.model + local seen = model:version() + local retained = { + jobs = {}, + ingest = {}, + components = {}, + } + + scope:finally(function () + bus_cleanup.unretain(conn, topics.update_summary()) + bus_cleanup.unretain(conn, topics.update_manager_meta()) + bus_cleanup.unretain(conn, topics.update_manager_status()) + bus_cleanup.unretain(conn, topics.artifact_ingest_meta()) + bus_cleanup.unretain(conn, topics.artifact_ingest_status()) + for id in pairs(retained.jobs) do + bus_cleanup.unretain(conn, topics.workflow_update_job(id)) + bus_cleanup.unretain(conn, topics.workflow_update_job_timeline(id)) + retained.jobs[id] = nil + end + retained_set_clear(conn, retained.ingest, topics.workflow_artifact_ingest) + retained_set_clear(conn, retained.components, topics.update_component) + end) + + local initial = model:snapshot() + local ok, err = publish_all(conn, retained, initial) + if ok ~= true then + error(err or 'publisher_initial_publication_failed', 0) + end + + while true do + local version, snapshot = fibers.perform(model:changed_op(seen)) + if version == nil then + return { + role = 'update_publisher', + reason = 'model_closed', + } + end + + seen = version + + local ok_pub, pub_err = publish_all(conn, retained, snapshot) + if ok_pub ~= true then + error(pub_err or 'update publisher retain failed', 0) + end + end +end + +function Publisher:stop(reason) + if self._handle and self._handle.cancel then + self._handle:cancel(reason or 'publisher_stopped') + end + return true +end + +function M.start(scope, params) + require_params(scope, params, 'publisher.start') + + local handle, err = scoped_work.start { + lifetime_scope = scope, + reaper_scope = scope, + report_scope = scope, + + identity = { + kind = 'component_done', + component = 'publisher', + }, + + run = function (child_scope) + return M.run(child_scope, params) + end, + + report = function (ev) + if ev.status == 'failed' then + return nil, ev.primary or 'publisher_failed' + end + return true, nil + end, + } + + if not handle then + return nil, err + end + + return setmetatable({ + _handle = handle, + _model = params.model, + _conn = params.conn, + state_topic = topics.update_summary(), + }, Publisher), nil +end + +M.Publisher = Publisher + +return M diff --git a/src/services/update/service.lua b/src/services/update/service.lua new file mode 100644 index 00000000..6192b672 --- /dev/null +++ b/src/services/update/service.lua @@ -0,0 +1,1460 @@ +-- services/update/service.lua +-- +-- Top-level update service coordinator. +-- +-- The service owns service lifecycle, config watch, generation replacement, +-- manager endpoint binding, service model, publisher, and accepted active +-- update work. Generation-local update policy and projections live in +-- services.update.generation. + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local config_watch = require 'devicecode.support.config_watch' +local service_events = require 'devicecode.support.service_events' +local service_base = require 'devicecode.service_base' +local cap_deps_mod = require 'devicecode.support.capability_dependencies' + +local model_mod = require 'services.update.model' +local config_mod = require 'services.update.config' +local events = require 'services.update.events' +local generation = require 'services.update.generation' +local publisher = require 'services.update.publisher' +local projection = require 'services.update.projection' +local topics = require 'services.update.topics' +local job_store_memory = require 'services.update.job_store_memory' +local control_store_jobs = require 'services.update.job_store_control_store' +local job_runtime_mod = require 'services.update.job_runtime' +local active_runtime = require 'services.update.active_runtime' +local observe_mod = require 'services.update.observe' +local component_watch = require 'services.update.component_watch' +local artifact_store_bus = require 'services.update.artifacts.store_bus' +local component_backend_mod = require 'services.update.backends.component' +local router_backend_mod = require 'services.update.backends.router' + +local M = {} + +local Service = {} +Service.__index = Service + +local DEFAULT_DONE_QUEUE = 32 + +local handle_dependency_changed +local stop_job_runtime +local stop_runtime_dependents +local update_dependencies_projection +local set_waiting_for_job_store +local ensure_runtime_dependents +local reconcile_runtime_components +local classify_dependency_route_missing +local handle_artifact_route_missing + + +local function copy(v) + return model_mod.deep_copy(v) +end + +local function config_from_value(value, opts) + local raw, rev = config_mod.extract_payload(value) + return config_mod.normalise(raw or {}, { + rev = rev, + service_id = opts and opts.service_id or 'update', + }) +end + + +local function sorted_component_ids(cfg) + local ids = {} + for id in pairs((cfg and cfg.components) or {}) do ids[#ids + 1] = tostring(id) end + table.sort(ids) + return ids +end + +local function update_ready_summary(self) + local ids = sorted_component_ids(self and self._config or nil) + local suffix = (#ids > 0) and (' components=' .. table.concat(ids, ',')) or '' + return 'update ready; job runtime available' .. suffix +end + +local function pending_for_state(state, reason) + if state == 'waiting_for_job_store' then + return { + kind = 'runtime_start', + dependency = 'job_store', + reason = reason or 'job_store_unavailable', + } + end + if state == 'waiting_for_artifact_store' then + return { + kind = 'runtime_dependents_start', + dependency = 'artifact_store', + reason = reason or 'artifact_store_unavailable', + } + end + if state == 'starting' and reason == 'job_runtime_loading' then + return { + kind = 'job_runtime_load', + reason = reason, + } + end + return nil +end + +local function update_model_state(self, state, reason) + local ready = state == 'running' + self._model:update(function (s) + s.state = state + s.ready = ready + s.reason = reason + s.pending = s.pending or {} + if ready then + s.pending.runtime = nil + else + s.pending.runtime = pending_for_state(state, reason) + end + return s + end) + + if self._svc and type(self._svc.status) == 'function' then + local snapshot = self._model:snapshot() + local extra = { ready = ready } + if reason ~= nil then extra.reason = reason end + if snapshot and snapshot.pending ~= nil then extra.pending = snapshot.pending end + extra.dependencies = self._deps:snapshot() + self._svc:status(state, extra) + if self._last_logged_state ~= state or self._last_logged_reason ~= reason then + local level = state == 'failed' and 'error' or state == 'degraded' and 'warn' or ready and 'info' or 'debug' + local what = ready and 'update_ready' or 'update_state_changed' + local summary = ready and update_ready_summary(self) or string.format('update %s%s', tostring(state), reason and (' (' .. tostring(reason) .. ')') or '') + self._svc:log(level, what, { summary = summary, state = state, ready = ready, reason = reason, components = table.concat(sorted_component_ids(self._config), ',') }) + self._last_logged_state = state + self._last_logged_reason = reason + end + end +end + +local function apply_generation_snapshot(self, snapshot) + self._model:update(function (s) + s.generation = snapshot.generation or s.generation + s.config = snapshot.config or s.config + s.active = { + generation = snapshot.generation, + state = snapshot.state or 'running', + } + if self._jobs then + s.jobs = self._jobs:snapshot() + else + s.jobs = snapshot.jobs or s.jobs + end + s.ingest = snapshot.ingest or s.ingest + -- Active update work is service-owned. A generation snapshot may carry + -- the current service-owned active snapshot for projection, but service + -- completion handling remains authoritative. + s.update_active = snapshot.update_active + return s + end) +end + +local function clear_generation_snapshot(self) + self._model:update(function (s) + s.active = nil + s.update_active = nil + return s + end) +end + +local function make_generation_identity(self, generation_id) + return { + kind = 'generation_done', + service_id = self._service_id, + generation = generation_id, + } +end + +local function default_generation_runner(scope, params) + return generation.run(scope, params) +end + +local function close_active_manager_route(active, reason) + if active and active.manager_tx then + active.manager_tx:close(reason or 'generation_route_closed') + end + if active and active.service_tx then + active.service_tx:close(reason or 'generation_route_closed') + end +end + +local function cancel_active_generation(self, reason) + local active = self._current_generation + if not active then return end + + active.state = 'replacing' + close_active_manager_route(active, reason or 'generation_replaced') + + if active.handle and active.handle.cancel then + active.handle:cancel(reason or 'generation_replaced') + end +end + + +local function active_snapshot(self) + if not self._active_component or type(self._active_component.snapshot) ~= 'function' then + return nil + end + local snap = self._active_component:snapshot() + return snap and snap.active or nil +end + +local function route_generation_event(self, ev, label) + local active = self._current_generation + if not active or active.state ~= 'running' or not active.route_port then + return false, 'generation_not_ready' + end + return active.route_port:emit_required(ev, label or 'update_generation_route_event_failed') +end + +local function start_generation(self, cfg, reason) + local generation_id = self._next_generation + self._next_generation = generation_id + 1 + + local runner = self._generation_runner or default_generation_runner + local manager_tx, manager_rx = mailbox.new(self._manager_route_queue_len or 32, { + full = 'reject_newest', + }) + local service_tx, service_rx = mailbox.new(self._generation_service_queue_len or 16, { + full = 'reject_newest', + }) + local route_port = service_events.port(service_tx, { + service_id = self._service_id, + source = 'update_service', + source_id = self._service_id, + generation = generation_id, + }, { + mark_route_events = true, + label = 'update_generation_route_event_admission_failed', + }) + local generation_events_port = service_events.port(self._done_tx, { + service_id = self._service_id, + source = 'update_generation_scope', + source_id = tostring(generation_id), + generation = generation_id, + }, { + label = 'update_generation_completion_report_failed', + }) + + if self._svc then self._svc:debug('update_generation_started', { summary = string.format('update runtime generation %s started%s', tostring(generation_id), reason and (' (' .. tostring(reason) .. ')') or ''), generation = generation_id, reason = reason }) end + + local handle, err = scoped_work.start { + lifetime_scope = self._scope, + reaper_scope = self._scope, + report_scope = self._scope, + + identity = make_generation_identity(self, generation_id), + + run = function (gen_scope) + gen_scope:finally(function (_, status, primary) + local close_reason = primary or status or 'generation_closed' + manager_tx:close(close_reason) + service_tx:close(close_reason) + end) + + return runner(gen_scope, { + service_id = self._service_id, + generation = generation_id, + config = cfg, + reason = reason, + jobs = self._jobs, + manager_rx = manager_rx, + service_rx = service_rx, + active_snapshot = active_snapshot(self), + observer = self._component_observer, + artifact_store = self._artifact_store, + events_tx = self._done_tx, + done_queue_len = self._generation_done_queue_len, + }) + end, + + report = service_events.reporter( + generation_events_port, + 'update_generation_completion_report_failed' + ), + } + + if not handle then + local reason2 = err or 'generation_start_failed' + manager_tx:close(reason2) + service_tx:close(reason2) + return nil, err + end + + if self._jobs and type(self._jobs.set_current_generation) == 'function' then + self._jobs:set_current_generation(generation_id) + end + + self._current_generation = { + generation = generation_id, + config = cfg, + handle = handle, + manager_tx = manager_tx, + manager_rx = manager_rx, + service_tx = service_tx, + service_rx = service_rx, + route_port = route_port, + state = 'running', + } + + self._model:update(function (s) + s.generation = generation_id + s.config = config_mod.summary(cfg) + s.active = { + generation = generation_id, + state = 'running', + } + s.update_active = active_snapshot(self) + return s + end) + update_model_state(self, 'running') + + return handle, nil +end + +local function replace_generation(self, cfg, reason) + cancel_active_generation(self, reason or 'generation_replaced') + return start_generation(self, cfg, reason or 'config_changed') +end + +local function apply_config(self, payload, reason) + local cfg, err = config_from_value(payload, { service_id = self._service_id }) + if not cfg then + update_model_state(self, 'degraded', err or 'invalid_config') + return true, err + end + + if self._config and config_mod.material_equal(self._config, cfg) then + return true, nil + end + + self._config = cfg + self._model:update(function (s) + s.config = config_mod.summary(cfg) + return s + end) + + if self._jobs == nil or (self._artifact_store_dependency_required and not artifact_store_available_for_work(self)) then + return reconcile_runtime_components(self, reason or 'config_changed') + end + + local ok, start_err = replace_generation(self, cfg, reason or 'config_changed') + if not ok then + update_model_state(self, 'failed', start_err or 'generation_start_failed') + error(start_err or 'generation_start_failed', 0) + end + + return true, nil +end + +local function handle_generation_snapshot(self, ev) + if ev.service_id ~= self._service_id then return end + + local active = self._current_generation + if not active or active.generation ~= ev.generation then + return + end + + active.last_snapshot = ev.snapshot + apply_generation_snapshot(self, ev.snapshot or {}) +end + +local function handle_generation_done(self, ev) + if ev.service_id ~= self._service_id then + return + end + + local active = self._current_generation + if not active or active.generation ~= ev.generation then + return + end + + self._current_generation = nil + clear_generation_snapshot(self) + + if active.state == 'replacing' and ev.status == 'cancelled' then + return + end + + if ev.status == 'ok' then + if self._svc then self._svc:info('update_generation_completed', { generation = ev.generation, status = ev.status }) end + update_model_state(self, 'stopped', 'generation_completed') + self._complete = true + return + end + + if ev.status == 'cancelled' then + if self._svc then self._svc:warn('update_generation_cancelled', { generation = ev.generation, reason = ev.primary or 'generation_cancelled' }) end + update_model_state(self, 'stopped', ev.primary or 'generation_cancelled') + self._complete = true + return + end + + local reason = ev.primary or 'generation_failed' + if self._artifact_store_dependency_required and classify_dependency_route_missing(self, 'artifact_store', ev, reason) then + handle_artifact_route_missing(self, reason) + return + end + if self._svc then self._svc:error('update_generation_failed', { generation = ev.generation, reason = reason }) end + update_model_state(self, 'failed', reason) + error('update generation failed: ' .. tostring(reason), 0) +end + +local function request_owner_for(req) + return require('devicecode.support.request_owner').new(req) +end + +local function fail_manager_request(req, reason) + local owner = request_owner_for(req) + local ok, err = owner:fail_once(reason) + if ok ~= true then error(err or tostring(reason or 'manager_request_failed'), 0) end +end + +local function reply_manager_request(req, value, label) + local owner = request_owner_for(req) + local ok, err = owner:reply_once(value) + if ok ~= true then error(err or tostring(label or 'manager_reply_failed'), 0) end +end + +local function handle_manager_without_generation(_, req) + fail_manager_request(req, 'generation_not_ready') +end + +local function method_dependency_reason(self, method) + if method == 'status' then return nil end + + if self._job_store_dependency_required and not self._deps:available('job_store') then + return 'job_store_unavailable' + end + + -- list/get are read-only job-runtime operations. They need the durable job + -- runtime, but not the artifact store. Mutating manager and ingest operations + -- remain gated by the artifact-store dependency when the production artifact + -- path is configured. + if method == 'list-jobs' or method == 'get-job' then return nil end + + if self._artifact_store_dependency_required and not self._deps:available('artifact_store') then + return 'artifact_store_unavailable' + end + + return nil +end + +local function reply_shell_manager_request(self, req, method) + if method == 'status' then + reply_manager_request(req, projection.manager_status(self._model:snapshot()), 'status_reply_failed') + return true + end + + if self._jobs == nil or self._job_runtime_ready ~= true then return false end + + if method == 'list-jobs' then + reply_manager_request(req, { ok = true, jobs = self._jobs:list() }, 'list_reply_failed') + return true + end + + if method == 'get-job' then + local payload = req and req.payload or nil + payload = type(payload) == 'table' and payload or {} + local job = payload.job_id and self._jobs:get(payload.job_id) or nil + if not job then + fail_manager_request(req, 'not_found') + else + reply_manager_request(req, { ok = true, job = job }, 'get_reply_failed') + end + return true + end + + return false +end + +local function route_manager_request(self, req, method) + -- Public status is shell-owned. It reports service-level facts such as + -- dependencies, pending runtime work, readiness and the latest projected job + -- state. It is not routed through a generation-private queue. + if method == 'status' then + reply_shell_manager_request(self, req, method) + return + end + + local dependency_reason = method_dependency_reason(self, method) + if dependency_reason then + fail_manager_request(req, dependency_reason) + return + end + + -- Read-only job inspection is safe for the service shell to answer directly + -- while no generation has been admitted, provided the durable job runtime is + -- ready. Mutating work belongs to the active generation. + local active = self._current_generation + if not (active and active.state == 'running' and active.manager_tx ~= nil) then + if reply_shell_manager_request(self, req, method) then return end + + if self._jobs == nil then + fail_manager_request(req, self._job_store_dependency_required and 'job_store_unavailable' or 'job_runtime_not_ready') + return + end + if self._job_runtime_ready ~= true then + fail_manager_request(req, 'job_runtime_not_ready') + return + end + handle_manager_without_generation(self, req) + return + end + + if method ~= nil and type(req) == 'table' then + req._update_method = method + end + + local ok, err = queue.try_admit_now(active.manager_tx, req) + if ok == true then return end + + fail_manager_request(req, err or 'generation_busy') +end + +local function update_active_projection(self) + self._model:update(function (s) + s.update_active = active_snapshot(self) + return s + end) +end + +local function update_service_jobs_projection(self) + if not self._jobs then return end + self._model:update(function (s) + s.jobs = self._jobs:snapshot() + return s + end) +end + +local function consider_active_jobs(self) + if not self._active_component or type(self._active_component.consider_jobs) ~= 'function' then + return false, 'not_ready' + end + + local ok, err = self._active_component:consider_jobs() + update_active_projection(self) + + if ok == nil and err ~= 'slot_busy' and err ~= 'no_active_intent' and err ~= 'not_ready' then + update_model_state(self, 'failed', err) + error('update active runtime launch failed: ' .. tostring(err), 0) + end + + return ok, err +end + +local function handle_job_runtime_changed(self, ev) + self._jobs_seen = ev.version + if self._jobs and self._jobs:ready() and not self._job_runtime_ready then + self._job_runtime_ready = true + update_service_jobs_projection(self) + local ok, err = reconcile_runtime_components(self, 'job_runtime_ready') + if ok ~= true then + update_model_state(self, 'failed', err or 'runtime_dependents_start_failed') + error(err or 'runtime_dependents_start_failed', 0) + end + else + update_service_jobs_projection(self) + end + if self._current_generation then + apply_generation_snapshot(self, self._current_generation.last_snapshot or { + generation = self._current_generation.generation, + config = config_mod.summary(self._current_generation.config), + state = self._current_generation.state, + update_active = active_snapshot(self), + }) + end + consider_active_jobs(self) +end + +local function handle_active_runtime_changed(self, ev) + update_active_projection(self) + update_service_jobs_projection(self) + if self._current_generation then + apply_generation_snapshot(self, self._current_generation.last_snapshot or { + generation = self._current_generation.generation, + config = config_mod.summary(self._current_generation.config), + state = self._current_generation.state, + update_active = active_snapshot(self), + }) + if self._current_generation.state == 'running' then + local ok, err = route_generation_event(self, { + kind = 'service_active_snapshot', + snapshot = active_snapshot(self), + reason = ev and ev.reason or 'active_runtime_changed', + }, 'update_generation_active_snapshot_admission_failed') + if ok ~= true then + update_model_state(self, 'failed', err or 'active_snapshot_route_failed') + error(err or 'active_snapshot_route_failed', 0) + end + end + end +end + +local function reduce_event(self, ev) + if ev.kind == 'generation_done_queue_closed' then + error('update generation completion queue closed', 0) + end + + if ev.kind == 'config_watch_closed' then + update_model_state(self, 'degraded', 'config_watch_closed') + return + end + + if ev.kind == 'config_replay_done' then + return + end + + if ev.kind == 'config_removed' then + update_model_state(self, 'degraded', 'config_removed') + return + end + + if ev.kind == 'config_changed' then + apply_config(self, ev.payload, 'config_changed') + return + end + + if ev.kind == 'job_runtime_model_closed' then + local reason = ev.reason or 'job_runtime_model_closed' + if self._deps:classify_call_failure('job_store', ev, reason) == 'route_missing' then + stop_runtime_dependents(self, 'job_store_unavailable') + stop_job_runtime(self, 'job_store_unavailable') + update_dependencies_projection(self) + set_waiting_for_job_store(self, 'job_store_unavailable') + return + end + + -- changed_op() close notifications can arrive before the scoped-work + -- component_done event that carries the authoritative worker failure. While + -- the job runtime has not become ready, defer policy to that completion. + -- Clear the closed runtime model as an event source so the coordinator does + -- not spin on an already-closed changed_op() and starve manager/dependency + -- events. + if self._suppress_dependents_reason or reason == 'job_store_unavailable' then + return + end + + if self._job_store_dependency_required and self._job_runtime_ready ~= true then + self._jobs = nil + self._jobs_seen = nil + self._job_runtime_ready = false + return + end + + update_model_state(self, 'failed', reason) + error(reason, 0) + end + + if ev.kind == 'job_runtime_changed' then + handle_job_runtime_changed(self, ev) + return + end + + if ev.kind == 'manager_closed' then + update_model_state(self, 'degraded', 'manager_closed') + return + end + + if ev.kind == 'manager_request' then + route_manager_request(self, ev.request, ev.method) + return + end + + if ev.kind == 'capability_dependency_changed' or ev.kind == 'capability_dependency_closed' then + handle_dependency_changed(self, ev) + return + end + + if ev.kind == 'active_runtime_changed' then + handle_active_runtime_changed(self, ev) + return + end + + if ev.kind == 'generation_snapshot' then + handle_generation_snapshot(self, ev) + return + end + + if ev.kind == 'generation_done' then + handle_generation_done(self, ev) + return + end + + if ev.kind == 'component_done' and ev.component == 'job_runtime' then + if ev.status == 'failed' then + local reason = ev.primary or 'job_runtime_failed' + if self._deps:classify_call_failure('job_store', ev, reason) == 'route_missing' then + stop_runtime_dependents(self, 'job_store_unavailable') + stop_job_runtime(self, 'job_store_unavailable') + update_dependencies_projection(self) + set_waiting_for_job_store(self, 'job_store_unavailable') + return + end + update_model_state(self, 'failed', reason) + error('update job runtime failed: ' .. tostring(reason), 0) + end + if ev.status == 'cancelled' and (self._suppress_dependents_reason or ev.primary == 'job_store_unavailable' or ev.primary == 'artifact_store_unavailable') then + return + end + if not self._complete then + update_model_state(self, 'degraded', ev.primary or ev.status or 'job_runtime_stopped') + end + return + end + + if ev.kind == 'component_done' and ev.component == 'active_runtime' then + if ev.status == 'cancelled' and (self._suppress_dependents_reason or ev.primary == 'job_store_unavailable' or ev.primary == 'artifact_store_unavailable') then + return + end + self._active_component = nil + if ev.status == 'failed' then + local reason = ev.primary or 'active_runtime_failed' + if self._artifact_store_dependency_required and classify_dependency_route_missing(self, 'artifact_store', ev, reason) then + handle_artifact_route_missing(self, reason) + return + end + update_model_state(self, 'failed', reason) + error('update active runtime failed: ' .. tostring(reason), 0) + end + if not self._complete then + update_model_state(self, 'degraded', ev.primary or ev.status or 'active_runtime_stopped') + end + return + end + + if ev.kind == 'component_done' and ev.component == 'component_watch' then + if ev.status == 'cancelled' and (self._suppress_dependents_reason or ev.primary == 'job_store_unavailable' or ev.primary == 'artifact_store_unavailable') then + return + end + self._component_watch = nil + if ev.status == 'failed' then + local reason = ev.primary or 'component_watch_failed' + update_model_state(self, 'failed', reason) + error('update component watch failed: ' .. tostring(reason), 0) + end + if not self._complete then + update_model_state(self, 'degraded', ev.primary or ev.status or 'component_watch_stopped') + end + return + end + + if ev.kind == 'component_done' and ev.component == 'publisher' then + self.publisher = nil + self._publisher = nil + if ev.status == 'failed' then + local reason = ev.primary or 'publisher_failed' + update_model_state(self, 'failed', reason) + error('update publisher failed: ' .. tostring(reason), 0) + end + if not self._complete then + update_model_state(self, 'degraded', ev.primary or ev.status or 'publisher_stopped') + end + return + end + + error('update.service: unknown event kind: ' .. tostring(ev.kind), 0) +end + +local function coordinator_loop(self) + while not self._complete do + local ev = fibers.perform(events.next_service_event_op(self)) + reduce_event(self, ev) + end + + if self.publisher then + if self.publisher.cancel then + self.publisher:cancel('service_complete') + elseif self.publisher.stop then + self.publisher:stop('service_complete') + end + end + if self._jobs and self._jobs.cancel then + self._jobs:cancel('service_complete') + end + if self._active_component and self._active_component.cancel then + self._active_component:cancel('service_complete') + end + if self._component_watch and self._component_watch.cancel then + self._component_watch:cancel('service_complete') + end + if self._active_scope and self._active_scope.cancel then + self._active_scope:cancel('service_complete') + end + + return { + role = 'update_service', + service_id = self._service_id, + snapshot = self._model:snapshot(), + } +end + +local function start_publisher_component(self, conn, model) + if not conn then return nil, nil end + + local handle, err = scoped_work.start { + lifetime_scope = self._scope, + reaper_scope = self._scope, + report_scope = self._scope, + + identity = { + kind = 'component_done', + service_id = self._service_id, + component = 'publisher', + }, + + run = function (component_scope) + return publisher.run(component_scope, { + conn = conn, + model = model, + }) + end, + + report = service_events.reporter( + service_events.port(self._done_tx, { + service_id = self._service_id, + source = 'update_publisher', + source_id = 'publisher', + }, { + label = 'update_publisher_completion_report_failed', + }), + 'update_publisher_completion_report_failed' + ), + } + + if not handle then + return nil, err + end + + return handle, nil +end + +local function bind_manager(scope, conn, opts) + if not conn then return nil, nil end + if opts and opts.bind_manager == false then return nil, nil end + + local methods = topics.manager_methods() + local tx, rx = mailbox.new((opts and opts.manager_queue_len) or 16, { + full = 'reject_newest', + }) + local endpoints = {} + + local function cleanup_bound() + for _, ep in pairs(endpoints) do + bus_cleanup.unbind(conn, ep) + end + if tx and type(tx.close) == 'function' then tx:close('manager endpoints closed') end + end + + for _, method in ipairs(methods) do + local ep, err = bus_cleanup.bind(conn, topics.update_manager_rpc(method), { + queue_len = opts and opts.manager_queue_len or 16, + }) + if not ep then + cleanup_bound() + return nil, err + end + endpoints[method] = ep + end + + local ingest_methods = topics.ingest_methods() + for _, method in ipairs(ingest_methods) do + local ep, err = bus_cleanup.bind(conn, topics.artifact_ingest_rpc(method), { + queue_len = opts and opts.ingest_queue_len or opts and opts.manager_queue_len or 16, + }) + if not ep then + cleanup_bound() + return nil, err + end + endpoints['ingest:' .. method] = ep + end + + scope:finally(cleanup_bound) + + for key, ep in pairs(endpoints) do + local method = key + local is_ingest = false + if type(key) == 'string' and key:sub(1, 7) == 'ingest:' then + method = key:sub(8) + is_ingest = true + end + + local loop_method = method + local loop_is_ingest = is_ingest + local loop_ep = ep + local handle, spawn_err = scoped_work.start { + lifetime_scope = scope, + reaper_scope = scope, + report_scope = scope, + + identity = { + kind = 'manager_endpoint_loop_done', + method = loop_method, + is_ingest = loop_is_ingest, + }, + + run = function (ep_scope) + -- The endpoint loop is the first owner that is guaranteed to be + -- joined during service shutdown. Unbind here as well as in the + -- parent finaliser so that a restarted update service cannot race + -- stale manager endpoints left registered on the bus. + ep_scope:finally(function () + bus_cleanup.unbind(conn, loop_ep) + end) + + while true do + local req = fibers.perform(loop_ep:recv_op()) + if req == nil then + queue.try_admit_now(tx, { closed = true, method = loop_method, reason = 'endpoint_closed' }) + return { role = 'update_manager_endpoint', method = loop_method, reason = 'endpoint_closed' } + end + local routed_method = loop_is_ingest and ('ingest-' .. loop_method) or loop_method + local ok_admit, admit_err = queue.try_admit_now(tx, { + method = routed_method, + request = req, + }) + if ok_admit ~= true then + fail_manager_request(req, admit_err or 'manager_busy') + end + end + end, + } + if not handle then + cleanup_bound() + return nil, spawn_err or 'manager_endpoint_loop_start_failed' + end + end + + return rx, nil +end + +local function open_config_watch(scope, conn, opts) + opts = opts or {} + local watch = opts.config_watch or opts.config_feed + local owns_watch = false + + if watch == nil and opts.config_rx ~= nil then + return nil, nil + end + + if watch == nil and opts.watch_config ~= false then + if not conn then return nil, nil end + + local err + watch, err = config_watch.open(conn, opts.name or opts.service_id or 'update', { + topic = opts.config_topic or topics.config(), + queue_len = opts.config_queue_len or 8, + full = 'reject_newest', + changed_kind = 'config_changed', + closed_kind = 'config_watch_closed', + }) + if not watch then return nil, err end + owns_watch = true + end + + if owns_watch then + scope:finally(function () + if type(watch.close) == 'function' then + watch:close() + end + end) + end + + return watch, nil +end + + +local function job_store_dependency_required(params) + if params.job_store ~= nil then return false end + if params.job_store_kind == 'memory' or params.memory_job_store == true then return false end + return params.conn ~= nil +end + +local function artifact_store_dependency_required(params) + if params.artifact_store_dependency_required ~= nil then + return params.artifact_store_dependency_required == true + end + if params.require_artifact_store ~= nil then + return params.require_artifact_store == true + end + if params.artifact_store ~= nil then return false end + if params.conn == nil then return false end + -- The default production backend is artifact-store backed. Tests and + -- specialised services that supply their own backend or in-memory job store do + -- not need the bus artifact-store capability for service admission unless they + -- opt in explicitly with artifact_store_dependency_required=true. + if params.backend ~= nil then return false end + if params.job_store_kind == 'memory' or params.memory_job_store == true or params.job_store ~= nil then return false end + return true +end + +update_dependencies_projection = function(self) + self._model:update(function (s) + s.dependencies = self._deps:snapshot() + return s + end) +end + +local function make_job_store(params) + if params.job_store ~= nil then return params.job_store end + if params.job_store_kind == 'memory' or params.memory_job_store == true then + return job_store_memory.new(params.initial_jobs) + end + return control_store_jobs.new(params.conn, { + id = params.job_store_id or params.control_store_id or 'update', + prefix = params.job_store_prefix or 'update-job-', + call_opts = params.job_store_call_opts, + }) +end + +local function dependency_effectively_available(self, key) + return self._deps:available(key) +end + +set_waiting_for_job_store = function(self, reason) + reason = reason or 'job_store_unavailable' + update_dependencies_projection(self) + update_model_state(self, 'waiting_for_job_store', reason) + return true, nil +end + +local function set_waiting_for_artifact_store(self, reason) + reason = reason or 'artifact_store_unavailable' + update_dependencies_projection(self) + update_model_state(self, 'waiting_for_artifact_store', reason) + return true, nil +end + +stop_job_runtime = function(self, reason) + reason = reason or 'job_store_unavailable' + local jobs = self._jobs + self._jobs = nil + self._jobs_seen = nil + self._job_runtime_ready = false + if jobs and type(jobs.cancel) == 'function' then + jobs:cancel(reason) + end + update_service_jobs_projection(self) + return true, nil +end + +stop_runtime_dependents = function(self, reason) + reason = reason or 'job_store_unavailable' + self._suppress_dependents_reason = reason + cancel_active_generation(self, reason) + if self._current_generation and self._current_generation.state == 'replacing' then + self._current_generation = nil + clear_generation_snapshot(self) + end + if self._active_component and self._active_component.cancel then self._active_component:cancel(reason or 'job_store_unavailable') end + if self._component_watch and self._component_watch.cancel then self._component_watch:cancel(reason or 'job_store_unavailable') end + if self._active_scope and self._active_scope.cancel then self._active_scope:cancel(reason or 'job_store_unavailable') end + self._active_scope = nil + self._active_component = nil + self._active_runtime = nil + self._component_watch = nil +end + +local function artifact_store_available_for_work(self) + return not self._artifact_store_dependency_required or dependency_effectively_available(self, 'artifact_store') +end + +classify_dependency_route_missing = function(self, key, ev, reason) + return self._deps:classify_call_failure(key, ev, reason) == 'route_missing' +end + +handle_artifact_route_missing = function(self, reason) + stop_runtime_dependents(self, reason or 'artifact_store_unavailable') + update_dependencies_projection(self) + return set_waiting_for_artifact_store(self, 'artifact_store_unavailable') +end + +local function ensure_job_runtime(self, reason) + if self._jobs ~= nil then return true, nil end + self._suppress_dependents_reason = nil + if self._job_store_dependency_required and not dependency_effectively_available(self, 'job_store') then + return set_waiting_for_job_store(self, 'job_store_unavailable') + end + + local params = self._params or {} + local job_store = self._job_store or make_job_store(params) + self._job_store = job_store + + local jobs, jobs_err = job_runtime_mod.start(self._scope, { + service_id = self._service_id, + store = job_store, + initial_jobs = params.initial_jobs, + done_tx = self._done_tx, + queue_len = params.job_runtime_queue_len, + retention = (self._config and self._config.retention) or params.job_retention, + }) + if not jobs then + return nil, jobs_err or 'update_job_repository_start_failed' + end + + self._jobs = jobs + self._jobs_seen = jobs:version() + self._job_runtime_ready = jobs:ready() + update_dependencies_projection(self) + update_service_jobs_projection(self) + return true, nil +end + +local function ensure_active_runtime(self, reason) + local params = self._params or {} + local adoption = self._jobs:adoption() or {} + + if self._active_component and type(self._active_component.update_adoption) == 'function' then + self._active_component:update_adoption(adoption) + end + if self._active_component and type(self._active_component.update_config) == 'function' then + self._active_component:update_config(self._config) + end + + if self._active_component ~= nil then return true, nil end + + local active_scope, active_scope_err = self._scope:child() + if not active_scope then + return nil, active_scope_err or 'update_active_runtime_scope_create_failed' + end + self._active_scope = active_scope + + local active_component, active_component_err = active_runtime.start_component(active_scope, { + service_id = self._service_id, + done_tx = self._done_tx, + work_scope = active_scope, + queue_len = params.active_runtime_queue_len, + jobs = self._jobs, + backend = self._backend, + observer = self._component_observer, + config = self._config, + adoption = adoption, + }) + if not active_component then + active_scope:cancel(active_component_err or 'update_active_runtime_start_failed') + self._active_scope = nil + return nil, active_component_err or 'update_active_runtime_start_failed' + end + self._active_component = active_component + self._active_runtime = active_component:state() + return true, nil +end + +local function ensure_component_watch(self, reason) + local params = self._params or {} + if self._component_watch ~= nil or params.conn == nil or params.watch_components == false then + return true, nil + end + + local cw_port = service_events.port(self._done_tx, { + service_id = self._service_id, + source = 'update_component_watch', + source_id = 'component_watch', + }, { + label = 'update_component_watch_completion_report_failed', + }) + local cwh, cwerr = component_watch.start(self._scope, { + service_id = self._service_id, + conn = params.conn, + observer = self._component_observer, + config = self._config, + queue_len = params.component_watch_queue_len, + report = service_events.reporter(cw_port, 'update_component_watch_completion_report_failed'), + }) + if not cwh then return nil, cwerr or 'update_component_watch_start_failed' end + self._component_watch = cwh + return true, nil +end + +ensure_runtime_dependents = function(self, reason) + if self._jobs == nil then return nil, 'job_runtime_not_ready' end + if self._artifact_store_dependency_required and not artifact_store_available_for_work(self) then + stop_runtime_dependents(self, 'artifact_store_unavailable') + return set_waiting_for_artifact_store(self, 'artifact_store_unavailable') + end + self._suppress_dependents_reason = nil + if self._jobs.ready and not self._jobs:ready() then + update_dependencies_projection(self) + update_service_jobs_projection(self) + update_model_state(self, 'starting', 'job_runtime_loading') + return true, nil + end + + local ok, err = ensure_active_runtime(self, reason or 'runtime_dependents') + if ok ~= true then return nil, err end + ok, err = ensure_component_watch(self, reason or 'runtime_dependents') + if ok ~= true then return nil, err end + + update_dependencies_projection(self) + update_service_jobs_projection(self) + return true, nil +end + +local function ensure_generation_running(self, reason) + if self._config and self._current_generation == nil then + local ok, err = start_generation(self, self._config, reason or 'initial') + if not ok then return nil, err or 'generation_start_failed' end + else + update_model_state(self, 'running') + end + return true, nil +end + +reconcile_runtime_components = function(self, reason) + if self._job_store_dependency_required and not dependency_effectively_available(self, 'job_store') then + stop_runtime_dependents(self, 'job_store_unavailable') + stop_job_runtime(self, 'job_store_unavailable') + return set_waiting_for_job_store(self, 'job_store_unavailable') + end + local ok, err = ensure_job_runtime(self, reason or 'runtime_reconcile') + if ok ~= true then return nil, err end + if self._job_runtime_ready ~= true then + update_dependencies_projection(self) + update_service_jobs_projection(self) + update_model_state(self, 'starting', 'job_runtime_loading') + return true, nil + end + + ok, err = ensure_runtime_dependents(self, reason or 'job_runtime_ready') + if ok ~= true then return nil, err end + if self._artifact_store_dependency_required and not artifact_store_available_for_work(self) then + return true, nil + end + ok, err = ensure_generation_running(self, reason or 'runtime_ready') + if ok ~= true then return nil, err end + consider_active_jobs(self) + return true, nil +end + +handle_dependency_changed = function(self, ev) + update_dependencies_projection(self) + if ev.key == 'job_store' then + if dependency_effectively_available(self, 'job_store') then + local ok, err = reconcile_runtime_components(self, 'job_store_available') + if ok ~= true then + update_model_state(self, 'failed', err or 'job_runtime_start_failed') + error(err or 'job_runtime_start_failed', 0) + end + elseif self._job_store_dependency_required then + -- Required durable storage disappeared or the status feed closed. Keep + -- the service shell and public status endpoint alive, but stop dependent + -- generation/active work, cancel the durable job runtime, and reject new + -- mutating manager requests until the dependency becomes effectively + -- available and the runtime has reloaded from the store. + stop_runtime_dependents(self, 'job_store_unavailable') + stop_job_runtime(self, 'job_store_unavailable') + set_waiting_for_job_store(self, 'job_store_unavailable') + end + elseif ev.key == 'artifact_store' then + if dependency_effectively_available(self, 'artifact_store') then + local ok, err = reconcile_runtime_components(self, 'artifact_store_available') + if ok ~= true then + update_model_state(self, 'failed', err or 'artifact_store_reconcile_failed') + error(err or 'artifact_store_reconcile_failed', 0) + end + elseif self._artifact_store_dependency_required then + stop_runtime_dependents(self, 'artifact_store_unavailable') + set_waiting_for_artifact_store(self, 'artifact_store_unavailable') + end + end +end + +function M.run(scope, params) + if type(scope) ~= 'table' then + error('update.service.run: scope required', 2) + end + params = params or {} + if type(params) ~= 'table' then + error('update.service.run: params table required', 2) + end + + local service_id = params.service_id or params.name or 'update' + local initial_cfg, cfg_err = config_from_value(params.config or {}, { service_id = service_id }) + if not initial_cfg then + error('update.service: ' .. tostring(cfg_err), 2) + end + + local initial = model_mod.service_initial(service_id, 0) + initial.config = config_mod.summary(initial_cfg) + + local service_model = model_mod.new(initial, { label = 'update.service' }) + scope:finally(function (_, status, primary) + service_model:terminate(primary or status or 'update service closed') + end) + + local done_tx, done_rx = mailbox.new(params.done_queue_len or DEFAULT_DONE_QUEUE, { + full = 'reject_newest', + }) + scope:finally(function () + done_tx:close('update service closed') + end) + + local manager_ep, merr = bind_manager(scope, params.conn, params) + if merr then error(merr, 2) end + + local config_watch, werr = open_config_watch(scope, params.conn, params) + if werr then error(werr, 2) end + + local job_dep_required = job_store_dependency_required(params) + local artifact_dep_required = artifact_store_dependency_required(params) + local dep_specs = {} + if job_dep_required then + dep_specs[#dep_specs + 1] = { + key = 'job_store', + class = 'control-store', + id = params.job_store_id or params.control_store_id or 'update', + required = true, + } + end + if artifact_dep_required then + dep_specs[#dep_specs + 1] = { + key = 'artifact_store', + class = 'artifact-store', + id = params.artifact_store_id or 'main', + required = true, + } + end + local deps, derr = cap_deps_mod.open(params.conn, dep_specs, { + queue_len = params.dependency_queue_len or params.status_queue_len or 8, + full = params.dependency_full or params.status_full or 'drop_oldest', + }) + if not deps then error(derr or 'update dependency watcher failed', 2) end + scope:finally(function (_, status, primary) + deps:terminate(primary or status or 'update dependencies closed') + end) + + local job_store = make_job_store(params) + + local artifact_store = params.artifact_store + if artifact_store == nil then + artifact_store = artifact_store_bus.new(params.conn, { + id = params.artifact_store_id or 'main', + call_opts = params.artifact_store_call_opts, + }) + end + + local component_observer = params.component_observer or observe_mod.new({ + service_id = service_id, + components = initial_cfg.components or {}, + }) + scope:finally(function (_, status, primary) + if component_observer and type(component_observer.terminate) == 'function' then + component_observer:terminate(primary or status or 'update component observer closed') + end + end) + + local backend = params.backend + if backend == nil then + local component_backend = component_backend_mod.new({ + conn = params.conn, + artifact_store = artifact_store, + observer = component_observer, + component = 'mcu', + }) + backend = router_backend_mod.new({ + components = { mcu = component_backend }, + default = component_backend, + commit_policy = 'no_duplicate', + }) + end + + local self = setmetatable({ + _scope = scope, + _service_id = service_id, + _svc = params.svc, + _model = service_model, + _done_tx = done_tx, + _done_rx = done_rx, + done_rx = done_rx, + _active_scope = nil, + _active_component = nil, + _active_runtime = nil, + _manager_ep = manager_ep, + manager_rx = manager_ep, + config_watch = config_watch, + config_rx = params.config_rx or config_watch, + publisher = nil, + _publisher = nil, + pending = {}, + _config = initial_cfg, + _current_generation = nil, + _next_generation = params.generation or 1, + _generation_runner = params.generation_runner, + _jobs_seen = nil, + _jobs = nil, + _job_store = job_store, + _job_store_dependency_required = job_dep_required, + _artifact_store_dependency_required = artifact_dep_required, + _deps = deps, + _artifact_store = artifact_store, + _component_observer = component_observer, + _component_watch = nil, + _backend = backend, + _job_runtime_ready = false, + _generation_done_queue_len = params.generation_done_queue_len, + _manager_route_queue_len = params.manager_route_queue_len, + _generation_service_queue_len = params.generation_service_queue_len, + _complete = false, + _params = params, + }, Service) + + update_dependencies_projection(self) + if job_dep_required and not deps:available('job_store') then + update_model_state(self, 'waiting_for_job_store', 'job_store_unavailable') + elseif artifact_dep_required and not deps:available('artifact_store') then + update_model_state(self, 'waiting_for_artifact_store', 'artifact_store_unavailable') + else + update_model_state(self, 'starting', 'job_runtime_loading') + end + + if params.conn and params.publish ~= false then + local pub, perr = start_publisher_component(self, params.conn, service_model) + if not pub then error(perr or 'update publisher start failed', 2) end + self.publisher = pub + self._publisher = pub + end + + local ok, err = reconcile_runtime_components(self, 'initial') + if ok ~= true then error(err or 'update runtime start failed', 2) end + + return coordinator_loop(self) +end + +function M.start(conn, opts) + opts = opts or {} + local scope = fibers.current_scope() + if not scope then + error('update.start must be called inside a fiber', 2) + end + + local svc = service_base.new(conn, { + name = opts.name or 'update', + env = opts.env, + meta = opts.meta, + announce = opts.announce, + }) + + svc:starting({ ready = false }) + + local params = copy(opts) + params.conn = conn + params.name = opts.name or 'update' + params.svc = svc + + M.run(scope, params) + + svc:stopped({ reason = 'returned' }) + error('update service returned unexpectedly', 0) +end + +M.Service = Service + +return M diff --git a/src/services/update/topics.lua b/src/services/update/topics.lua new file mode 100644 index 00000000..afd3f5c1 --- /dev/null +++ b/src/services/update/topics.lua @@ -0,0 +1,117 @@ +-- services/update/topics.lua +-- +-- Pure topic construction for the update service. + +local topic = require 'shared.topic' + +local M = {} + +local function t(...) + return { ... } +end + +local MANAGER_METHODS = { + 'status', + 'list-jobs', + 'get-job', + 'create-job', + 'start-job', + 'commit-job', + 'cancel-job', + 'retry-job', + 'discard-job', +} + +local INGEST_METHODS = { + 'create', + 'append', + 'commit', + 'abort', +} + +function M.config() + return t('cfg', 'update') +end + +function M.lifecycle_status() + return t('svc', 'update', 'status') +end + +function M.lifecycle_meta() + return t('svc', 'update', 'meta') +end + +function M.update_summary() + return t('state', 'update', 'summary') +end + +function M.update_component(component) + return t('state', 'update', 'component', component) +end + +function M.workflow_update_job(job_id) + return t('state', 'workflow', 'update-job', job_id) +end + +function M.workflow_update_job_timeline(job_id) + return t('state', 'workflow', 'update-job', job_id, 'timeline') +end + +function M.workflow_artifact_ingest(ingest_id) + return t('state', 'workflow', 'artifact-ingest', ingest_id) +end + +function M.update_manager_meta(id) + return t('cap', 'update-manager', id or 'main', 'meta') +end + +function M.update_manager_status(id) + return t('cap', 'update-manager', id or 'main', 'status') +end + +function M.update_manager_rpc(method, id) + return t('cap', 'update-manager', id or 'main', 'rpc', method) +end + +function M.artifact_ingest_meta(id) + return t('cap', 'artifact-ingest', id or 'main', 'meta') +end + +function M.artifact_ingest_status(id) + return t('cap', 'artifact-ingest', id or 'main', 'status') +end + +function M.artifact_ingest_rpc(method, id) + return t('cap', 'artifact-ingest', id or 'main', 'rpc', method) +end + +function M.manager_methods() + return topic.copy(MANAGER_METHODS) +end + +function M.ingest_methods() + return topic.copy(INGEST_METHODS) +end + +function M.obs_state(name) + return t('obs', 'v1', 'update', 'state', name) +end + + +function M.device_component(component) + return { 'state', 'device', 'component', component } +end + +function M.device_component_software(component) + return { 'state', 'device', 'component', component, 'software' } +end + +function M.device_component_update(component) + return { 'state', 'device', 'component', component, 'update' } +end + +function M.component_rpc(component, method) + return { 'cap', 'component', component, 'rpc', method } +end + +return M diff --git a/src/services/wifi.lua b/src/services/wifi.lua index e4cebcc3..b6d6aeda 100644 --- a/src/services/wifi.lua +++ b/src/services/wifi.lua @@ -1,1118 +1,888 @@ -local fiber = require "fibers.fiber" -local op = require "fibers.op" -local queue = require "fibers.queue" -local channel = require "fibers.channel" -local log = require "services.log" -local gen = require "services.wifi.gen" -local service = require "service" -local new_msg = require "bus".new_msg -local utils = require "services.wifi.utils" -local unpack = unpack or table.unpack - --- there's a connect/disconnect event available directly from hostapd. --- opkg install hostapd-utils will give you hostapd_cli - --- which you can run with an 'action file' (e.g. a simple shell script) --- hostapd_cli -a/bin/hostapd_eventscript -B - --- the script will be get interface cmd mac as parameters e.g. --- #!/bin/sh --- logger -t $0 "hostapd event received $1 $2 $3" - --- will result in something like this in the logs --- hostapd event received wlan1 AP-STA-CONNECTED xx:xx:xx:xx:xx:xx - --- I've used `iw event` for connection and disconnection events instead of the method above - -local INTERFACE_MODES = { - access_point = "ap", - client = "sta", - adhoc = "adhoc", - mesh = "mesh", - monitor = "monitor" +-- services/wifi.lua +-- +-- Wifi service: forwards radio/band configs to HAL capabilities, +-- manages SSIDs, subscribes to radio stats, handles MAC session tracking. +-- +-- Structure: +-- Constants / helpers +-- band_rpc / apply_band_steering +-- radio_rpc / apply_radio_config +-- on_radio_state / on_radio_event / radio_stats_loop +-- run_global_num_sta +-- Service context helpers (get_radio_cfg … reapply_all_radios) +-- Main-loop handlers (on_cfg … on_fs_cap) +-- WifiService.start + +local fibers = require "fibers" + +local perform = fibers.perform + +local base = require 'devicecode.service_base' +local cap_sdk = require 'services.hal.sdk.cap' +local gen = require 'services.wifi.gen' +local utils = require 'services.wifi.utils' + +------------------------------------------------------------------------ +-- Config types (annotation-only) +------------------------------------------------------------------------ + +---@class WifiRadioConfig +---@field name string UCI radio section name (e.g. "radio0") +---@field band RadioBand +---@field channel number|string +---@field htmode RadioHtmode +---@field channels? (number|string)[] required when channel == 'auto' +---@field txpower? number|string +---@field country? string ISO-3166-1 alpha-2 country code +---@field disabled? boolean +---@field report_period? number seconds between stats reports + +---@class WifiSsidConfig +---@field name? string SSID name +---@field mode string access_point | client | adhoc | mesh | monitor +---@field encryption? RadioEncryption +---@field password? string +---@field segment string NET segment id this SSID attaches to +---@field radios string[] radio section names this SSID should be added to +---@field mainflux_path? string path in the configs filesystem cap + +---@class WifiBandSteeringConfig +---@field globals? table global band-steering tunables +---@field timings? table timing / threshold tunables +---@field bands? table per-band scoring configuration + +---@class WifiServiceData +---@field schema string +---@field report_period? number +---@field radios WifiRadioConfig[] +---@field ssids WifiSsidConfig[] +---@field band_steering? WifiBandSteeringConfig + +---@class WifiServiceOpts +---@field name? string +---@field env? table + +-- Mode names the config uses vs what the radio driver expects +local MODE_MAP = { + access_point = 'ap', + client = 'sta', } +local function map_mode(mode) + return MODE_MAP[mode] or mode +end -local Radio = {} -Radio.__index = Radio - -function Radio:apply_config(config, report_period) - local reqs = {} - - local clear_req = self.conn:request(new_msg( - { 'hal', 'capability', 'wireless', self.index, 'control', 'clear_radio_config' } - )) - local resp, err = clear_req:next_msg_with_context(self.ctx) - clear_req:unsubscribe() - if err or resp and resp.payload and resp.payload.err then - log.error(string.format( - "%s - %s: Radio %s clear config error: %s", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - self.index or "(unknown)", - err or resp.payload.err - )) - return - end - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'wireless', self.index, 'control', 'set_report_period' }, - { report_period } - )) - - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'wireless', self.index, 'control', 'set_channels' }, - { - config.band, - config.channel, - config.htmode, - config.channels - } - )) - - if config.txpower then - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'wireless', self.index, 'control', 'set_txpower' }, - { config.txpower } - )) - end - - if config.country then - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'wireless', self.index, 'control', 'set_country' }, - { config.country } - )) - end +local function is_table(v) return type(v) == 'table' end - if config.disabled ~= nil then - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'wireless', self.index, 'control', 'set_enabled' }, - { not config.disabled } - )) - end - - local band = config.band - fiber.spawn(function() - self.band_ch:put(band) - end) +local function count_array(t) + return is_table(t) and #t or 0 +end - fiber.spawn(function() - local results = {} - for i, req in pairs(reqs) do - local resp, err = req:next_msg_with_context(self.ctx) - req:unsubscribe() - if err then return end - results[i] = resp and resp.payload or {} - end - if err then - log.error(string.format( - "%s - %s: Radio %s config error: %s", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - self.index or "(unknown)", - err - )) - return - end - for _, result in pairs(results) do - if result.err then - log.error(string.format( - "%s - %s: Radio %s config error: %s", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - self.index or "(unknown)", - result.err - )) +local function ssids_for_radio(ssid_cfgs, radio_id) + local count = 0 + for _, ssid_cfg in ipairs(ssid_cfgs or {}) do + if is_table(ssid_cfg.radios) then + for _, r in ipairs(ssid_cfg.radios) do + if r == radio_id then count = count + 1; break end end end - - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'wireless', self.index, 'control', 'apply' } - )) - local resp, ctx_err = req:next_msg_with_context(self.ctx) - req:unsubscribe() - if ctx_err or resp and resp.payload and resp.payload.err then - log.error(string.format( - "%s - %s: Radio %s apply config error: %s", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - self.index or "(unknown)", - ctx_err or resp.payload.err - )) - return - end - log.info(string.format( - "%s - %s: Radio %s applied configuration", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - self.index or "(unknown)" - )) - end) + end + return count end -function Radio:apply_ssids(ssid_configs) - fiber.spawn(function() - for i, ssid in ipairs(ssid_configs) do - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'wireless', self.index, 'control', 'add_interface' }, - { - ssid.name, - ssid.encryption or "none", - ssid.password or "", - ssid.network, - INTERFACE_MODES[ssid.mode], - { - enable_steering = ssid.has_band_steering or false - } - } - )) - self.interface_ch:put({ name = ssid.name, index = i }) - local resp, err = req:next_msg_with_context(self.ctx) - req:unsubscribe() - if err or resp and resp.payload and resp.payload.err then - log.error(string.format( - "%s - %s: Radio %s add SSID error: %s", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - self.index or "(unknown)", - err or resp.payload.err - )) - else - self.ssids[#self.ssids + 1] = resp.payload.result - end - end +------------------------------------------------------------------------ +-- Band steering application +------------------------------------------------------------------------ + +-- Remap tables for config-level scoring keys → band driver option keys +local RSSI_SCORING_REMAP = { + center = 'rssi_center', + weight = 'rssi_weight', + good_threshold = 'rssi_reward_threshold', + good_reward = 'rssi_reward', + bad_threshold = 'rssi_penalty_threshold', + bad_penalty = 'rssi_penalty', +} +local CHAN_UTIL_SCORING_REMAP = { + good_threshold = 'channel_util_reward_threshold', + good_reward = 'channel_util_reward', + bad_threshold = 'channel_util_penalty_threshold', + bad_penalty = 'channel_util_penalty', +} - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'wireless', self.index, 'control', 'apply' } - )) - local resp, ctx_err = req:next_msg_with_context(self.ctx) - req:unsubscribe() - if ctx_err or resp and resp.payload and resp.payload.err then - log.error(string.format( - "%s - %s: Radio %s failed to apply SSIDs, reason: %s", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - self.index or "(unknown)", - ctx_err or resp.payload.err - )) - return - end - log.info(string.format( - "%s - %s: Radio %s applied %d SSIDs", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - self.index or "(unknown)", - #ssid_configs - )) - end) +local function remap_keys(src, mapping) + local out = {} + for src_key, dst_key in pairs(mapping) do + if src[src_key] ~= nil then out[dst_key] = src[src_key] end + end + return out end -function Radio:remove_ssids() - local reqs = {} - for _, id in ipairs(self.ssids) do - reqs[#reqs + 1] = self.conn:request(new_msg( - { 'hal', 'capability', 'wireless', self.index, 'control', 'delete_interface' }, - { id } - )) - end - for _, req in ipairs(reqs) do - local resp, err = req:next_msg_with_context(self.ctx) - req:unsubscribe() - if err or resp and resp.payload and resp.payload.err then - log.error(string.format( - "%s - %s: Radio %s remove SSID error: %s", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - self.index or "(unknown)", - err or resp.payload.err - )) - end - end - local req = self.conn:request(new_msg( - { 'hal', 'capability', 'wireless', self.index, 'control', 'apply' } - )) - local resp, ctx_err = req:next_msg_with_context(self.ctx) - req:unsubscribe() - if ctx_err or resp and resp.payload and resp.payload.err then - log.error(string.format( - "%s - %s: Radio %s apply after remove SSIDs error: %s", - self.ctx:value("service_name"), - self.ctx:value("fiber_name"), - self.index or "(unknown)", - ctx_err or resp.payload.err - )) - return - end - self.ssids = {} +-- Parse interface index from generated name suffix (e.g. "radio0_i2" → "2") +local function iface_idx(name) + return (name and name:match('_i(%d+)$')) or '0' end -function Radio:_report_metrics(ctx, conn) - local total_num_sta = 0 - local interfaces_num_sta = {} - local radio_band = string.sub(self.band_ch:get(), 1, 1) -- wait for radio configs to give radio band, - -- get integer part only e.g. 2g -> 2 - local hw_platform = 1 -- hardcoded for now, do we really need this? - local radio_interface_indexes = { - by_ssid = {}, - by_phy = {} - } - local interface_info_endpoints = { - power = { 'txpower' }, - channel = { 'channel', 'chan' }, - noise = { 'noise' }, - rx_bytes = { 'rx_bytes' }, - rx_packets = { 'rx_packets' }, - rx_dropped = { 'rx_dropped' }, - rx_errors = { 'rx_errors' }, - tx_bytes = { 'tx_bytes' }, - tx_packets = { 'tx_packets' }, - tx_dropped = { 'tx_dropped' }, - tx_errors = { 'tx_errors' } - } - local interface_info_subs = {} - for key, topic in pairs(interface_info_endpoints) do - interface_info_subs[key] = conn:subscribe({ - 'hal', - 'capability', - 'wireless', - self.index, - 'info', - 'interface', - '+', - unpack(topic) - }) - end - local client_info_endpoints = { - tx_bytes = { 'tx_bytes' }, - rx_bytes = { 'rx_bytes' }, - signal = { 'signal' }, - hostname = { 'hostname' } - } - local client_info_subs = {} - for key, topic in pairs(client_info_endpoints) do - local tokens = { - 'hal', - 'capability', - 'wireless', - self.index, - 'info', - 'interface', - '+', - 'client', - '+', - unpack(topic) - } - client_info_subs[key] = conn:subscribe(tokens) - end - local client_session_ids = {} - local client_sub = conn:subscribe({ - 'hal', - 'capability', - 'wireless', - self.index, - 'info', - 'interface', - '+', - 'client', - '+' - }) +---@param key string +---@return Topic +local function t_obs_metric(key) return { 'obs', 'v1', 'wifi', 'metric', key } end - local interface_ssid_sub = conn:subscribe({ - 'hal', - 'capability', - 'wireless', - self.index, - 'info', - 'interface', - '+', - 'ssid' - }) +---@param key string +---@return Topic +local function t_obs_event(key) return { 'obs', 'v1', 'wifi', 'event', key } end - local function handle_client_event(msg) - if msg and msg.payload then - local client = msg.payload - local mac = msg.topic[#msg.topic] - local interface = msg.topic[7] - local client_hash = gen.userid(mac) - local session_id = client_session_ids[client_hash] - local key = client.connected and "session_start" or "session_end" - if client.connected then - if session_id then -- defend against duplicate connection events - log.warn(string.format( - "%s - %s: Duplicate connection event for client %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - client_hash - )) - return - end - client_session_ids[client_hash] = gen.gen_session_id() - session_id = client_session_ids[client_hash] - else - if session_id == nil then -- defend against duplicate disconnection events - log.warn(string.format( - "%s - %s: Duplicate disconnection event for client %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - client_hash - )) - return - end - client_session_ids[client_hash] = nil - end - log.info(string.format( - "%s - %s: Client %s %s (interface: %s)", - ctx:value("service_name"), - ctx:value("fiber_name"), - client_hash, - client.connected and "connected" or "disconnected", - interface - )) - local interface_idx = radio_interface_indexes.by_phy[interface] - if interface_idx then - if not interfaces_num_sta[interface_idx] then - interfaces_num_sta[interface_idx] = 0 - end - local interface_num_sta = interfaces_num_sta[interface_idx] - local num_change = client.connected and 1 or -1 - total_num_sta = total_num_sta + num_change - - interface_num_sta = interface_num_sta + num_change - conn:publish(new_msg( - { 'wifi', 'hp', tostring(hw_platform), 'num_sta' }, - total_num_sta - )) - conn:publish(new_msg( - { 'wifi', 'hp', tostring(hw_platform), 'rd' .. tostring(radio_band), interface_idx, 'num_sta' }, - interface_num_sta - )) - interfaces_num_sta[interface_idx] = interface_num_sta - end - conn:publish(new_msg( - { 'wifi', 'clients', client_hash, 'sessions', session_id, key }, - client.timestamp - )) - end - end +------------------------------------------------------------------------ +-- Band steering RPC helper +------------------------------------------------------------------------ - local function handle_client_info(key, msg) - if msg and msg.payload then - local mac = msg.topic[9] - local client_hash = gen.userid(mac) - local session_id = client_session_ids[client_hash] - if session_id then - conn:publish(new_msg( - { 'wifi', 'clients', client_hash, 'sessions', session_id, key }, - msg.payload - )) - end - end +local function band_rpc(band_cap, svc, method, args, opts) + local reply, err = perform(band_cap:call_control_op(method, args, opts)) + if err and err ~= "" then + svc:obs_log('warn', { what = 'band_rpc_failed', method = method, err = tostring(err) }) + return false + end + if not reply or reply.ok ~= true then + svc:obs_log('warn', { + what = 'band_rpc_error', + method = method, + reason = reply and reply.reason or 'nil', + }) + return false end + return true +end - local function handle_interface_info(key, msg) - if msg and msg.payload then - local interface = msg.topic[7] - if not radio_interface_indexes.by_phy[interface] then return end - local interface_index = radio_interface_indexes.by_phy[interface] - local topic = { - 'wifi', - 'hp', - tostring(hw_platform), - 'rd' .. tostring(radio_band), - tostring(interface_index), - key - } - conn:publish(new_msg( - topic, - msg.payload - )) +---@param band_cap CapabilityReference +---@param band_cfg WifiBandSteeringConfig +---@param svc ServiceBase +local function apply_band_steering(band_cap, band_cfg, svc) + if not is_table(band_cfg) then return end + local globals = band_cfg.globals or {} + local timings = band_cfg.timings or {} + local bands = band_cfg.bands or {} + + local slow = { timeout = 10.0 } + + svc:obs_log('debug', { what = 'band_config_start' }) + band_rpc(band_cap, svc, 'clear', {}, slow) + + -- kicking + local kicking = globals.kicking + if is_table(kicking) then + local args, err = cap_sdk.args.new.BandSetKickingOpts( + kicking.kick_mode or 'none', + kicking.bandwidth_threshold or 0, + kicking.kicking_threshold or 0, + kicking.evals_before_kick or 0) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_kicking', err = err }) + else + band_rpc(band_cap, svc, 'set_kicking', args) end end - local function handle_interface_ssid(msg) - if not msg.payload then return end - if radio_interface_indexes.by_ssid[msg.payload] and (not radio_interface_indexes.by_phy[msg.topic[7]]) then - radio_interface_indexes.by_phy[msg.topic[7]] = radio_interface_indexes.by_ssid[msg.payload] + -- station counting + if is_table(globals.stations) then + local st = globals.stations + local args, err = cap_sdk.args.new.BandSetStationCountingOpts( + st.use_station_count or false, + st.max_station_diff or 0) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_station_counting', err = err }) + else + band_rpc(band_cap, svc, 'set_station_counting', args) end end - log.info(string.format( - "%s - %s: Radio %s metrics reporting started", - ctx:value("service_name"), - ctx:value("fiber_name"), - self.index - )) - - while not ctx:err() do - local ops = { ctx:done_op() } - ops[#ops + 1] = self.band_ch:get_op():wrap(function(band) radio_band = string.sub(band, 1, 1) end) - ops[#ops + 1] = self.interface_ch:get_op():wrap(function(interface) - radio_interface_indexes.by_ssid[interface.name] = interface.index - end) - ops[#ops + 1] = interface_ssid_sub:next_msg_op():wrap(handle_interface_ssid) - ops[#ops + 1] = client_sub:next_msg_op():wrap(handle_client_event) - for key, sub in pairs(client_info_subs) do - ops[#ops + 1] = sub:next_msg_op():wrap(function(msg) handle_client_info(key, msg) end) + -- rrm_mode + if globals.rrm_mode then + local args, err = cap_sdk.args.new.BandSetRrmModeOpts(globals.rrm_mode) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_rrm_mode', err = err }) + else + band_rpc(band_cap, svc, 'set_rrm_mode', args) end - for key, sub in pairs(interface_info_subs) do - ops[#ops + 1] = sub:next_msg_op():wrap(function(msg) handle_interface_info(key, msg) end) - end - op.choice(unpack(ops)):perform() end -end -function Radio:get_index() - return self.index -end - -function Radio:remove() - self.ctx:cancel("Radio removed") -end - -function Radio.new(ctx, conn, index) - local self = setmetatable({}, Radio) - self.ctx = ctx - self.conn = conn - self.index = index - self.band_ch = channel.new() - self.interface_ch = channel.new() - self.ssids = {} - - service.spawn_fiber( - string.format('Radio %s Metrics', self.index), - conn, - ctx, - function(fctx) - self:_report_metrics(fctx, conn) + -- neighbour reports + if is_table(globals.neighbor_reports) then + local nr = globals.neighbor_reports + local args, err = cap_sdk.args.new.BandSetNeighbourReportsOpts( + nr.dyn_report_num or 0, + nr.disassoc_report_len or 0) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_neighbour_reports', err = err }) + else + band_rpc(band_cap, svc, 'set_neighbour_reports', args) end - ) - return self -end + end -local wifi_service = { - name = "wifi", - radio_add_queue = queue.new(), - radio_remove_queue = queue.new(), - config_queue = queue.new() -} -wifi_service.__index = wifi_service - -local function radio_listener(ctx, conn) - log.trace(string.format( - "%s - %s: Started", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - - local band_init_sub = conn:subscribe({ 'hal', 'capability', 'band', '+' }) - band_init_sub:next_msg() -- wait for band to be initialised - band_init_sub:unsubscribe() - local wireless_sub = conn:subscribe({ 'hal', 'capability', 'wireless', '+' }) - - while not ctx:err() do - local wireless_msg = wireless_sub:next_msg_with_context(ctx) - if wireless_msg and wireless_msg.payload then - local wireless_cap = wireless_msg.payload - if wireless_cap.connected then - wifi_service.radio_add_queue:put(wireless_cap.device.index) - else - wifi_service.radio_remove_queue:put(wireless_cap.device.index) - end + -- legacy options + if is_table(globals.legacy) then + local args, err = cap_sdk.args.new.BandSetLegacyOptionsOpts(globals.legacy) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_legacy_options', err = err }) + else + band_rpc(band_cap, svc, 'set_legacy_options', args) end end - wireless_sub:unsubscribe() - log.trace(string.format( - "%s - %s: Closed", - ctx:value("service_name"), - ctx:value("fiber_name") - )) -end - -local function radio_manager(ctx, conn) - local radios = {} - local radio_configs = {} - local report_period = nil - local ssid_configs = {} - - local function add_radio(radio_index) - if radios[radio_index] then - log.warn("Radio already exists:", radio_index) - return + -- update frequencies + if is_table(timings.updates) then + local args, err = cap_sdk.args.new.BandSetUpdateFreqOpts(timings.updates) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_update_freq', err = err }) + else + band_rpc(band_cap, svc, 'set_update_freq', args) end - log.info(string.format( - "%s - %s: New radio detected (%s)", - ctx:value("service_name"), - ctx:value("fiber_name"), - radio_index - )) - local radio = Radio.new(ctx, conn, radio_index) - local config = radio_configs[radio_index] - if config then - radio:apply_config(config, report_period) - radios[radio:get_index()] = radio - local ssid_config = ssid_configs[radio:get_index()] - if ssid_config then - radio:remove_ssids() - radio:apply_ssids(ssid_config) - end - end - radios[radio_index] = radio end - local function remove_radio(radio_index) - local radio = radios[radio_index] - if not radio then - log.warn(string.format( - "%s - %s: Radio not found for removal (%s)", - ctx:value("service_name"), - ctx:value("fiber_name"), - radio_index - )) - return + -- inactive client kickoff + if timings.inactive_client_kickoff ~= nil then + local args, err = cap_sdk.args.new.BandSetClientInactiveKickoffOpts(timings.inactive_client_kickoff) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_client_inactive_kickoff', err = err }) + else + band_rpc(band_cap, svc, 'set_client_inactive_kickoff', args) end - log.info(string.format( - "%s - %s: Removing radio (%s)", - ctx:value("service_name"), - ctx:value("fiber_name"), - radio_index - )) - radio:remove() - radios[radio_index] = nil end - local function validate_config(config) - if not config then - return "Missing configuration" - end - --- Check Radios config section - if not config.radios then - return "Missing radios configuration" - end - if not (type(config.radios) == "table") then - return string.format("Invalid radios type, should be a table but found %s", type(config.radios)) - end - for _, radio_cfg in ipairs(config.radios) do - if not radio_cfg.name then - return "Radio config missing name" - end - if not radio_cfg.band then - return string.format("Radio %s missing band", radio_cfg.name) - end + -- cleanup timeouts + if is_table(timings.cleanup) then + local args, err = cap_sdk.args.new.BandSetCleanupOpts(timings.cleanup) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_cleanup', err = err }) + else + band_rpc(band_cap, svc, 'set_cleanup', args) end + end - --- Check SSIDs config section - if not config.ssids then - return "Missing ssids configuration" - end - if not (type(config.ssids) == "table") then - return string.format("Invalid ssids type, should be a table but found %s", type(config.ssids)) - end - for _, ssid_cfg in ipairs(config.ssids) do - if not ssid_cfg.radios then - return "SSID config missing radios" - end - if not (type(ssid_cfg.radios) == "table") then - return string.format("SSID radios should be a table but found %s", type(ssid_cfg.radios)) - end - if #ssid_cfg.radios == 0 then - return "SSID config has empty radios list" - end - for _, radio_id in ipairs(ssid_cfg.radios) do - if not (type(radio_id) == "string") then - return string.format("SSID radio id should be a string but found %s", type(radio_id)) + -- per-band settings + for band_key, band_data in pairs(bands) do + if is_table(band_data) then + if band_data.initial_score ~= nil then + local args, err = cap_sdk.args.new.BandSetBandPriorityOpts(band_key, band_data.initial_score) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_band_priority', err = err }) + else + band_rpc(band_cap, svc, 'set_band_priority', args) end end - if not ssid_cfg.mode then - return "SSID config missing mode" - end - if not INTERFACE_MODES[ssid_cfg.mode] then - return string.format("SSID config has invalid mode: %s", ssid_cfg.mode) + if is_table(band_data.rssi_scoring) then + local args, err = cap_sdk.args.new.BandSetBandKickingOpts( + band_key, remap_keys(band_data.rssi_scoring, RSSI_SCORING_REMAP)) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_band_kicking', err = err }) + else + band_rpc(band_cap, svc, 'set_band_kicking', args) + end end - if not ssid_cfg.mainflux_path then - if not ssid_cfg.network then - return "SSID config missing network or mainflux_path" + if is_table(band_data.chan_util_scoring) then + local args, err = cap_sdk.args.new.BandSetBandKickingOpts( + band_key, remap_keys(band_data.chan_util_scoring, CHAN_UTIL_SCORING_REMAP)) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_band_kicking', err = err }) + else + band_rpc(band_cap, svc, 'set_band_kicking', args) end - if not (type(ssid_cfg.network) == "string") then - return string.format("SSID network should be a string but found %s", type(ssid_cfg.network)) + end + if is_table(band_data.support_bonuses) then + for support_key, reward in pairs(band_data.support_bonuses) do + local args, err = cap_sdk.args.new.BandSetSupportBonusOpts(band_key, support_key, reward) + if not args then + svc:obs_log('warn', { what = 'band_args_invalid', method = 'set_support_bonus', err = err }) + else + band_rpc(band_cap, svc, 'set_support_bonus', args) + end end end end + end - --- Check Band Steering config section - if not config.band_steering then - return "Missing band_steering configuration" - end - if not (type(config.band_steering) == "table") then - return string.format( - "Invalid band_steering type, should be a table but found %s", - type(config.band_steering) - ) - end - - local globals = config.band_steering.globals - if not globals then - return "Missing band_steering globals configuration" - end - if not (type(globals) == "table") then - return string.format("Invalid band_steering globals type, should be a table but found %s", type(globals)) - end + band_rpc(band_cap, svc, 'apply', {}, slow) + svc:obs_log('debug', { what = 'band_config_applied' }) +end - -- Validate the kicking settings - if not globals.kicking then - return "Missing band_steering kicking configuration" - end - local required_kicking_fields = { - "kick_mode", - "bandwidth_threshold", - "kicking_threshold", - "evals_before_kick" - } - for _, field in ipairs(required_kicking_fields) do - if globals.kicking[field] == nil then - return string.format("Missing band_steering kicking field: %s", field) - end - end +------------------------------------------------------------------------ +-- Radio configuration RPC helper +------------------------------------------------------------------------ + +local function radio_rpc(radio_cap, radio_id, svc, method, args) + local reply, err = radio_cap:call_control(method, args) + if err and err ~= "" then + svc:obs_log('warn', { + what = 'radio_rpc_failed', + radio = radio_id, + method = method, + err = tostring(err), + }) + return false + end + if not reply or reply.ok ~= true then + svc:obs_log('warn', { + what = 'radio_rpc_error', + radio = radio_id, + method = method, + reason = reply and reply.reason or 'nil', + }) + return false + end + return true, reply +end - local timing = config.band_steering.timings - if not timing then - return "Missing band_steering timing configuration" - end - if type(timing) ~= 'table' then - return string.format("Invalid timing type, should be a table but found %s", type(timing)) +------------------------------------------------------------------------ +-- Radio configuration sequence +------------------------------------------------------------------------ + +---@param radio_cap CapabilityReference +---@param radio_cfg WifiRadioConfig +---@param ssid_cfgs WifiSsidConfig[] +---@param fs_configs_cap CapabilityReference? +---@param band_steering_cfg WifiBandSteeringConfig? +---@param svc ServiceBase +local function apply_radio_config(radio_cap, radio_cfg, ssid_cfgs, fs_configs_cap, band_steering_cfg, svc) + local radio_id = radio_cap.id + + svc:obs_log('debug', { what = 'radio_config_start', radio = radio_id }) + + -- 1. Clear staged config + if not radio_rpc(radio_cap, radio_id, svc, 'clear_radio_config', {}) then return end + + -- 2. Set report period + do + local args, err = cap_sdk.args.new.RadioSetReportPeriodOpts(radio_cfg.report_period or 60) + if not args then + svc:obs_log('warn', { what = 'radio_args_invalid', method = 'set_report_period', err = err }) + return end + if not radio_rpc(radio_cap, radio_id, svc, 'set_report_period', args) then return end + end - -- Check updates structure - if not timing.updates then - return "Missing band_steering updates timing configuration" - end - if type(timing.updates) ~= 'table' then - return string.format("Invalid updates timing type, should be a table but found %s", type(timing.updates)) - end - local required_update_fields = { - "client", - "chan_util", - "hostapd" - } - for _, field in ipairs(required_update_fields) do - if timing.updates[field] == nil then - return string.format("Missing band_steering updates timing field: %s", field) - end + -- 3. Set channels + do + local args, err = cap_sdk.args.new.RadioSetChannelsOpts( + radio_cfg.band, radio_cfg.channel, radio_cfg.htmode, radio_cfg.channels) + if not args then + svc:obs_log('warn', { what = 'radio_args_invalid', method = 'set_channels', err = err }) + return end + if not radio_rpc(radio_cap, radio_id, svc, 'set_channels', args) then return end + svc:obs_log('debug', { + what = 'radio_channels_set', + radio = radio_id, + band = radio_cfg.band, + channel = radio_cfg.channel, + htmode = radio_cfg.htmode, + }) + end - -- Check cleanup structure - if not timing.cleanup then - return "Missing band_steering cleanup timing configuration" - end - if type(timing.cleanup) ~= 'table' then - return string.format("Invalid cleanup timing type, should be a table but found %s", type(timing.cleanup)) - end - local required_cleanup_fields = { - "client", - "probe", - "ap" - } - for _, field in ipairs(required_cleanup_fields) do - if timing.cleanup[field] == nil then - return string.format("Missing band_steering cleanup timing field: %s", field) - end + -- 4. txpower (optional) + if radio_cfg.txpower ~= nil then + local args, err = cap_sdk.args.new.RadioSetTxpowerOpts(radio_cfg.txpower) + if not args then + svc:obs_log('warn', { what = 'radio_args_invalid', method = 'set_txpower', err = err }) + else + radio_rpc(radio_cap, radio_id, svc, 'set_txpower', args) end + end - -- Check inactive_client_kickoff - if timing.inactive_client_kickoff == nil then - return "Missing band_steering inactive_client_kickoff timing configuration" + -- 5. country (optional) + if radio_cfg.country then + local args, err = cap_sdk.args.new.RadioSetCountryOpts(radio_cfg.country) + if not args then + svc:obs_log('warn', { what = 'radio_args_invalid', method = 'set_country', err = err }) + else + radio_rpc(radio_cap, radio_id, svc, 'set_country', args) end + end - local bands = config.band_steering.bands - if not bands then - return "Missing band_steering bands configuration" - end - if not (type(bands) == "table") then - return string.format("Invalid band_steering bands type, should be a table but found %s", type(bands)) + -- 6. enabled / disabled (optional) + if radio_cfg.disabled ~= nil then + local args, err = cap_sdk.args.new.RadioSetEnabledOpts(not radio_cfg.disabled) + if not args then + svc:obs_log('warn', { what = 'radio_args_invalid', method = 'set_enabled', err = err }) + else + radio_rpc(radio_cap, radio_id, svc, 'set_enabled', args) end + end - -- Validate each band configuration - for band_name, band_cfg in pairs(bands) do - if not (type(band_cfg) == "table") then - return string.format("Invalid band_steering band type for %s, should be a table but found %s", band_name, - type(band_cfg)) - end - - -- Check initial score - if band_cfg.initial_score == nil then - return string.format("Missing initial_score for band %s", band_name) + -- Determine enable_steering from band steering kick_mode + local kick_mode = band_steering_cfg + and is_table(band_steering_cfg.globals) + and is_table(band_steering_cfg.globals.kicking) + and band_steering_cfg.globals.kicking.kick_mode + local enable_steering = kick_mode and kick_mode ~= 'none' or false + + -- 7. Add interfaces per SSID + for _, ssid_cfg in ipairs(ssid_cfgs) do + repeat + -- Check if this SSID applies to this radio + local applies = false + if is_table(ssid_cfg.radios) then + for _, r in ipairs(ssid_cfg.radios) do + if r == radio_cap.id then + applies = true; break + end + end end + if not applies then break end - -- Check RSSI scoring configuration if present - if band_cfg.rssi_scoring then - if not (type(band_cfg.rssi_scoring) == "table") then - return string.format("Invalid rssi_scoring type for band %s, should be a table", band_name) + if ssid_cfg.mainflux_path then + -- Source credentials from filesystem cap; strict: skip on failure + if not fs_configs_cap then + svc:obs_log('warn', { what = 'no_fs_configs_cap', ssid = ssid_cfg.name }) + break end - - local required_rssi_fields = { - "good_threshold", - "good_reward", - "bad_threshold", - "bad_penalty" + local base_cfg = { + segment = ssid_cfg.segment, + encryption = ssid_cfg.encryption or 'none', + password = ssid_cfg.password or '', + mode = map_mode(ssid_cfg.mode or 'access_point'), } - for _, field in ipairs(required_rssi_fields) do - if band_cfg.rssi_scoring[field] == nil then - return string.format("Missing rssi_scoring.%s for band %s", field, band_name) + local ssids, serr = utils.parse_mainflux_ssids(fs_configs_cap, ssid_cfg.mainflux_path, base_cfg) + if not ssids then + svc:obs_log('warn', { what = 'mainflux_ssid_failed', ssid = ssid_cfg.name, err = serr }) + break + end + for _, mssd in ipairs(ssids) do + if type(mssd.segment) ~= 'string' or mssd.segment == '' then + svc:obs_log('warn', { what = 'ssid_missing_segment', ssid = mssd.name or mssd.ssid }) + break + end + local args, err = cap_sdk.args.new.RadioAddInterfaceOpts( + mssd.name or mssd.ssid or '', + mssd.encryption or 'none', + mssd.password or '', + mssd.segment, + map_mode(mssd.mode or 'ap'), + enable_steering) + if not args then + svc:obs_log('warn', { what = 'radio_args_invalid', method = 'add_interface', err = err }) + else + local ok, reply = radio_rpc(radio_cap, radio_id, svc, 'add_interface', args) + if ok then + svc:obs_log('debug', { + what = 'radio_iface_added', + radio = radio_id, + ssid = mssd.name or mssd.ssid, + segment = mssd.segment, + iface = reply and reply.reason or nil, + }) + end end end - end - - -- Check channel utilization configuration if present - if band_cfg.chan_util_scoring then - if not (type(band_cfg.chan_util_scoring) == "table") then - return string.format("Invalid chan_util_scoring type for band %s, should be a table", band_name) + else + -- Direct SSID from config + if type(ssid_cfg.segment) ~= 'string' or ssid_cfg.segment == '' then + svc:obs_log('warn', { what = 'ssid_missing_segment', ssid = ssid_cfg.name }) + break end - - local required_chan_fields = { - "good_threshold", - "good_reward", - "bad_threshold", - "bad_penalty" - } - for _, field in ipairs(required_chan_fields) do - if band_cfg.chan_util_scoring[field] == nil then - return string.format("Missing chan_util_scoring.%s for band %s", field, band_name) + local args, err = cap_sdk.args.new.RadioAddInterfaceOpts( + ssid_cfg.name or '', + ssid_cfg.encryption or 'none', + ssid_cfg.password or '', + ssid_cfg.segment, + map_mode(ssid_cfg.mode or 'access_point'), + enable_steering) + if not args then + svc:obs_log('warn', { what = 'radio_args_invalid', method = 'add_interface', err = err }) + else + local ok, reply = radio_rpc(radio_cap, radio_id, svc, 'add_interface', args) + if ok then + svc:obs_log('debug', { + what = 'radio_iface_added', + radio = radio_id, + ssid = ssid_cfg.name, + segment = ssid_cfg.segment, + iface = reply and reply.reason or nil, + }) end end end + until true + end - -- Check support bonuses if present - if band_cfg.support_bonuses and not (type(band_cfg.support_bonuses) == "table") then - return string.format("Invalid support_bonuses type for band %s, should be a table", band_name) - end + -- 8. Apply + if not radio_rpc(radio_cap, radio_id, svc, 'apply', {}) then return end + svc:obs_log('info', { + what = 'radio_ready', + summary = string.format('wifi radio %s ready band=%s channel=%s ssids=%d', tostring(radio_id), tostring(radio_cfg.band or '?'), tostring(radio_cfg.channel or '?'), ssids_for_radio(ssid_cfgs, radio_id)), + radio = radio_id, + band = radio_cfg.band, + channel = radio_cfg.channel, + htmode = radio_cfg.htmode, + txpower = radio_cfg.txpower, + ssids = ssids_for_radio(ssid_cfgs, radio_id), + }) +end + +------------------------------------------------------------------------ +-- Per-radio stats and session forwarding loop +------------------------------------------------------------------------ + +---@param conn Connection +---@param id string +---@param radio_cfg WifiRadioConfig? +---@param svc ServiceBase +local function radio_stats_loop(conn, id, radio_cfg, svc) + local band = ((radio_cfg and radio_cfg.band) or ''):sub(1, 1) + local hw_platform = '1' + local sessions = {} + local iface_sta = {} + local radio_sta = 0 + + local function on_radio_state(msg) + local key = msg.topic and msg.topic[5] + local p = msg.payload + if not (key and is_table(p)) then return end + + local prefix, stat = key:match('^(iface)_(.+)$') + if prefix then + local idx = iface_idx(p.interface) + conn:retain(t_obs_metric(stat), { + value = p.value, + namespace = { 'wifi', 'hp', hw_platform, 'rd' .. band, idx, stat }, + }) + return end - return nil + prefix, stat = key:match('^(client)_(.+)$') + if prefix then + local uid = gen.userid(p.mac) + local sid = sessions[p.mac] + if sid then + conn:retain(t_obs_metric(stat), { + value = p.value, + namespace = { 'wifi', 'clients', uid, 'sessions', sid, stat }, + }) + end + end end - -- New function to apply band steering configuration - local function apply_band_steering_config(ctx, conn, band_configs) - local reqs = {} + local function on_radio_event(msg) + local event_name = msg.topic and msg.topic[5] + if event_name ~= 'client_event' then return end - if band_configs.log_level then - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_log_level' }, - { band_configs.log_level } - )) - end + local p = msg.payload + if not (is_table(p) and p.mac) then return end - -- Configure global settings - local globals = band_configs.globals or {} - if globals.kicking then - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_kicking' }, - { - globals.kicking.kick_mode, - globals.kicking.bandwidth_threshold, - globals.kicking.kicking_threshold, - globals.kicking.evals_before_kick - } - )) - end + local mac = p.mac + local iface = p.interface + local connected = p.connected + local timestamp = p.timestamp or os.time() + local uid = gen.userid(mac) + local change = connected and 1 or -1 - if globals.stations then - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_station_counting' }, - { - globals.stations.use_station_count, - globals.stations.max_station_diff - } - )) - end + iface_sta[iface] = math.max(0, (iface_sta[iface] or 0) + change) + radio_sta = math.max(0, radio_sta + change) - if globals.rrm_mode then - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_rrm_mode' }, - { globals.rrm_mode } - )) - end + local idx = iface_idx(iface) + conn:retain(t_obs_metric('num_sta'), { + value = iface_sta[iface], + namespace = { 'wifi', 'hp', hw_platform, 'rd' .. band, idx, 'num_sta' }, + }) + conn:retain(t_obs_metric('num_sta'), { + value = radio_sta, + namespace = { 'wifi', 'hp', hw_platform, 'rd' .. band, 'num_sta' }, + }) - if globals.neighbor_reports then - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_neighbour_reports' }, - { - globals.neighbor_reports.dyn_report_num, - globals.neighbor_reports.disassoc_report_len - } - )) + if connected then + local sid = gen.gen_session_id() + sessions[mac] = sid + conn:publish(t_obs_event('session_start'), { + value = timestamp, + namespace = { 'wifi', 'clients', uid, 'sessions', sid, 'session_start' }, + }) + else + local sid = sessions[mac] + if sid then + conn:publish(t_obs_event('session_end'), { + value = timestamp, + namespace = { 'wifi', 'clients', uid, 'sessions', sid, 'session_end' }, + }) + sessions[mac] = nil + end end + end - if globals.legacy then - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_legacy_options' }, - { globals.legacy } - )) - end + local state_sub = conn:subscribe({ 'cap', 'radio', id, 'state', '+' }) + local event_sub = conn:subscribe({ 'cap', 'radio', id, 'event', '+' }) - -- Configure timing settings - local timing = band_configs.timings or {} - if timing.updates then - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_update_freq' }, - { timing.updates } - )) - end + fibers.current_scope():finally(function() + state_sub:unsubscribe() + event_sub:unsubscribe() + end) - if timing.inactive_client_kickoff then - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_client_inactive_kickoff' }, - { timing.inactive_client_kickoff } - )) - end + while true do + local which, msg = perform(fibers.named_choice({ + state = state_sub:recv_op(), + event = event_sub:recv_op(), + cancel = fibers.current_scope():cancel_op(), + })) + + if which == 'cancel' then break end - if timing.cleanup then - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_cleanup' }, - { timing.cleanup } - )) + if not msg then + svc:obs_log('debug', { what = 'radio_sub_closed', radio = id }) + break end - -- Configure networking settings - local networking = band_configs.networking or {} - if networking.method then - local network_options = { - ip = networking.ip, - port = networking.port, - broadcast_port = networking.broadcast_port, - enable_encryption = networking.enable_encryption - } - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_networking' }, - { networking.method, network_options } - )) + if which == 'state' then + on_radio_state(msg) + elseif which == 'event' then + on_radio_event(msg) end + end +end - -- Configure band-specific settings - local bands = band_configs.bands or {} - for band_name, band_cfg in pairs(bands) do - -- Set band priority - if band_cfg.initial_score then - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_band_priority' }, - { band_name, band_cfg.initial_score } - )) - end +------------------------------------------------------------------------ +-- Global num_sta aggregator fiber +------------------------------------------------------------------------ - -- Set RSSI scoring - if band_cfg.rssi_scoring then - local rssi_options = { - rssi_center = band_cfg.rssi_scoring.center, - rssi_weight = band_cfg.rssi_scoring.weight, - rssi_reward_threshold = band_cfg.rssi_scoring.good_threshold, - rssi_reward = band_cfg.rssi_scoring.good_reward, - rssi_penalty_threshold = band_cfg.rssi_scoring.bad_threshold, - rssi_penalty = band_cfg.rssi_scoring.bad_penalty - } - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_band_kicking' }, - { band_name, rssi_options } - )) - end +---Subscribes to all radio client_event emissions and maintains a single +---wifi/num_sta count across all radios. +---@param conn Connection +---@param parent_scope Scope +local function run_global_num_sta(conn, parent_scope) + local global_sta = 0 + local global_event_sub = conn:subscribe({ 'cap', 'radio', '+', 'event', 'client_event' }) - -- Set channel utilization scoring - if band_cfg.chan_util_scoring then - local channel_options = { - channel_util_reward_threshold = band_cfg.chan_util_scoring.good_threshold, - channel_util_reward = band_cfg.chan_util_scoring.good_reward, - channel_util_penalty_threshold = band_cfg.chan_util_scoring.bad_threshold, - channel_util_penalty = band_cfg.chan_util_scoring.bad_penalty - } - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_band_kicking' }, - { band_name, channel_options } - )) - end + fibers.current_scope():finally(function() + global_event_sub:unsubscribe() + end) - -- Set support bonuses - if band_cfg.support_bonuses then - for support_type, bonus in pairs(band_cfg.support_bonuses) do - reqs[#reqs + 1] = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'set_support_bonus' }, - { band_name, support_type, bonus } - )) - end - end + while true do + local which, msg = perform(fibers.named_choice({ + event = global_event_sub:recv_op(), + cancel = parent_scope:cancel_op(), + })) + + if which == 'cancel' then break end + + if msg and is_table(msg.payload) and msg.payload.mac then + local change = msg.payload.connected and 1 or -1 + global_sta = math.max(0, global_sta + change) + conn:retain(t_obs_metric('num_sta'), { value = global_sta, namespace = { 'wifi', 'num_sta' } }) end + end +end - fiber.spawn(function() - for _, req in ipairs(reqs) do - local resp, err = req:next_msg_with_context(ctx) - req:unsubscribe() - if err or (resp and resp.payload and resp.payload.err) then - log.error(string.format( - "%s - %s: Band steering config error: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - err or resp.payload.err - )) - end - end +------------------------------------------------------------------------ +-- Service context helpers +------------------------------------------------------------------------ - -- Apply band steering configuration - local req = conn:request(new_msg( - { 'hal', 'capability', 'band', '1', 'control', 'apply' } - )) - local resp, ctx_err = req:next_msg_with_context(ctx) - req:unsubscribe() - if ctx_err or resp and resp.payload and resp.payload.err then - log.error(string.format( - "%s - %s: Band steering apply error: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - ctx_err or resp.payload.err - )) - end - end) +local function get_radio_cfg(ctx, radio_id) + if not is_table(ctx.data) or not is_table(ctx.data.radios) then return nil end + for _, r in ipairs(ctx.data.radios) do + if r.name == radio_id then return r end end + return nil +end - local function apply_config(config) - report_period = config.report_period - for _, radio_cfg in ipairs(config.radios) do - local radio = radios[radio_cfg.name] - if radio then - radio:apply_config(radio_cfg, report_period) - end - radio_configs[radio_cfg.name] = radio_cfg - end +local function configure_radio(ctx, id) + local radio_cfg = get_radio_cfg(ctx, id) + if not radio_cfg then + ctx.svc:obs_log('debug', { what = 'no_config_for_radio', radio = id }) + return + end + local cap = cap_sdk.new_cap_ref(ctx.conn, 'radio', id) + local ssids = is_table(ctx.data.ssids) and ctx.data.ssids or {} + apply_radio_config(cap, radio_cfg, ssids, ctx.fs_configs_cap, + is_table(ctx.data) and ctx.data.band_steering or nil, ctx.svc) +end - local radio_ssids = {} - for _, ssid_cfg in ipairs(config.ssids) do - local ssids, err - ssid_cfg.has_band_steering = config.band_steering.globals.kicking.kick_mode ~= "none" - if ssid_cfg.mainflux_path then - local base_cfg = {} - for k, v in pairs(ssid_cfg) do - if k ~= 'mainflux_path' then - base_cfg[k] = v - end - end - ssids, err = utils.parse_mainflux_ssids(ctx, conn, ssid_cfg.mainflux_path, base_cfg) - if err then - log.error(string.format( - "%s - %s: SSID %s mainflux config error: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - ssid_cfg.name or "(unknown)", - err - )) - ssids = {} - end - end - -- a mainflux path may spawn multiple ssids from one ssid_cfg - for _, ssid in ipairs(ssids or { ssid_cfg }) do - for _, radio_id in ipairs(ssid.radios) do - if radio_ssids[radio_id] then - table.insert(radio_ssids[radio_id], ssid) - else - radio_ssids[radio_id] = { ssid } - end - end - end - end - ssid_configs = radio_ssids - for radio_id, ssids in pairs(ssid_configs) do - local radio = radios[radio_id] - if radio then - radio:remove_ssids() - radio:apply_ssids(ssids) - end - end +local function radio_fiber_body(ctx, id) + configure_radio(ctx, id) + radio_stats_loop(ctx.conn, id, get_radio_cfg(ctx, id), ctx.svc) +end - -- Configure band steering if it exists - if config.band_steering then - apply_band_steering_config(ctx, conn, config.band_steering) - end +local function spawn_radio_scope(ctx, id) + if ctx.radio_scopes[id] then + ctx.radio_scopes[id]:cancel('reconfigure') + ctx.radio_scopes[id] = nil end + local scope, serr = ctx.parent_scope:child() + if not scope then + ctx.svc:obs_log('error', { what = 'radio_scope_failed', radio = id, err = serr }) + return + end + ctx.radio_scopes[id] = scope + scope:spawn(function() radio_fiber_body(ctx, id) end) +end - local function handle_config(msg) - log.trace(string.format( - "%s - %s: Config Received", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - local config = msg and msg.payload or nil - local err = validate_config(config) - if err then - log.error(string.format( - "%s - %s: Config validation error: %s", - ctx:value("service_name"), - ctx:value("fiber_name"), - err - )) - return - end - apply_config(config) +local function remove_radio(ctx, id) + if ctx.radio_scopes[id] then + ctx.radio_scopes[id]:cancel('radio removed') + ctx.radio_scopes[id] = nil end +end - local num_sta = 0 - local function handle_client(msg) - if not msg or not msg.payload then return end - local connected = msg.payload.connected - local sta_change = connected and 1 or -1 - num_sta = num_sta + sta_change - conn:publish(new_msg( - { 'wifi', 'num_sta' }, - num_sta - )) +local function apply_band_config(ctx) + if ctx.band_cap and is_table(ctx.data) and is_table(ctx.data.band_steering) then + apply_band_steering(ctx.band_cap, ctx.data.band_steering, ctx.svc) end +end +local function reapply_all_radios(ctx) + for id in pairs(ctx.radio_scopes) do + spawn_radio_scope(ctx, id) + end +end - local band_sub = conn:subscribe({ 'hal', 'capability', 'band', '+' }) - band_sub:next_msg_with_context(ctx) -- wait for band driver to be initialised - band_sub:unsubscribe() - - log.trace(string.format( - "%s - %s: Started", - ctx:value("service_name"), - ctx:value("fiber_name") - )) - - local config_sub = conn:subscribe({ 'config', 'wifi' }) - local client_sub = conn:subscribe({ 'hal', 'capability', 'wireless', '+', 'info', 'interface', '+', 'client', '+' }) - while not ctx:err() do - op.choice( - wifi_service.radio_add_queue:get_op():wrap(add_radio), - wifi_service.radio_remove_queue:get_op():wrap(remove_radio), - wifi_service.config_queue:get_op():wrap(handle_config), - config_sub:next_msg_op():wrap(handle_config), - client_sub:next_msg_op():wrap(handle_client), - ctx:done_op() - ):perform() +------------------------------------------------------------------------ +-- Main service loop: message handlers +------------------------------------------------------------------------ + +local function on_cfg(ctx, msg) + local payload = msg.payload + if not is_table(payload) then + ctx.svc:obs_log('warn', { what = 'invalid_config_payload' }) + return end - config_sub:unsubscribe() + local rev = payload.rev + local data = payload.data + if type(rev) == 'number' and rev <= ctx.last_rev then + ctx.svc:obs_log('debug', { what = 'stale_config', rev = rev, last_rev = ctx.last_rev }) + elseif not is_table(data) then + ctx.svc:obs_log('warn', { what = 'config_data_not_table' }) + else + ctx.last_rev = rev or ctx.last_rev + ctx.data = data + ctx.svc:obs_event('config_applied', { rev = rev }) + ctx.svc:obs_log('info', { + what = 'wifi_config_applied', + summary = string.format('wifi config applied radios=%d ssids=%d', count_array(data.radios), count_array(data.ssids)), + rev = rev, + radios = count_array(data.radios), + ssids = count_array(data.ssids), + }) + reapply_all_radios(ctx) + apply_band_config(ctx) + end +end - log.trace(string.format( - "%s - %s: Closed", - ctx:value("service_name"), - ctx:value("fiber_name") - )) +local function on_radio_cap(ctx, msg) + local id = msg.topic and msg.topic[3] + local state = msg.payload + if state == 'added' then + ctx.svc:obs_log('debug', { what = 'radio_cap_added', radio = id }) + spawn_radio_scope(ctx, id) + elseif state == 'removed' then + ctx.svc:obs_log('debug', { what = 'radio_cap_removed', radio = id }) + remove_radio(ctx, id) + end end -function wifi_service:start(ctx, conn) - log.trace(string.format( - "%s - %s: Starting", - ctx:value("service_name"), - ctx:value("fiber_name") - )) +local function on_band_cap(ctx, msg) + local state = msg.payload + if state == 'added' then + ctx.svc:obs_log('debug', { what = 'band_cap_added' }) + ctx.band_cap = cap_sdk.new_cap_ref(ctx.conn, 'band', '1') + apply_band_config(ctx) + elseif state == 'removed' then + ctx.svc:obs_log('debug', { what = 'band_cap_removed' }) + ctx.band_cap = nil + end +end - service.spawn_fiber('Radio Listener', conn, ctx, function(fctx) - radio_listener(fctx, conn) - end) - service.spawn_fiber('Radio Manager', conn, ctx, function(fctx) - radio_manager(fctx, conn) - end) +local function on_fs_cap(ctx, msg) + local state = msg.payload + if state == 'added' then + ctx.svc:obs_log('debug', { what = 'fs_configs_cap_added' }) + ctx.fs_configs_cap = cap_sdk.new_cap_ref(ctx.conn, 'fs', 'credentials') + reapply_all_radios(ctx) + elseif state == 'removed' then + ctx.svc:obs_log('debug', { what = 'fs_configs_cap_removed' }) + ctx.fs_configs_cap = nil + end end -if _G._TEST then - return { - wifi_service = wifi_service, - new_radio = Radio.new, +------------------------------------------------------------------------ +-- Service entry point +------------------------------------------------------------------------ + +local WifiService = {} + +---@param conn Connection +---@param opts? WifiServiceOpts +function WifiService.start(conn, opts) + local svc = base.new(conn, { name = opts and opts.name or 'wifi', env = opts and opts.env }) + + svc:obs_state('boot', { at = svc:wall(), ts = svc:now(), state = 'entered' }) + svc:obs_log('debug', 'service start() entered') + svc:status('starting') + svc:spawn_heartbeat(10, 'tick') + + local parent_scope = fibers.current_scope() + + parent_scope:finally(function() + local scope = fibers.current_scope() + local st, primary = scope:status() + if st == 'failed' then + svc:obs_log('error', { what = 'scope_failed', err = tostring(primary) }) + end + svc:status('stopped', primary and { reason = tostring(primary) } or nil) + svc:obs_log('debug', 'service stopped') + end) + + local ctx = { + conn = conn, + svc = svc, + parent_scope = parent_scope, + data = nil, -- data field of last valid config + last_rev = -1, + fs_configs_cap = nil, -- configs filesystem cap reference + band_cap = nil, -- band capability reference + radio_scopes = {}, -- radio_id → child scope } -else - return wifi_service + + -- Subscribe to config topic + local cfg_sub = conn:subscribe({ 'cfg', 'wifi' }) + + -- Subscribe to cap state notifications (radio, band, fs/configs) + local radio_cap_listener = cap_sdk.new_cap_listener(conn, 'radio', '+') + local band_cap_listener = cap_sdk.new_cap_listener(conn, 'band', '1') + local fs_cap_listener = cap_sdk.new_cap_listener(conn, 'fs', 'credentials') + + svc:status('running') + svc:obs_log('debug', 'service running') + + parent_scope:spawn(function() run_global_num_sta(conn, parent_scope) end) + + while true do + local which, msg = perform(fibers.named_choice({ + cfg = cfg_sub:recv_op(), + radio = radio_cap_listener.sub:recv_op(), + band = band_cap_listener.sub:recv_op(), + fs = fs_cap_listener.sub:recv_op(), + cancel = parent_scope:cancel_op(), + })) + + if which == 'cancel' then break end + + if not msg then + svc:obs_log('debug', { what = 'subscription_closed', source = which }) + elseif which == 'cfg' then + on_cfg(ctx, msg) + elseif which == 'radio' then + on_radio_cap(ctx, msg) + elseif which == 'band' then + on_band_cap(ctx, msg) + elseif which == 'fs' then + on_fs_cap(ctx, msg) + end + end + + -- Cleanup all radio scopes on exit + for id, scope in pairs(ctx.radio_scopes) do + scope:cancel('service stopping') + ctx.radio_scopes[id] = nil + end + + cfg_sub:unsubscribe() + radio_cap_listener:close() + band_cap_listener:close() + fs_cap_listener:close() end + +return WifiService diff --git a/src/services/wifi/utils.lua b/src/services/wifi/utils.lua index d9f67ec7..36be7d3b 100644 --- a/src/services/wifi/utils.lua +++ b/src/services/wifi/utils.lua @@ -1,29 +1,54 @@ local json = require "cjson.safe" - -local BUS_TIMEOUT = 2 +local cap_args = require "services.hal.types.capability_args" local mainflux_to_ssid_keys = { - name = "network", - ssid = "name" + name = "segment", + ssid = "name", } -local function parse_mainflux_ssids(ctx, conn, mainflux_path, base_ssid_cfg) - local mainflux_topic = {} - for token in string.gmatch(mainflux_path, "[^/]+") do - table.insert(mainflux_topic, token) +---Read mainflux SSID credentials from the configs filesystem capability +---and merge them with the base SSID config. +---@param fs_cap any CapabilityReference for the configs filesystem cap +---@param mainflux_path string Filename within the filesystem cap (e.g. "mainflux.json") +---@param base_ssid_cfg table Base SSID fields to merge into each resulting SSID +---@return table? ssids Array of SSID config tables, or nil on any error +---@return string err Error message, or "" on success +local function parse_mainflux_ssids(fs_cap, mainflux_path, base_ssid_cfg) + local opts, opts_err = cap_args.new.FilesystemReadOpts(mainflux_path) + if not opts then + return nil, tostring(opts_err) + end + + local reply, call_err = fs_cap:call_control('read', opts) + if not reply then + return nil, tostring(call_err) end - local mainflux_sub = conn:subscribe(mainflux_topic) - local mainflux_msg, err = mainflux_sub:next_msg_with_context(ctx, BUS_TIMEOUT) - if err then return nil, err end - local content = mainflux_msg.payload and mainflux_msg.payload.content or nil - if not content then - return nil, "no content field found" + if reply.ok ~= true then + return nil, tostring(reply.reason or "filesystem read failed") + end + + local outer, derr = json.decode(reply.reason or '') + if not outer then + return nil, tostring(derr) end - local content_parsed, err = json.decode(content) - if err then return nil, err end - local ssid_cfgs = content_parsed.networks and content_parsed.networks.networks or nil - if (not ssid_cfgs) then - return nil, "No ssid configs found" + + -- The mainflux file wraps the actual config as a JSON-encoded string in `content` + local content_parsed + if type(outer.content) == 'string' then + local inner_err + content_parsed, inner_err = json.decode(outer.content) + if not content_parsed then + return nil, "failed to decode mainflux content field: " .. tostring(inner_err) + end + else + content_parsed = outer + end + + local ssid_cfgs = content_parsed.networks + and content_parsed.networks.networks + or nil + if not ssid_cfgs then + return nil, "no ssid configs found in mainflux file" end local ssids = {} @@ -34,17 +59,17 @@ local function parse_mainflux_ssids(ctx, conn, mainflux_path, base_ssid_cfg) end for k, v in pairs(ssid_cfg) do local key = mainflux_to_ssid_keys[k] or k - -- This is a temporary workaround until we fix our mainflux naming - if key == "network" and v == "jng" then + -- Temporary workaround: mainflux uses "jng" for what we call "adm" + if key == "segment" and v == "jng" then v = "adm" end ssid[key] = v end table.insert(ssids, ssid) end - return ssids + return ssids, "" end return { - parse_mainflux_ssids = parse_mainflux_ssids + parse_mainflux_ssids = parse_mainflux_ssids, } diff --git a/src/services/wired.lua b/src/services/wired.lua new file mode 100644 index 00000000..c1b694cb --- /dev/null +++ b/src/services/wired.lua @@ -0,0 +1,25 @@ +-- services/wired.lua +-- +-- Public entry point for the Wired service. + +local fibers = require 'fibers' +local service_base = require 'devicecode.service_base' +local service = require 'services.wired.service' + +local M = {} + +function M.start(conn, opts) + opts = opts or {} + opts.conn = conn + local svc = service_base.new(conn, { name = opts.name or 'wired', env = opts.env, meta = opts.meta, announce = opts.announce }) + svc:starting({ ready = false }) + opts.svc = svc + local scope = fibers.current_scope() + scope:finally(function (_, status, primary) + if status == 'failed' then svc:failed(primary or 'wired_failed') else svc:stopped({ reason = primary or status or 'wired_stopped' }) end + end) + svc:running({ ready = false }) + return service.run(scope, opts) +end + +return M diff --git a/src/services/wired/config.lua b/src/services/wired/config.lua new file mode 100644 index 00000000..d45499b4 --- /dev/null +++ b/src/services/wired/config.lua @@ -0,0 +1,228 @@ +-- services/wired/config.lua +-- Strict cfg/wired normalisation boundary. +-- +-- Wired owns appliance-level wired surfaces and their attachment to net +-- segments. It does not own segment identity, VLAN allocation, routing, +-- firewall or switch-driver implementation. + +local tablex = require 'shared.table' + +local M = {} + +M.SCHEMA = 'devicecode.config/wired/1' +M.INTENT_SCHEMA = 'devicecode.wired.intent/1' +M.DEFAULT_VERSION = 1 + +local ID_PATTERN = '^[%w][%w%._%-]*$' + +local function copy(v) return tablex.deep_copy(v) end +local function is_plain_table(v) return type(v) == 'table' and getmetatable(v) == nil end + +local function path(p) + if type(p) == 'table' then + local out = {} + for i = 1, #p do out[i] = tostring(p[i]) end + return table.concat(out, '.') + end + return tostring(p or 'value') +end + +local function err(p, msg) + return ('%s: %s'):format(path(p), tostring(msg)) +end + +local function id(v, p) + if type(v) ~= 'string' or v == '' then return nil, err(p, 'must be a non-empty string') end + if not v:match(ID_PATTERN) then + return nil, err(p, 'must contain only letters, digits, underscore, hyphen or dot, and must start with a word character') + end + return v, nil +end + +local function string_list(v, p) + if v == nil then return {}, nil end + if type(v) ~= 'table' or not tablex.is_array(v) then return nil, err(p, 'must be an array of strings') end + local out, seen = {}, {} + for i = 1, #v do + local item, ierr = id(v[i], { path(p), i }) + if not item then return nil, ierr end + if not seen[item] then + seen[item] = true + out[#out + 1] = item + end + end + return out, nil +end + +local function optional_string(v, p) + if v == nil then return nil, nil end + if type(v) ~= 'string' then return nil, err(p, 'must be a string') end + return v, nil +end + +local function check_allowed(t, allowed, p) + if not is_plain_table(t) then return nil, err(p, 'must be a plain table') end + local ok = {} + for i = 1, #allowed do ok[allowed[i]] = true end + for k in pairs(t) do + if not ok[k] then return nil, err({ path(p), k }, 'field is not part of devicecode.config/wired/1') end + end + return true, nil +end + +local ATTACHMENT_FIELDS = { + 'mode', 'segment', 'segments', 'required_segments', 'user_segments', 'native_segment', + 'tagged', 'untagged', 'metadata', 'extensions', +} + +local function normalise_attachment(v, p) + if not is_plain_table(v) then return nil, err(p, 'attachment must be a table') end + local ok, ferr = check_allowed(v, ATTACHMENT_FIELDS, p) + if not ok then return nil, ferr end + local mode = v.mode + if mode == nil then return nil, err({ path(p), 'mode' }, 'is required') end + if mode ~= 'access' and mode ~= 'trunk' and mode ~= 'none' then + return nil, err({ path(p), 'mode' }, 'must be access, trunk or none') + end + local segment = nil + if v.segment ~= nil then + local serr + segment, serr = id(v.segment, { path(p), 'segment' }) + if not segment then return nil, serr end + end + local segments, sgerr = string_list(v.segments, { path(p), 'segments' }) + if not segments then return nil, sgerr end + local required, rerr = string_list(v.required_segments, { path(p), 'required_segments' }) + if not required then return nil, rerr end + local native = nil + if v.native_segment ~= nil then + local nerr + native, nerr = id(v.native_segment, { path(p), 'native_segment' }) + if not native then return nil, nerr end + end + if mode == 'access' and not segment then + return nil, err({ path(p), 'segment' }, 'is required for access attachments') + end + if mode == 'trunk' and segment ~= nil then + return nil, err({ path(p), 'segment' }, 'must not be used for trunk attachments') + end + if v.user_segments ~= nil and v.user_segments ~= 'all-realised-user-segments' and type(v.user_segments) ~= 'table' then + return nil, err({ path(p), 'user_segments' }, 'must be all-realised-user-segments or an array of segment ids') + end + local user_segments = v.user_segments + if type(user_segments) == 'table' then + local uerr + user_segments, uerr = string_list(user_segments, { path(p), 'user_segments' }) + if not user_segments then return nil, uerr end + end + return { + mode = mode, + segment = segment, + segments = segments, + required_segments = required, + user_segments = user_segments, + native_segment = native, + tagged = copy(v.tagged), + untagged = copy(v.untagged), + metadata = copy(v.metadata), + extensions = copy(v.extensions), + }, nil +end + +local SURFACE_FIELDS = { + 'id', 'name', 'description', 'kind', 'role', 'enabled', 'protected', + 'attachment', 'capabilities', 'tags', 'metadata', 'extensions', +} + +local function normalise_surface(surface_id, rec, p) + if not is_plain_table(rec) then return nil, err(p, 'surface must be a table') end + local ok, ferr = check_allowed(rec, SURFACE_FIELDS, p) + if not ok then return nil, ferr end + if rec.id ~= nil and rec.id ~= surface_id then return nil, err({ path(p), 'id' }, 'must match the map key') end + if rec.attachment == nil then return nil, err({ path(p), 'attachment' }, 'is required') end + local attachment, aerr = normalise_attachment(rec.attachment, { path(p), 'attachment' }) + if not attachment then return nil, aerr end + local protected = rec.protected == true + local enabled = rec.enabled ~= false + if protected then + if not enabled then + return nil, err({ path(p), 'enabled' }, 'protected surfaces cannot be disabled') + end + if attachment.mode ~= 'trunk' then + return nil, err({ path(p), 'attachment', 'mode' }, 'protected surfaces must be trunk attachments') + end + if #(attachment.required_segments or {}) == 0 then + return nil, err({ path(p), 'attachment', 'required_segments' }, 'protected trunks must declare required system segments') + end + end + local tags, terr = string_list(rec.tags, { path(p), 'tags' }) + if not tags then return nil, terr end + local name, nerr = optional_string(rec.name, { path(p), 'name' }) + if nerr then return nil, nerr end + return { + id = surface_id, + surface_id = surface_id, + name = name or surface_id, + description = rec.description, + kind = rec.kind or 'ethernet-port', + role = rec.role or 'access', + enabled = enabled, + protected = protected, + attachment = attachment, + capabilities = copy(rec.capabilities or {}), + tags = tags, + metadata = copy(rec.metadata), + extensions = copy(rec.extensions), + }, nil +end + +local function normalise_surfaces(v) + if v == nil then return {}, nil end + if not is_plain_table(v) or tablex.is_array(v) then return nil, err({ 'wired', 'surfaces' }, 'must be a map keyed by surface id') end + local out = {} + local keys = tablex.sorted_keys(v) + for i = 1, #keys do + local surface_id, ierr = id(tostring(keys[i]), { 'wired', 'surfaces', keys[i] }) + if not surface_id then return nil, ierr end + local rec, rerr = normalise_surface(surface_id, v[keys[i]], { 'wired', 'surfaces', surface_id }) + if not rec then return nil, rerr end + out[surface_id] = rec + end + return out, nil +end + +local TOP_LEVEL = { 'schema', 'version', 'product', 'description', 'surfaces', 'policies', 'runtime', 'metadata', 'extensions' } + +function M.normalise(raw, opts) + opts = opts or {} + if raw ~= nil and raw.data ~= nil then raw = raw.data end + if raw == nil then raw = { schema = M.SCHEMA, version = M.DEFAULT_VERSION, surfaces = {} } end + if not is_plain_table(raw) then return nil, 'cfg/wired must be a table' end + if raw.schema ~= M.SCHEMA then return nil, ('cfg/wired schema must be %s'):format(M.SCHEMA) end + local ok, ferr = check_allowed(raw, TOP_LEVEL, { 'cfg', 'wired' }) + if not ok then return nil, ferr end + local version = raw.version or M.DEFAULT_VERSION + if type(version) ~= 'number' or version % 1 ~= 0 or version < 1 then return nil, 'cfg/wired.version must be a positive integer' end + local surfaces, serr = normalise_surfaces(raw.surfaces) + if not surfaces then return nil, serr end + return { + schema = M.INTENT_SCHEMA, + config_schema = M.SCHEMA, + version = version, + rev = opts.rev or raw.rev or 0, + generation = opts.generation or 0, + product = raw.product, + description = raw.description, + surfaces = surfaces, + policies = copy(raw.policies or {}), + runtime = copy(raw.runtime or {}), + metadata = copy(raw.metadata), + extensions = copy(raw.extensions), + }, nil +end + +M._test = { + normalise_attachment = normalise_attachment, +} + +return M diff --git a/src/services/wired/dependencies.lua b/src/services/wired/dependencies.lua new file mode 100644 index 00000000..d9f1dc52 --- /dev/null +++ b/src/services/wired/dependencies.lua @@ -0,0 +1,11 @@ +-- services/wired/dependencies.lua +-- Wired does not derive capability dependencies from cfg/wired. Physical +-- backing is resolved through Device assembly plus raw wired observations. + +local M = {} + +function M.observation_dependencies(_intent) + return {} +end + +return M diff --git a/src/services/wired/model.lua b/src/services/wired/model.lua new file mode 100644 index 00000000..0e661756 --- /dev/null +++ b/src/services/wired/model.lua @@ -0,0 +1,39 @@ +-- services/wired/model.lua +-- Observable Wired service model. + +local support_model = require 'devicecode.support.model' +local tablex = require 'shared.table' + +local M = {} + +local function copy(v) return tablex.deep_copy(v) end + +function M.initial(service_id) + return { + service = 'wired', + service_id = service_id, + state = 'starting', + ready = false, + reason = nil, + generation = 0, + config = { rev = nil, schema = nil, config_schema = nil, version = nil }, + net = { segments_rev = nil, segments = {}, vlan_policy = {}, missing_segments = {} }, + observations = {}, + assembly = {}, + dependencies = {}, + surfaces = {}, + counters = {}, + topology = { protected_trunks = {}, access = {}, trunks = {} }, + violations = {}, + stats = { config_updates = 0, segment_updates = 0, observation_updates = 0, assembly_updates = 0, publications = 0 }, + } +end + +function M.new(service_id, opts) + opts = opts or {} + return support_model.new(M.initial(service_id), { label = opts.label or 'wired.model', copy = copy }) +end + +function M.deep_copy(v) return copy(v) end + +return M diff --git a/src/services/wired/projection.lua b/src/services/wired/projection.lua new file mode 100644 index 00000000..14646f3c --- /dev/null +++ b/src/services/wired/projection.lua @@ -0,0 +1,41 @@ +-- services/wired/projection.lua +-- Public state projection for Wired. + +local tablex = require 'shared.table' +local topics = require 'services.wired.topics' + +local M = {} + +local function copy(v) return tablex.deep_copy(v) end +local function count_map(t) local n = 0; for _ in pairs(t or {}) do n = n + 1 end; return n end + +function M.summary_topic() return topics.summary() end +function M.surface_topic(id) return topics.surface(id) end +function M.topology_topic() return topics.topology() end +function M.violations_topic() return topics.violations() end +function M.surface_counters_topic(id) return topics.surface_counters(id) end + +function M.summary(snapshot) + snapshot = snapshot or {} + return { + service = 'wired', + state = snapshot.state, + ready = snapshot.ready == true, + reason = snapshot.reason, + generation = snapshot.generation, + config = copy(snapshot.config), + counts = { + surfaces = count_map(snapshot.surfaces), + sources = count_map(snapshot.observations), + violations = #(snapshot.violations or {}), + }, + stats = copy(snapshot.stats), + } +end + +function M.surface(rec) return copy(rec or {}) end +function M.surface_counters(rec) return copy(rec or {}) end +function M.topology(snapshot) return copy((snapshot or {}).topology or {}) end +function M.violations(snapshot) return copy((snapshot or {}).violations or {}) end + +return M diff --git a/src/services/wired/publisher.lua b/src/services/wired/publisher.lua new file mode 100644 index 00000000..94d131c2 --- /dev/null +++ b/src/services/wired/publisher.lua @@ -0,0 +1,171 @@ +-- services/wired/publisher.lua +-- Immediate retained-state publication for Wired. + +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local retained_publish = require 'devicecode.support.retained_publish' +local projection = require 'services.wired.projection' + +local M = {} + +local function new_dirty_surfaces() + return { all = false, ids = {} } +end + +local function ensure_dirty(dirty) + dirty = dirty or M.new_dirty_state() + dirty.surfaces = dirty.surfaces or new_dirty_surfaces() + dirty.surfaces.ids = dirty.surfaces.ids or {} + dirty.counters = dirty.counters or new_dirty_surfaces() + dirty.counters.ids = dirty.counters.ids or {} + return dirty +end + +function M.new_state() + return { surfaces = {}, counters = {}, summary = nil, topology = nil, violations = nil } +end + +function M.new_dirty_state() + return { all = false, surfaces = new_dirty_surfaces(), counters = new_dirty_surfaces(), summary = false, topology = false, violations = false } +end + +function M.mark_all(dirty) + dirty = ensure_dirty(dirty) + dirty.all = true + dirty.surfaces.all = true + dirty.counters.all = true + dirty.summary = true + dirty.topology = true + dirty.violations = true + return dirty +end + +function M.mark_surface(dirty, id) + dirty = ensure_dirty(dirty) + if id ~= nil then dirty.surfaces.ids[id] = true end + return dirty +end + +function M.mark_counter(dirty, id) + dirty = ensure_dirty(dirty) + if id ~= nil then dirty.counters.ids[id] = true end + return dirty +end + +function M.mark_summary(dirty) dirty = ensure_dirty(dirty); dirty.summary = true; return dirty end +function M.mark_topology(dirty) dirty = ensure_dirty(dirty); dirty.topology = true; return dirty end +function M.mark_violations(dirty) dirty = ensure_dirty(dirty); dirty.violations = true; return dirty end + +local function clear_dirty(dirty) + if not dirty then return M.new_dirty_state() end + dirty.all = false + dirty.summary = false + dirty.topology = false + dirty.violations = false + dirty.surfaces = new_dirty_surfaces() + dirty.counters = new_dirty_surfaces() + return dirty +end + +local function retain_one(conn, published, key, topic, payload) + local ok, err, changed = retained_publish.retain_if_changed(conn, published, key, topic, payload) + if ok ~= true then return nil, err, changed end + return true, nil, changed +end + +local function publish_all_surfaces(conn, published, surfaces) + return retained_publish.publish_map_changed(conn, published.surfaces, surfaces, projection.surface_topic, projection.surface) +end + +local function publish_all_counters(conn, published, counters) + return retained_publish.publish_map_changed(conn, published.counters, counters, projection.surface_counters_topic, projection.surface_counters) +end + +local function publish_dirty_surfaces(conn, published, surfaces, dirty_surfaces) + dirty_surfaces = dirty_surfaces or new_dirty_surfaces() + if dirty_surfaces.all then return publish_all_surfaces(conn, published, surfaces) end + local changed = 0 + for id in pairs(dirty_surfaces.ids or {}) do + local rec = surfaces and surfaces[id] or nil + if rec == nil then + local ok, err, did = retained_publish.unretain_if_present(conn, published.surfaces, id, projection.surface_topic(id)) + if ok ~= true then return nil, err, changed end + if did then changed = changed + 1 end + else + local ok, err, did = retained_publish.retain_if_changed(conn, published.surfaces, id, projection.surface_topic(id), projection.surface(rec)) + if ok ~= true then return nil, err, changed end + if did then changed = changed + 1 end + end + end + return true, nil, changed +end + +local function publish_dirty_counters(conn, published, counters, dirty_counters) + dirty_counters = dirty_counters or new_dirty_surfaces() + if dirty_counters.all then return publish_all_counters(conn, published, counters) end + local changed = 0 + for id in pairs(dirty_counters.ids or {}) do + local rec = counters and counters[id] or nil + if rec == nil then + local ok, err, did = retained_publish.unretain_if_present(conn, published.counters, id, projection.surface_counters_topic(id)) + if ok ~= true then return nil, err, changed end + if did then changed = changed + 1 end + else + local ok, err, did = retained_publish.retain_if_changed(conn, published.counters, id, projection.surface_counters_topic(id), projection.surface_counters(rec)) + if ok ~= true then return nil, err, changed end + if did then changed = changed + 1 end + end + end + return true, nil, changed +end + +function M.publish_dirty_now(conn, snapshot, published, dirty) + published = published or M.new_state() + dirty = ensure_dirty(dirty) + local changed = 0 + local ok, err, did + if dirty.all or dirty.summary then + ok, err, did = retain_one(conn, published, 'summary', projection.summary_topic(), projection.summary(snapshot)) + if ok ~= true then return nil, err, changed end + if did then changed = changed + 1 end + end + if dirty.all or dirty.surfaces.all or next(dirty.surfaces.ids or {}) ~= nil then + ok, err, did = publish_dirty_surfaces(conn, published, snapshot.surfaces, dirty.surfaces) + if ok ~= true then return nil, err, changed end + changed = changed + (did or 0) + end + if dirty.all or dirty.counters.all or next(dirty.counters.ids or {}) ~= nil then + ok, err, did = publish_dirty_counters(conn, published, snapshot.counters, dirty.counters) + if ok ~= true then return nil, err, changed end + changed = changed + (did or 0) + end + if dirty.all or dirty.topology then + ok, err, did = retain_one(conn, published, 'topology', projection.topology_topic(), projection.topology(snapshot)) + if ok ~= true then return nil, err, changed end + if did then changed = changed + 1 end + end + if dirty.all or dirty.violations then + ok, err, did = retain_one(conn, published, 'violations', projection.violations_topic(), projection.violations(snapshot)) + if ok ~= true then return nil, err, changed end + if did then changed = changed + 1 end + end + clear_dirty(dirty) + return true, nil, changed +end + +function M.publish_all_now(conn, snapshot, published) + return M.publish_dirty_now(conn, snapshot, published, M.mark_all(M.new_dirty_state())) +end + +function M.cleanup_now(conn, published) + if not published then return true, nil end + for id in pairs(published.surfaces or {}) do bus_cleanup.unretain(conn, projection.surface_topic(id)); published.surfaces[id] = nil end + for id in pairs(published.counters or {}) do bus_cleanup.unretain(conn, projection.surface_counters_topic(id)); published.counters[id] = nil end + if published.summary ~= nil then bus_cleanup.unretain(conn, projection.summary_topic()) end + if published.topology ~= nil then bus_cleanup.unretain(conn, projection.topology_topic()) end + if published.violations ~= nil then bus_cleanup.unretain(conn, projection.violations_topic()) end + published.summary, published.topology, published.violations = nil, nil, nil + return true, nil +end + +M.clear_dirty = clear_dirty +return M diff --git a/src/services/wired/service.lua b/src/services/wired/service.lua new file mode 100644 index 00000000..b473068c --- /dev/null +++ b/src/services/wired/service.lua @@ -0,0 +1,1029 @@ +-- services/wired/service.lua +-- +-- Minimal house-style Wired service. +-- The coordinator owns no OS-facing work. It composes configured appliance +-- surfaces from net segment state, Device physical assembly and raw wired observations. + +local fibers = require 'fibers' +local runtime = require 'fibers.runtime' +local config_mod = require 'services.wired.config' +local model_mod = require 'services.wired.model' +local topics = require 'services.wired.topics' +local publisher = require 'services.wired.publisher' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local config_watch = require 'devicecode.support.config_watch' +local tablex = require 'shared.table' + +local M = {} + +local OPERATOR_SETTLE_S = 12 +local COUNTER_FIRST_THRESHOLD = 25 +local COUNTER_REPEAT_S = 60 + +local function now() return runtime.now() end +local function copy(v) return tablex.deep_copy(v) end +local function is_plain_table(v) return type(v) == 'table' and getmetatable(v) == nil end + +local function count_map(t) + local n = 0 + for _ in pairs(t or {}) do n = n + 1 end + return n +end + +local function count_where(t, pred) + local n = 0 + for k, v in pairs(t or {}) do if pred(k, v) then n = n + 1 end end + return n +end + +local function surface_label(id, rec) + return tostring((rec and (rec.operator_label or rec.name)) or id) +end + +local function operator_settling(state) + local started = state and state.operator_started_at or nil + if not started then return false end + return (now() - started) < (state.operator_settle_s or OPERATOR_SETTLE_S) +end + +local function link_status(link) + link = link or {} + local v = link.state or link.status + if v == nil and link.carrier ~= nil then v = link.carrier end + if v == true or v == 'up' or v == 'online' or v == 'available' then return 'up' end + if v == false or v == 'down' or v == 'offline' or v == 'unavailable' then return 'down' end + return nil +end + +local function link_is_up(link) + return link_status(link) == 'up' +end + +local function link_speed(link) + link = link or {} + return link.speed or link.speed_mbps +end + +local function link_summary(link) + link = link or {} + local parts = {} + local speed = link_speed(link) + if speed then parts[#parts + 1] = 'speed=' .. tostring(speed) end + if link.duplex then parts[#parts + 1] = 'duplex=' .. tostring(link.duplex) end + return table.concat(parts, ' ') +end + +local function segment_summary(rec) + local parts = {} + if rec and rec.segment_id then parts[#parts + 1] = 'segment=' .. tostring(rec.segment_id) end + if rec and rec.vlan_id then parts[#parts + 1] = 'vlan=' .. tostring(rec.vlan_id) end + return table.concat(parts, ' ') +end + +local function has_observed_surfaces(surfaces) + for _, rec in pairs(surfaces or {}) do + if rec and rec.source and rec.source.exposure then return true end + end + return false +end + +local function has_observed_link_state(surfaces) + for _, rec in pairs(surfaces or {}) do + if link_status(rec and rec.link or {}) ~= nil then return true end + end + return false +end + +local function count_link_states(surfaces) + local up, down, unknown = 0, 0, 0 + for _, rec in pairs(surfaces or {}) do + local st = link_status(rec and rec.link or {}) + if st == 'up' then up = up + 1 + elseif st == 'down' then down = down + 1 + else unknown = unknown + 1 end + end + return up, down, unknown +end + + +local function link_brief(rec) + if not rec then return 'unknown' end + local st = link_status(rec.link or {}) or 'unknown' + local speed = link_speed(rec.link or {}) + local duplex = rec.link and rec.link.duplex or nil + local parts = { st } + if speed then parts[#parts + 1] = tostring(speed) end + if duplex then parts[#parts + 1] = tostring(duplex) end + return table.concat(parts, '/') +end + +local function choose_wan_surface(surfaces) + for id, rec in pairs(surfaces or {}) do + if rec and rec.role == 'wan-uplink' then return id, rec end + end + return nil, nil +end + +local function choose_trunk_surface(surfaces) + for id, rec in pairs(surfaces or {}) do + if rec and rec.protected == true and link_status(rec.link or {}) ~= nil then return id, rec end + end + for id, rec in pairs(surfaces or {}) do + if rec and rec.protected == true and rec.source and rec.source.exposure == 'internal' then return id, rec end + end + for id, rec in pairs(surfaces or {}) do + if rec and rec.protected == true then return id, rec end + end + return nil, nil +end + +local function log_wired_summary(state, snap, reason) + if not state.svc then return end + local surfaces = snap and snap.surfaces or {} + if not has_observed_surfaces(surfaces) or not has_observed_link_state(surfaces) then return end + local up, down, unknown = count_link_states(surfaces) + local wan_id, wan = choose_wan_surface(surfaces) + if not wan then return end + local trunk_id, trunk = choose_trunk_surface(surfaces) + local parts = {} + if wan then parts[#parts + 1] = 'wan=' .. surface_label(wan_id, wan) .. ':' .. link_brief(wan) end + if trunk then parts[#parts + 1] = 'trunk=' .. surface_label(trunk_id, trunk) .. ':' .. link_brief(trunk) end + parts[#parts + 1] = 'links_up=' .. tostring(up) + parts[#parts + 1] = 'links_down=' .. tostring(down) + if unknown and unknown > 0 then parts[#parts + 1] = 'links_unknown=' .. tostring(unknown) end + local summary = 'wired summary ' .. table.concat(parts, ' ') + local tnow = now() + if state.operator_wired_summary_key == summary and (tnow - (state.operator_wired_summary_at or 0)) < 600 then return end + state.operator_wired_summary_key = summary + state.operator_wired_summary_at = tnow + state.svc:info('wired_summary', { + summary = summary, + reason = reason, + wan_surface = wan_id, + trunk_surface = trunk_id, + links_up = up, + links_down = down, + links_unknown = unknown, + }) +end + +local function log_wired_inventory(state, snap) + if not state.svc then return end + local surfaces = snap and snap.surfaces or {} + if count_map(surfaces) == 0 then return end + -- Configuration alone produces placeholder surfaces. Wait until we have at + -- least one observed provider surface before calling the runtime inventory ready. + if not has_observed_surfaces(surfaces) or not has_observed_link_state(surfaces) then return end + local protected = count_where(surfaces, function(_, rec) return rec and rec.protected == true end) + local external = count_where(surfaces, function(_, rec) return rec and rec.source and rec.source.exposure == 'external' end) + local key = tostring(count_map(surfaces)) .. '|' .. tostring(protected) .. '|' .. tostring(external) + if state._operator_inventory_key == key then return end + state._operator_inventory_key = key + local up, down, unknown = count_link_states(surfaces) + state.svc:info('wired_ready', { + summary = string.format('wired ready surfaces=%d protected=%d external=%d links_up=%d links_down=%d links_unknown=%d', count_map(surfaces), protected, external, up, down, unknown), + surfaces = count_map(surfaces), + protected = protected, + external = external, + links_up = up, + links_down = down, + links_unknown = unknown, + }) + log_wired_summary(state, snap, 'inventory_ready') +end + +local function log_protected_backhaul(state, snap) + if not state.svc then return end + state._operator_backhaul_key = state._operator_backhaul_key or {} + state._operator_backhaul_up_seen = state._operator_backhaul_up_seen or {} + local surfaces = (snap and snap.surfaces) or {} + if not has_observed_surfaces(surfaces) or not has_observed_link_state(surfaces) then return end + for id, rec in pairs(surfaces) do + if rec and rec.protected == true then + local avail = rec.availability or {} + local link = rec.link or {} + local lst = link_status(link) + local available = avail.state == 'available' or avail.state == 'ok' + local up = available and lst == 'up' + local state_now = up and 'up' or (lst or avail.state or 'unknown') + local state_key = table.concat({ + tostring(state_now), + tostring(link.speed or ''), + tostring(link.duplex or ''), + tostring(avail.reason or ''), + }, '|') + local pending = false + if not up then + pending = (operator_settling(state) and not state._operator_backhaul_up_seen[id]) + or (not state.operator_surface_baselined and not state._operator_backhaul_up_seen[id]) + or (available and lst == nil) + end + local key = (up and 'up' or (pending and 'pending' or 'degraded')) .. '|' .. state_key + if state._operator_backhaul_key[id] ~= key then + local ls = link_summary(link) + if up then + state._operator_backhaul_key[id] = key + state._operator_backhaul_up_seen[id] = true + state.svc:info('backhaul_up', { + summary = string.format('wired backhaul %s link up%s', surface_label(id, rec), ls ~= '' and (' ' .. ls) or ''), + surface_id = id, + state = state_now, + speed = link_speed(link), + duplex = link.duplex, + }) + else + if pending then + state._operator_backhaul_key[id] = key + state.svc:debug('backhaul_pending', { + summary = string.format('wired backhaul %s pending state=%s%s', surface_label(id, rec), tostring(state_now), avail.reason and (' reason=' .. tostring(avail.reason)) or ''), + surface_id = id, + state = state_now, + reason = avail.reason, + }) + else + state._operator_backhaul_key[id] = key + state.svc:warn('backhaul_degraded', { + summary = string.format('wired backhaul %s degraded state=%s%s%s', surface_label(id, rec), tostring(state_now), ls ~= '' and (' ' .. ls) or '', avail.reason and (' reason=' .. tostring(avail.reason)) or ''), + surface_id = id, + state = state_now, + reason = avail.reason, + speed = link_speed(link), + duplex = link.duplex, + }) + end + end + end + end + end +end + +local function counter_number(t, a, b) + local v = t and t[a] + if type(v) == 'table' and b then v = v[b] end + return tonumber(v) or 0 +end + +local function counter_total(c, key) + return counter_number(c, key) + counter_number(c, 'rx', key) + counter_number(c, 'tx', key) +end + +local function log_counter_anomalies(state, snap) + if not state.svc then return end + state.logged_counter_state = state.logged_counter_state or {} + state.counter_windows = state.counter_windows or {} + local tnow = now() + for id, rec in pairs((snap and snap.counters) or {}) do + local counters = rec.counters or {} + local errors = counter_total(counters, 'errors') + local drops = counter_total(counters, 'drops') + local prev = state.logged_counter_state[id] + state.logged_counter_state[id] = { errors = errors, drops = drops } + if prev then + local de = errors - (prev.errors or 0) + local dd = drops - (prev.drops or 0) + if de > 0 or dd > 0 then + local surf = snap.surfaces and snap.surfaces[id] + -- Operator warnings are reserved for protected/backhaul surfaces. Other + -- counters remain available as metrics and debug state. + if not (surf and surf.protected) then + state.svc:debug('counter_changed', { surface_id = id, errors_delta = de, drops_delta = dd }) + else + local win = state.counter_windows[id] or { started_at = tnow, errors = 0, drops = 0, last_emit = nil } + win.errors = (win.errors or 0) + de + win.drops = (win.drops or 0) + dd + local first = win.last_emit == nil + local should_emit = (first and ((win.errors or 0) >= COUNTER_FIRST_THRESHOLD or (win.drops or 0) >= COUNTER_FIRST_THRESHOLD)) + or ((not first) and (tnow - win.last_emit) >= COUNTER_REPEAT_S) + if should_emit then + local elapsed = math.max(1, math.floor(tnow - (win.started_at or tnow) + 0.5)) + state.svc:warn(first and 'counter_anomaly' or 'counter_anomaly_continuing', { + summary = string.format('wired %s errors=+%d drops=+%d over=%ds', surface_label(id, surf), win.errors or 0, win.drops or 0, elapsed), + surface_id = id, + errors_delta = win.errors or 0, + drops_delta = win.drops or 0, + protected = true, + elapsed_s = elapsed, + }) + win = { started_at = tnow, errors = 0, drops = 0, last_emit = tnow } + end + state.counter_windows[id] = win + end + end + else + local win = state.counter_windows[id] + if win and win.last_emit and (tnow - win.last_emit) >= COUNTER_REPEAT_S and (win.errors or 0) == 0 and (win.drops or 0) == 0 then + local surf = snap.surfaces and snap.surfaces[id] + if surf and surf.protected then + state.svc:info('counter_anomaly_stable', { + summary = string.format('wired %s counters stable for %ds', surface_label(id, surf), COUNTER_REPEAT_S), + surface_id = id, + }) + end + state.counter_windows[id] = nil + end + end + end +end + +local function new_service_id() + return ('wired-%d-%d'):format(os.time(), math.random(1, 1000000)) +end + +local function topic_eq(t, a) + if type(t) ~= 'table' or #t ~= #a then return false end + for i = 1, #a do if t[i] ~= a[i] then return false end end + return true +end + +local function update_observation_from_event(snap, ev) + local topic = ev and ev.topic or {} + if topic[1] ~= 'raw' + or topic[2] ~= 'host' + or topic[3] ~= 'wired' + or topic[4] ~= 'provider' + then + return false + end + local id = topic[5] + if type(id) ~= 'string' or id == '' then return false end + local current = snap.observations[id] + local rec = copy(current or { id = id, status = {}, identity = {}, runtime = {}, power = {}, surfaces = {}, counters = {}, topology = {}, meta = {} }) + if ev.op == 'unretain' then + if topic[6] == 'status' then rec.status = { state = 'unavailable', available = false, reason = 'unretained' } + elseif topic[6] == 'state' and topic[7] == 'identity' then rec.identity = {} + elseif topic[6] == 'state' and topic[7] == 'runtime' then rec.runtime = {} + elseif topic[6] == 'state' and topic[7] == 'power' then rec.power = {} + elseif topic[6] == 'state' and topic[7] == 'surfaces' then rec.surfaces = {} + elseif topic[6] == 'state' and topic[7] == 'counters' then rec.counters = {} + elseif topic[6] == 'state' and topic[7] == 'topology' then rec.topology = {} + elseif topic[6] == 'meta' then rec.meta = {} end + else + local payload = copy(ev.payload or {}) + if topic[6] == 'status' then rec.status = payload + elseif topic[6] == 'state' and topic[7] == 'identity' then rec.identity = payload + elseif topic[6] == 'state' and topic[7] == 'runtime' then rec.runtime = payload + elseif topic[6] == 'state' and topic[7] == 'power' then rec.power = payload + elseif topic[6] == 'state' and topic[7] == 'surfaces' then rec.surfaces = payload.surfaces or payload + elseif topic[6] == 'state' and topic[7] == 'counters' then rec.counters = payload.counters or payload + elseif topic[6] == 'state' and topic[7] == 'topology' then rec.topology = payload + elseif topic[6] == 'meta' then rec.meta = payload end + end + if tablex.deep_equal(current, rec) then return false, id end + snap.observations[id] = rec + return true, id +end + +local function net_segments_from_payload(payload) + if type(payload) ~= 'table' then return {}, {} end + return copy(payload.segments or payload), copy(payload.vlan_policy or {}) +end + +local function observed_surface_record(observation, observed_surface) + local surfaces = observation and observation.surfaces or nil + if type(surfaces) ~= 'table' then return nil end + return surfaces[observed_surface] +end + + +local function assembly_surface(snap, surface_id) + local assembly = snap and snap.assembly or {} + local surfaces = assembly and assembly.surfaces or {} + return type(surfaces) == 'table' and surfaces[surface_id] or nil +end + +local function resolve_observation_binding(snap, desired) + local a = assembly_surface(snap, desired and desired.surface_id or desired and desired.id) + if type(a) ~= 'table' then return nil, 'assembly_surface_missing' end + if type(a.component) ~= 'string' or a.component == '' then return nil, 'assembly_component_missing' end + if type(a.observed_surface) ~= 'string' or a.observed_surface == '' then return nil, 'assembly_observed_surface_missing' end + return { + component = a.component, + observed_surface = a.observed_surface, + exposure = a.exposure, + connector = a.connector, + assembly = copy(a), + }, nil +end + +local function source_from_binding(binding) + if not binding then return nil end + return { + kind = 'local-provider', + component = binding.component, + observed_surface = binding.observed_surface, + exposure = binding.exposure, + connector = binding.connector, + } +end + +local function observation_available(_snap, _component, observation) + if observation == nil then return false, 'source_missing' end + local st = observation.status or {} + if st.available == false or st.state == 'removed' or st.state == 'unavailable' or st.state == 'not_configured' then + return false, st.reason or st.state or 'source_unavailable' + end + return true, nil +end + +local function append_violation(violations, kind, fields) + local v = { kind = kind, severity = 'error' } + for k, val in pairs(fields or {}) do v[k] = val end + violations[#violations + 1] = v +end + +local function collect_attachment_segments(attachment) + local out = {} + if not attachment then return out end + if attachment.segment then out[#out + 1] = attachment.segment end + for i = 1, #(attachment.segments or {}) do out[#out + 1] = attachment.segments[i] end + for i = 1, #(attachment.required_segments or {}) do out[#out + 1] = attachment.required_segments[i] end + if type(attachment.user_segments) == 'table' then + for i = 1, #attachment.user_segments do out[#out + 1] = attachment.user_segments[i] end + end + if attachment.native_segment then out[#out + 1] = attachment.native_segment end + return out +end + +local function normalise_vlan_id(v) + local n = tonumber(v) + if n == nil or n % 1 ~= 0 or n < 1 or n > 4094 then return nil end + return n +end + +local function add_vlan_value(set, list, v) + if v == nil then return end + local tv = type(v) + if tv == 'number' or tv == 'string' then + local id = normalise_vlan_id(v) + if id then + set[id] = true + list[#list + 1] = id + end + elseif tv == 'table' then + if v.id ~= nil then add_vlan_value(set, list, v.id) end + if v.vlan ~= nil then add_vlan_value(set, list, v.vlan) end + if v.vid ~= nil then add_vlan_value(set, list, v.vid) end + for k, val in pairs(v) do + if type(k) == 'number' then + add_vlan_value(set, list, val) + elseif val == true then + add_vlan_value(set, list, k) + end + end + end +end + +local function observed_vlan_set(attachment) + local set, list = {}, {} + attachment = attachment or {} + add_vlan_value(set, list, attachment.vlan) + add_vlan_value(set, list, attachment.pvid) + add_vlan_value(set, list, attachment.vlans) + add_vlan_value(set, list, attachment.tagged) + add_vlan_value(set, list, attachment.untagged) + add_vlan_value(set, list, attachment.tagged_vlans) + add_vlan_value(set, list, attachment.untagged_vlans) + table.sort(list) + return set, list +end + +local function segment_vlan_id(segment) + if type(segment) ~= 'table' then return nil end + local vlan = segment.vlan + if type(vlan) == 'table' then + return normalise_vlan_id(vlan.id or vlan.vlan or vlan.vid) + end + return normalise_vlan_id(vlan) +end + +local function is_realised_user_segment(seg) + if type(seg) ~= 'table' then return false end + if seg.enabled == false then return false end + if seg.protected == true then return false end + if seg.kind == 'system' then return false end + return segment_vlan_id(seg) ~= nil +end + +local function append_expected_vlan(violations, expected, observed, observed_list, surface_id, seg_id, seg, role) + if not seg then + if role == 'required' then + append_violation(violations, 'missing_required_segment_definition', { + severity = 'critical', + surface_id = surface_id, + segment = seg_id, + }) + end + return + end + local vlan_id = segment_vlan_id(seg) + if vlan_id == nil then + append_violation(violations, role == 'user' and 'missing_user_segment_vlan' or 'missing_required_segment_vlan', { + severity = role == 'required' and 'critical' or 'error', + surface_id = surface_id, + segment = seg_id, + }) + return + end + expected[#expected + 1] = { segment = seg_id, vlan = vlan_id, role = role } + if not observed[vlan_id] then + append_violation(violations, role == 'user' and 'missing_user_segment_carriage' or 'missing_required_segment_carriage', { + severity = role == 'required' and 'critical' or 'error', + surface_id = surface_id, + segment = seg_id, + vlan = vlan_id, + observed_vlans = copy(observed_list), + }) + end +end + +local function expand_user_segments(attachment, segments) + local us = attachment and attachment.user_segments or nil + local out = {} + if us == 'all-realised-user-segments' then + local keys = tablex.sorted_keys(segments or {}) + for i = 1, #keys do + local sid = keys[i] + if is_realised_user_segment(segments[sid]) then out[#out + 1] = sid end + end + elseif type(us) == 'table' then + for i = 1, #us do out[#out + 1] = us[i] end + end + return out +end + +local function segment_ids_for_vlans(segments, vlan_list) + local out, seen = {}, {} + for _, vlan in ipairs(vlan_list or {}) do + for sid, seg in pairs(segments or {}) do + if segment_vlan_id(seg) == vlan and not seen[sid] then + seen[sid] = true + out[#out + 1] = sid + end + end + end + table.sort(out) + return out +end + +local function connector_label(binding) + local c = binding and binding.connector or nil + if type(c) == 'string' and c ~= '' then return c end + return nil +end + +local function infer_surface_semantics(_surface_id, desired, binding, observed_attachment, segments) + local inferred = {} + local observed, observed_list = observed_vlan_set(observed_attachment or {}) + local carried = segment_ids_for_vlans(segments, observed_list) + if #carried > 0 then inferred.carried_segments = carried end + + if desired and desired.protected == true then + inferred.role = 'protected-trunk' + inferred.label = desired.name + return inferred + end + + local wan_vlan = segments and segments.wan and segment_vlan_id(segments.wan) or nil + local untagged = observed_attachment and observed_attachment.untagged_vlans or nil + local pvid = normalise_vlan_id(observed_attachment and observed_attachment.pvid) + local untagged_wan = false + if wan_vlan then + if pvid == wan_vlan then untagged_wan = true end + for _, vlan in ipairs(untagged or {}) do + if normalise_vlan_id(vlan) == wan_vlan then untagged_wan = true end + end + end + + if wan_vlan and observed[wan_vlan] and untagged_wan then + inferred.role = 'wan-uplink' + inferred.segment_id = 'wan' + inferred.vlan_id = wan_vlan + local c = connector_label(binding) + inferred.label = c and ('WAN uplink ' .. c) or 'WAN uplink' + end + + return inferred +end + +local function validate_observed_capabilities(violations, surface_id, desired, binding, p_surface) + if p_surface == nil then return end + local caps = p_surface.capabilities + if not is_plain_table(caps) then return end + local mode = desired.attachment and desired.attachment.mode or 'none' + if mode == 'access' and caps.access ~= true then + append_violation(violations, 'observed_surface_does_not_support_access', { + surface_id = surface_id, + observed_surface = binding and binding.observed_surface or nil, + }) + elseif mode == 'trunk' and caps.trunk ~= true then + append_violation(violations, 'observed_surface_does_not_support_trunk', { + surface_id = surface_id, + observed_surface = binding and binding.observed_surface or nil, + }) + end + local dcaps = desired.capabilities + if is_plain_table(dcaps) and dcaps.poe == true and caps.poe ~= true then + append_violation(violations, 'observed_surface_does_not_support_poe', { + surface_id = surface_id, + observed_surface = binding and binding.observed_surface or nil, + }) + end +end + +local function validate_protected_trunk_carriage(violations, surface_id, desired, p_ok, p_surface, observed_attachment, segments) + if desired.enabled == false or desired.attachment.mode ~= 'trunk' then return {} end + if not p_ok or p_surface == nil then return {} end + + local expected = {} + if observed_attachment.mode ~= nil and observed_attachment.mode ~= 'trunk' then + append_violation(violations, 'protected_trunk_observed_not_trunk', { + severity = 'critical', + surface_id = surface_id, + observed_mode = observed_attachment.mode, + }) + end + + local observed, observed_list = observed_vlan_set(observed_attachment) + for _, seg in ipairs(desired.attachment.required_segments or {}) do + append_expected_vlan(violations, expected, observed, observed_list, surface_id, seg, segments[seg], 'required') + end + for _, seg in ipairs(expand_user_segments(desired.attachment, segments)) do + append_expected_vlan(violations, expected, observed, observed_list, surface_id, seg, segments[seg], 'user') + end + return expected, observed_list +end + +local function rebuild_derived(snap) + local surfaces = {} + local counters = {} + local violations = {} + local topology = { protected_trunks = {}, access = {}, trunks = {} } + local segments = snap.net and snap.net.segments or {} + + for id, desired in pairs((snap.config_intent and snap.config_intent.surfaces) or {}) do + local binding, binding_reason = resolve_observation_binding(snap, desired) + local component = binding and binding.component or nil + local observation = component and snap.observations[component] or nil + local p_ok, p_reason = observation_available(snap, component, observation) + if not binding then p_ok, p_reason = false, binding_reason or 'assembly_binding_missing' end + local p_surface = binding and observed_surface_record(observation, binding.observed_surface) or nil + local link = copy((p_surface and p_surface.link) or {}) + local observed_attachment = copy((p_surface and p_surface.attachment) or {}) + local observed_counters = copy((observation and observation.counters and binding and observation.counters[binding.observed_surface]) or {}) + local observed_poe = copy((p_surface and p_surface.poe) or {}) + local availability = { state = 'available', reason = nil } + if desired.enabled == false then + availability = { state = 'disabled', reason = 'disabled_by_config' } + elseif not p_ok then + availability = { state = 'unavailable', reason = p_reason } + elseif p_surface == nil then + availability = { state = 'degraded', reason = 'observed_surface_missing' } + end + + for _, segment_id in ipairs(collect_attachment_segments(desired.attachment)) do + if not segments[segment_id] then + append_violation(violations, 'unknown_segment', { surface_id = id, segment = segment_id }) + end + end + + validate_observed_capabilities(violations, id, desired, binding, p_surface) + + if next(observed_counters) ~= nil then + counters[id] = { + surface_id = id, + source = source_from_binding(binding), + counters = observed_counters, + } + end + + if desired.protected then + if desired.enabled == false then append_violation(violations, 'protected_surface_disabled', { surface_id = id, severity = 'critical' }) end + if desired.attachment.mode ~= 'trunk' then append_violation(violations, 'protected_surface_not_trunk', { surface_id = id, severity = 'critical' }) end + if observation == nil then + append_violation(violations, 'protected_source_missing', { surface_id = id, component = component, severity = 'critical' }) + elseif not p_ok then + append_violation(violations, 'protected_source_unavailable', { surface_id = id, component = component, reason = p_reason, severity = 'critical' }) + elseif p_surface == nil then + append_violation(violations, 'protected_observed_surface_missing', { + surface_id = id, + component = component, + observed_surface = binding and binding.observed_surface, + severity = 'critical', + }) + end + local expected_vlans, observed_vlans = validate_protected_trunk_carriage(violations, id, desired, p_ok, p_surface, observed_attachment, segments) + topology.protected_trunks[id] = { + surface_id = id, + required_segments = copy(desired.attachment.required_segments), + required_vlans = expected_vlans, + observed_vlans = observed_vlans, + source = source_from_binding(binding), + } + end + + if desired.attachment.mode == 'access' then topology.access[id] = { surface_id = id, segment = desired.attachment.segment } + elseif desired.attachment.mode == 'trunk' then topology.trunks[id] = { surface_id = id, segments = copy(desired.attachment.segments), required_segments = copy(desired.attachment.required_segments), user_segments = copy(desired.attachment.user_segments), expanded_user_segments = expand_user_segments(desired.attachment, segments) } + end + + local inferred = infer_surface_semantics(id, desired, binding, observed_attachment, segments) + surfaces[id] = { + surface_id = id, + name = desired.name, + operator_label = inferred.label, + description = desired.description, + kind = desired.kind, + role = inferred.role or desired.role, + configured_role = desired.role, + segment_id = inferred.segment_id, + vlan_id = inferred.vlan_id, + carried_segments = inferred.carried_segments, + enabled = desired.enabled, + protected = desired.protected, + source = source_from_binding(binding), + capabilities = copy(desired.capabilities), + attachment = copy(desired.attachment), + observed = { attachment = observed_attachment, poe = observed_poe, source = source_from_binding(binding) }, + link = link, + poe = observed_poe, + availability = availability, + } + end + + snap.surfaces = surfaces + snap.counters = counters + snap.topology = topology + snap.violations = violations + snap.ready = true + snap.state = (#violations == 0) and 'running' or 'degraded' + snap.reason = (#violations == 0) and nil or 'validation_violations' + return snap +end + +local function mark_surfaces_for_provider(dirty, snap, provider_id) + local marked = false + if provider_id == nil then return marked end + for surface_id, desired in pairs((snap.config_intent and snap.config_intent.surfaces) or {}) do + local binding = resolve_observation_binding(snap, desired) + if binding and binding.component == provider_id then + publisher.mark_surface(dirty, surface_id) + marked = true + end + end + return marked +end + + +local function mark_counters_for_provider(dirty, snap, provider_id) + local marked = false + if provider_id == nil then return marked end + for surface_id, desired in pairs((snap.config_intent and snap.config_intent.surfaces) or {}) do + local binding = resolve_observation_binding(snap, desired) + if binding and binding.component == provider_id then + publisher.mark_counter(dirty, surface_id) + marked = true + end + end + return marked +end + + +local function log_surface_transitions(state, before, after) + if not state.svc then return end + local after_surfaces = after and after.surfaces or {} + if not has_observed_surfaces(after_surfaces) or not has_observed_link_state(after_surfaces) then return end + + local function key_for(rec) + local link = rec and rec.link or {} + return tostring(link_status(link) or 'unknown') .. '|' .. tostring(link_speed(link) or '') .. '|' .. tostring(link.duplex or '') + end + + -- First observed provider snapshot establishes the baseline. Do not narrate + -- every initially-down LAN port as a transition. + if not state.operator_surface_baselined then + state.operator_surface_baselined = true + local up, down, unknown = count_link_states(after_surfaces) + for id, rec in pairs(after_surfaces) do state.logged_surface_state[id] = key_for(rec) end + state.svc:info('wired_ports_ready', { + summary = string.format('wired ports observed links_up=%d links_down=%d links_unknown=%d', up, down, unknown), + links_up = up, + links_down = down, + links_unknown = unknown, + }) + return + end + + for id, rec in pairs(after_surfaces or {}) do + local link = rec and rec.link or {} + local lst = link_status(link) + local key = key_for(rec) + if state.logged_surface_state[id] ~= key then + local prev_key = state.logged_surface_state[id] + state.logged_surface_state[id] = key + if lst == nil then + state.svc:debug('surface_link_unknown', { surface_id = id, previous = prev_key, current = key }) + else + local up = lst == 'up' + local important = rec and (rec.protected == true or rec.role == 'wan-uplink') + local level = (important and not up) and 'warn' or 'info' + local what = up and 'link_up' or 'link_down' + local ls = link_summary(link) + local ss = segment_summary(rec) + state.svc:log(level, what, { + summary = string.format('wired %s link %s%s%s', surface_label(id, rec), up and 'up' or 'down', ls ~= '' and (' ' .. ls) or '', ss ~= '' and (' ' .. ss) or ''), + surface_id = id, + name = rec and rec.name, + operator_label = rec and rec.operator_label, + role = rec and rec.role, + segment_id = rec and rec.segment_id, + vlan_id = rec and rec.vlan_id, + protected = rec and rec.protected, + link_state = lst, + speed = link_speed(link), + duplex = link.duplex, + }) + end + end + end +end + +local function publish(state) + local snap = state.model:snapshot() + local ok, err, changed = publisher.publish_dirty_now(state.conn, snap, state.published, state.dirty) + if ok ~= true then return nil, err end + if (changed or 0) > 0 then + state.model:update(function (s) + s.stats.publications = (s.stats.publications or 0) + 1 + return s + end) + end + return true, nil +end + + +local function apply_config(state, ev) + local intent, err = config_mod.normalise(ev and ev.raw or nil, { rev = ev and ev.rev, generation = ev and ev.generation }) + if not intent then return nil, err end + local before = state.model:snapshot() + local _changed, _version, uerr = state.model:update(function (snap) + snap.generation = (ev and ev.generation) or (snap.generation + 1) + snap.config = { rev = intent.rev, schema = intent.schema, config_schema = intent.config_schema, version = intent.version } + snap.config_intent = intent + snap.stats.config_updates = (snap.stats.config_updates or 0) + 1 + return rebuild_derived(snap) + end) + if uerr ~= nil then return nil, uerr end + publisher.mark_all(state.dirty) + if state.svc then state.svc:info('wired_config_applied', { summary = string.format('wired config applied surfaces=%s', tostring(count_map(intent.surfaces or {}))), generation = intent.generation, surfaces = count_map(intent.surfaces or {}) }) end + -- Runtime inventory is narrated after provider observations arrive; config + -- alone only tells us intended surfaces. + return publish(state) +end + +local function apply_net_segments(state, ev) + if ev and ev.op == 'replay_done' then return true, nil end + local segments, vlan_policy = net_segments_from_payload(ev and ev.payload) + local _changed, _version, uerr = state.model:update(function (snap) + snap.net.segments = segments + snap.net.vlan_policy = vlan_policy + snap.net.segments_rev = ev and (ev.payload and ev.payload.rev or ev.payload and ev.payload.generation) + snap.stats.segment_updates = (snap.stats.segment_updates or 0) + 1 + return rebuild_derived(snap) + end) + if uerr ~= nil then return nil, uerr end + publisher.mark_all(state.dirty) + return publish(state) +end + +local function apply_assembly_event(state, ev) + if ev and ev.op == 'replay_done' then return true, nil end + local assembly = (ev and ev.op == 'unretain') and {} or copy((ev and ev.payload) or {}) + local _changed, _version, uerr = state.model:update(function (snap) + snap.assembly = assembly + snap.stats.assembly_updates = (snap.stats.assembly_updates or 0) + 1 + return rebuild_derived(snap) + end) + if uerr ~= nil then return nil, uerr end + publisher.mark_all(state.dirty) + return publish(state) +end + +local function apply_observation_event(state, ev) + if ev and ev.op == 'replay_done' then return true, nil end + local changed_provider_id + local observation_changed = false + local before = state.model:snapshot() + local _changed, _version, uerr = state.model:update(function (snap) + local changed, provider_id = update_observation_from_event(snap, ev) + changed_provider_id = provider_id + observation_changed = changed == true + if changed then snap.stats.observation_updates = (snap.stats.observation_updates or 0) + 1 end + return rebuild_derived(snap) + end) + if uerr ~= nil then return nil, uerr end + if not observation_changed then return true, nil end + local snap = state.model:snapshot() + log_surface_transitions(state, before, snap) + log_wired_inventory(state, snap) + log_protected_backhaul(state, snap) + log_wired_summary(state, snap, 'observation') + log_counter_anomalies(state, snap) + local topic = ev and ev.topic or {} + if topic[6] == 'state' and topic[7] == 'counters' then + if not mark_counters_for_provider(state.dirty, snap, changed_provider_id) then + publisher.mark_summary(state.dirty) + end + return publish(state) + end + local surface_marked = mark_surfaces_for_provider(state.dirty, snap, changed_provider_id) + local counter_marked = mark_counters_for_provider(state.dirty, snap, changed_provider_id) + if not surface_marked and not counter_marked then + publisher.mark_summary(state.dirty) + end + publisher.mark_topology(state.dirty) + publisher.mark_violations(state.dirty) + return publish(state) +end + + +function M.build_state(scope, opts) + opts = opts or {} + local conn = assert(opts.conn, 'wired service requires conn') + local cfg, cfg_err = config_watch.open(conn, 'wired', { changed_kind = 'wired_config_changed', closed_kind = 'wired_config_closed' }) + if not cfg then return nil, cfg_err end + local net_watch, nerr = bus_cleanup.watch_retained(conn, topics.net_segments(), { replay = true, queue_len = 8, full = 'reject_newest' }) + if not net_watch then cfg:close(); return nil, nerr end + local assembly_watch, aerr = bus_cleanup.watch_retained(conn, topics.device_assembly(), { replay = true, queue_len = 8, full = 'reject_newest' }) + if not assembly_watch then cfg:close(); bus_cleanup.unwatch_retained(conn, net_watch); return nil, aerr end + local observation_watch, perr = bus_cleanup.watch_retained(conn, topics.raw_wired_provider_pattern(), { replay = true, queue_len = 32, full = 'reject_newest' }) + if not observation_watch then cfg:close(); bus_cleanup.unwatch_retained(conn, net_watch); bus_cleanup.unwatch_retained(conn, assembly_watch); return nil, perr end + local state = { + scope = scope, + conn = conn, + model = opts.model or model_mod.new(opts.service_id or new_service_id()), + svc = opts.svc, + config_watch = cfg, + net_watch = net_watch, + assembly_watch = assembly_watch, + observation_watch = observation_watch, + published = publisher.new_state(), + dirty = publisher.mark_all(publisher.new_dirty_state()), + logged_surface_state = {}, + logged_counter_state = {}, + operator_started_at = now(), + operator_settle_s = tonumber(opts.operator_settle_s) or OPERATOR_SETTLE_S, + } + scope:finally(function () + cfg:close() + bus_cleanup.unwatch_retained(conn, net_watch) + bus_cleanup.unwatch_retained(conn, assembly_watch) + bus_cleanup.unwatch_retained(conn, observation_watch) + publisher.cleanup_now(conn, state.published) + state.model:terminate('wired_service_stopped') + end) + return state, nil +end + +function M.next_event_op(state) + local arms = { + config = state.config_watch:recv_op(), + net = state.net_watch:recv_op():wrap(function (ev, err) return { kind = ev and 'net_segments_changed' or 'net_segments_closed', ev = ev, err = err } end), + assembly = state.assembly_watch:recv_op():wrap(function (ev, err) return { kind = ev and 'assembly_changed' or 'assembly_closed', ev = ev, err = err } end), + observation = state.observation_watch:recv_op():wrap(function (ev, err) return { kind = ev and 'observation_changed' or 'observation_closed', ev = ev, err = err } end), + } + return fibers.named_choice(arms):wrap(function (_, ev) return ev end) +end + +function M.handle_event(state, ev) + if not ev then return nil, 'wired event stream closed' end + if ev.kind == 'wired_config_changed' then return apply_config(state, ev) end + if ev.kind == 'wired_config_closed' then return nil, ev.err or 'wired config closed' end + if ev.kind == 'net_segments_changed' then return apply_net_segments(state, ev.ev) end + if ev.kind == 'net_segments_closed' then return nil, ev.err or 'net segments watch closed' end + if ev.kind == 'assembly_changed' then return apply_assembly_event(state, ev.ev) end + if ev.kind == 'assembly_closed' then return nil, ev.err or 'device assembly watch closed' end + if ev.kind == 'observation_changed' then return apply_observation_event(state, ev.ev) end + if ev.kind == 'observation_closed' then return nil, ev.err or 'wired observation watch closed' end + return true, nil +end + +M._test = { + rebuild_derived = rebuild_derived, + observed_vlan_set = observed_vlan_set, + update_observation_from_event = update_observation_from_event, +} + +function M.run(scope, opts) + opts = opts or {} + local state, err = M.build_state(scope, opts) + if not state then error(err or 'wired service start failed', 0) end + log_wired_inventory(state, state.model:snapshot()) + log_protected_backhaul(state, state.model:snapshot()) + publish(state) + while true do + local ev = fibers.perform(M.next_event_op(state)) + local ok, herr = M.handle_event(state, ev) + if ok ~= true then error(herr or 'wired service failed', 0) end + end +end + +return M diff --git a/src/services/wired/topics.lua b/src/services/wired/topics.lua new file mode 100644 index 00000000..e4fea263 --- /dev/null +++ b/src/services/wired/topics.lua @@ -0,0 +1,27 @@ +-- services/wired/topics.lua +-- Public topic vocabulary owned by the Wired service. + +local M = {} + +local function token(v) + local s = tostring(v or '') + s = s:gsub('[^%w%._%-]', '_') + if s == '' then s = 'unknown' end + return s +end + +function M.config() return { 'cfg', 'wired' } end +function M.summary() return { 'state', 'wired', 'summary' } end +function M.surface(id) return { 'state', 'wired', 'surface', token(id) } end +function M.topology() return { 'state', 'wired', 'topology' } end +function M.violations() return { 'state', 'wired', 'violations' } end +function M.surface_counters(id) return { 'obs', 'v1', 'wired', 'metric', 'surface_counters', token(id) } end +function M.event(name) return { 'event', 'wired', token(name) } end + +function M.net_segments() return { 'state', 'net', 'segments' } end +function M.device_assembly() return { 'state', 'device', 'assembly' } end +function M.raw_wired_provider_pattern() return { 'raw', 'host', 'wired', 'provider', '#' } end + +M._test = { token = token } + +return M diff --git a/src/binser.lua b/src/shared/binser.lua similarity index 98% rename from src/binser.lua rename to src/shared/binser.lua index 7dd30207..29316d14 100644 --- a/src/binser.lua +++ b/src/shared/binser.lua @@ -21,7 +21,7 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ]] -local fiber_file = require "fibers.stream.file" +local fiber_file = require "fibers.io.file" local assert = assert local error = error @@ -44,11 +44,11 @@ local unpack = unpack or table.unpack -- Lua 5.3 frexp polyfill -- From https://github.com/excessive/cpml/blob/master/modules/utils.lua if not frexp then - local log, abs, floor = math.log, math.abs, math.floor + local log, abs, lfloor = math.log, math.abs, math.floor local log2 = log(2) frexp = function(x) if x == 0 then return 0, 0 end - local e = floor(log(abs(x)) / log2 + 1) + local e = lfloor(log(abs(x)) / log2 + 1) return x / 2 ^ e, e end end @@ -228,15 +228,15 @@ local function newbinser() local resources_by_name = {} local types = {} - types["nil"] = function(x, visited, accum) + types["nil"] = function(_, _, accum) accum[#accum + 1] = "\202" end - function types.number(x, visited, accum) + function types.number(x, _, accum) accum[#accum + 1] = number_to_str(x) end - function types.boolean(x, visited, accum) + function types.boolean(x, _, accum) accum[#accum + 1] = x and "\204" or "\205" end @@ -379,7 +379,7 @@ local function newbinser() elseif t == 206 then local length, dataindex = number_from_str(str, index + 1) local nextindex = dataindex + length - if not (length >= 0) then error("Bad string length") end + if length < 0 then error("Bad string length") end if #str < nextindex - 1 then error("Expected more bytes of string") end local substr = sub(str, dataindex, nextindex - 1) visited[#visited + 1] = substr @@ -400,7 +400,7 @@ local function newbinser() if nextindex == oldindex then error("Expected more bytes of input.") end end count, nextindex = number_from_str(str, nextindex) - for i = 1, count do + for _ = 1, count do local k, v local oldindex = nextindex k, nextindex = deserialize_value(str, nextindex, visited) @@ -435,7 +435,7 @@ local function newbinser() elseif t == 210 then local length, dataindex = number_from_str(str, index + 1) local nextindex = dataindex + length - if not (length >= 0) then error("Bad string length") end + if length < 0 then error("Bad string length") end if #str < nextindex - 1 then error("Expected more bytes of string") end local ret = loadstring(sub(str, dataindex, nextindex - 1)) visited[#visited + 1] = ret @@ -542,7 +542,7 @@ local function newbinser() local function readFile(path) local file, err = fiber_file.open(path, "rb") assert(file, err) - local file_chars = file:read_all_chars() + local file_chars = file:read("*a") file:close() return deserialize(file_chars) end diff --git a/src/shared/cache.lua b/src/shared/cache.lua new file mode 100644 index 00000000..f4bee5d4 --- /dev/null +++ b/src/shared/cache.lua @@ -0,0 +1,97 @@ +---@alias CacheKey string|string[] +---@alias CacheEntry {value: any, timestamp: number, timeout: number} + +---@class Cache +---@field default_timeout number +---@field time_func function +---@field separator string +---@field store table +local Cache = {} +Cache.__index = Cache + +--- Cache constructor +---@param default_timeout number? +---@param custom_time_func function? +---@param separator string? +---@return Cache +local function new(default_timeout, custom_time_func, separator) + local self = setmetatable({}, Cache) + self.default_timeout = default_timeout or 10 + self.time_func = custom_time_func or os.time + self.separator = separator or string.char(31) + self.store = {} + return self +end + +--- Setting a value in the cache +---@param key string +---@param value any +---@param timeout number? +function Cache:set(key, value, timeout) + timeout = timeout or self.default_timeout + -- if type(value) == 'table' then + -- if is_array(value) then + -- self.store[key] = {value=value, timestamp=self.time_func() + timeout, timeout = timeout} + -- else + -- for k, v in pairs(value) do + -- self:set(key .. self.separator .. k, v, timeout) + -- end + -- end + -- else + -- self.store[key] = {value = value, timestamp=self.time_func(), timeout = timeout} + -- end + self.store[key] = {value = value, timestamp=self.time_func(), timeout = timeout} +end + + +--- Delete a value from the cache. +---@param key CacheKey +function Cache:delete(key) + if type(key) ~= 'string' then + key = table.concat(key, self.separator) + end + self.store[key] = nil +end + +--- Delete all cache entries whose string key starts with prefix. +---@param prefix string +function Cache:clear_prefix(prefix) + prefix = tostring(prefix or '') + local remove = {} + for key in pairs(self.store) do + if key == prefix or key:sub(1, #prefix) == prefix then + remove[#remove + 1] = key + end + end + for _, key in ipairs(remove) do + self.store[key] = nil + end +end + +--- Delete all cache entries. +function Cache:clear() + self.store = {} +end + +--- Getting a value from the cache +---@param key CacheKey +---@param timeout number? +---@return any +function Cache:get(key, timeout) + if type(key) ~= 'string' then + key = table.concat(key, self.separator) + end + local item = self.store[key] + if item then + timeout = timeout or item.timeout + if self.time_func() < (item.timestamp + timeout) then + return item.value + end + end + return nil -- or a default value +end + +return { + new = new, + Cache = Cache +} diff --git a/src/shared/encoding/b64url.lua b/src/shared/encoding/b64url.lua new file mode 100644 index 00000000..d23f0bf7 --- /dev/null +++ b/src/shared/encoding/b64url.lua @@ -0,0 +1,98 @@ +---@module 'shared.encoding.b64url' + +local M = {} + +local ENC = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' +local DEC = {} +for i = 1, #ENC do + DEC[ENC:sub(i, i)] = i - 1 +end + +local function to_bits(n, width) + local t = {} + for i = width - 1, 0, -1 do + t[#t + 1] = math.floor(n / (2 ^ i)) % 2 + end + return t +end + +local function from_bits(bits, i, width) + local n = 0 + for k = 0, width - 1 do + n = n * 2 + bits[i + k] + end + return n +end + +---@param s string +---@return string +function M.encode(s) + assert(type(s) == 'string', 'b64url.encode expects string') + + local bits = {} + for i = 1, #s do + local bb = to_bits(s:byte(i), 8) + for j = 1, #bb do + bits[#bits + 1] = bb[j] + end + end + + while (#bits % 6) ~= 0 do + bits[#bits + 1] = 0 + end + + local out = {} + for i = 1, #bits, 6 do + local idx = from_bits(bits, i, 6) + 1 + out[#out + 1] = ENC:sub(idx, idx) + end + + local pad = ({ '', '==', '=' })[#s % 3 + 1] + local encoded = table.concat(out) .. pad + encoded = encoded:gsub('+', '-') + encoded = encoded:gsub('/', '_') + encoded = encoded:gsub('=', '') + return encoded +end + +---@param s string +---@return string|nil +---@return string|nil +function M.decode(s) + assert(type(s) == 'string', 'b64url.decode expects string') + + s = s:gsub('-', '+'):gsub('_', '/') + + local rem = #s % 4 + if rem == 2 then + s = s .. '==' + elseif rem == 3 then + s = s .. '=' + elseif rem == 1 then + return nil, 'invalid_base64url_length' + end + + local bits = {} + for i = 1, #s do + local ch = s:sub(i, i) + if ch ~= '=' then + local v = DEC[ch] + if v == nil then + return nil, 'invalid_base64url_character' + end + local bb = to_bits(v, 6) + for j = 1, #bb do + bits[#bits + 1] = bb[j] + end + end + end + + local out = {} + for i = 1, #bits - 7, 8 do + out[#out + 1] = string.char(from_bits(bits, i, 8)) + end + + return table.concat(out), nil +end + +return M diff --git a/src/shared/hash/xxhash32.lua b/src/shared/hash/xxhash32.lua new file mode 100644 index 00000000..5941391a --- /dev/null +++ b/src/shared/hash/xxhash32.lua @@ -0,0 +1,224 @@ +-- shared/hash/xxhash32.lua +-- +-- Portable xxHash32, seed 0 by default. +-- +-- This implementation does not use bit.band(x, 0xffffffff) or +-- bit32.band(x, 0xffffffff) as a 32-bit normalisation primitive. Some bit32 +-- compatibility modules under LuaJIT/OpenWrt mishandle large Lua numeric +-- intermediates in that pattern. Unsigned 32-bit normalisation is therefore +-- done with arithmetic modulo 2^32, while bit libraries are used only for +-- true bitwise operations on already-normalised values. + +local ok_bit, bit_mod = pcall(require, 'bit') +local ok_bit32, bit32_mod = pcall(require, 'bit32') + +local raw_bitops = ok_bit and bit_mod or (ok_bit32 and bit32_mod or nil) +assert(raw_bitops, 'shared.hash.xxhash32 requires bit or bit32') + +local TWO32 = 4294967296 +local TWO16 = 65536 + +local function u32(n) + -- Lua numbers are doubles in LuaJIT/Lua 5.1. All intermediates in this + -- implementation are kept well below 2^53, so modulo is exact here. + n = n % TWO32 + if n < 0 then n = n + TWO32 end + return n +end + +local function band(a, b) + return u32(raw_bitops.band(u32(a), u32(b))) +end + +local function bor(a, b) + return u32(raw_bitops.bor(u32(a), u32(b))) +end + +local function bxor(a, b) + return u32(raw_bitops.bxor(u32(a), u32(b))) +end + +local function lshift(a, n) + n = n % 32 + return u32(raw_bitops.lshift(u32(a), n)) +end + +local function rshift(a, n) + n = n % 32 + return u32(raw_bitops.rshift(u32(a), n)) +end + +local function rol(a, n) + n = n % 32 + a = u32(a) + if n == 0 then return a end + return bor(lshift(a, n), rshift(a, 32 - n)) +end + +local M = {} +M._backend = ok_bit and 'bit' or 'bit32' + +local P1 = 0x9E3779B1 +local P2 = 0x85EBCA77 +local P3 = 0xC2B2AE3D +local P4 = 0x27D4EB2F +local P5 = 0x165667B1 + +local function hex8(n) + -- Avoid string.format('%x', n): on some plain Lua runtimes n is a float, + -- even when it represents an exact unsigned 32-bit integer. + n = u32(n) + local digits = '0123456789abcdef' + local out = {} + for shift = 28, 0, -4 do + local nibble = math.floor(n / (2 ^ shift)) % 16 + out[#out + 1] = digits:sub(nibble + 1, nibble + 1) + end + return table.concat(out) +end + +local function mul32(a, b) + -- Exact low 32 bits of a 32x32 multiply using 16-bit limbs. + -- This avoids relying on Lua's double precision for the full product and + -- avoids passing a >32-bit intermediate into the bit backend. + a = u32(a) + b = u32(b) + + local a_lo = a % TWO16 + local a_hi = math.floor(a / TWO16) + local b_lo = b % TWO16 + local b_hi = math.floor(b / TWO16) + + local lo = a_lo * b_lo + local mid = a_hi * b_lo + a_lo * b_hi + return u32((lo % TWO32) + ((mid % TWO16) * TWO16)) +end + +local function read_u32_le(s, i) + local b1, b2, b3, b4 = s:byte(i, i + 3) + return u32( + (b1 or 0) + + (b2 or 0) * 0x100 + + (b3 or 0) * 0x10000 + + (b4 or 0) * 0x1000000 + ) +end + +local function round_lane(acc, lane) + acc = u32(acc + mul32(lane, P2)) + acc = rol(acc, 13) + acc = mul32(acc, P1) + return acc +end + +local function avalanche(h) + h = bxor(h, rshift(h, 15)) + h = mul32(h, P2) + h = bxor(h, rshift(h, 13)) + h = mul32(h, P3) + h = bxor(h, rshift(h, 16)) + return u32(h) +end + +function M.new(seed) + seed = u32(seed or 0) + return { + seed = seed, + total_len = 0, + mem = '', + v1 = u32(seed + P1 + P2), + v2 = u32(seed + P2), + v3 = seed, + v4 = u32(seed - P1), + large = false, + } +end + +function M.update(state, s) + assert(type(state) == 'table', 'checksum.update expects state') + assert(type(s) == 'string', 'checksum.update expects string') + if s == '' then return state end + + state.total_len = state.total_len + #s + local buf = state.mem .. s + local idx = 1 + local n = #buf + + if n >= 16 then + state.large = true + while idx + 15 <= n do + state.v1 = round_lane(state.v1, read_u32_le(buf, idx)); idx = idx + 4 + state.v2 = round_lane(state.v2, read_u32_le(buf, idx)); idx = idx + 4 + state.v3 = round_lane(state.v3, read_u32_le(buf, idx)); idx = idx + 4 + state.v4 = round_lane(state.v4, read_u32_le(buf, idx)); idx = idx + 4 + end + end + + state.mem = buf:sub(idx) + return state +end + +function M.digest(state) + assert(type(state) == 'table', 'checksum.digest expects state') + + local h + if state.large then + h = u32( + rol(state.v1, 1) + + rol(state.v2, 7) + + rol(state.v3, 12) + + rol(state.v4, 18) + ) + else + h = u32(state.seed + P5) + end + + h = u32(h + state.total_len) + + local i = 1 + local n = #state.mem + + while i + 3 <= n do + h = u32(h + mul32(read_u32_le(state.mem, i), P3)) + h = mul32(rol(h, 17), P4) + i = i + 4 + end + + while i <= n do + h = u32(h + mul32(state.mem:byte(i) or 0, P5)) + h = mul32(rol(h, 11), P1) + i = i + 1 + end + + return avalanche(h) +end + +function M.digest_hex_state(state) + return hex8(M.digest(state)) +end + +function M.xxhash32(s, seed) + assert(type(s) == 'string', 'checksum.xxhash32 expects string') + local st = M.new(seed) + M.update(st, s) + return M.digest(st) +end + +function M.digest_hex(s) + return hex8(M.xxhash32(s)) +end + +function M.verify_hex(s, expected) + return M.digest_hex(s) == tostring(expected) +end + +-- Expose for diagnostics/tests only. Production callers should use digest_hex. +M._test = { + u32 = u32, + mul32 = mul32, + rol = rol, + read_u32_le = read_u32_le, + hex8 = hex8, +} + +return M diff --git a/src/shared/table.lua b/src/shared/table.lua new file mode 100644 index 00000000..9224cfb8 --- /dev/null +++ b/src/shared/table.lua @@ -0,0 +1,103 @@ +-- shared/table.lua +-- +-- Pure table helpers. This module must not depend on fibers, bus or services. + +local M = {} + +function M.shallow_copy(t) + if t == nil then return {} end + local out = {} + for k, v in pairs(t) do out[k] = v end + return out +end + +M.copy = M.shallow_copy + +function M.array_copy(t) + local out = {} + for i = 1, #(t or {}) do out[i] = t[i] end + return out +end + +function M.copy_array(t) + return M.array_copy(t) +end + +function M.table_or_empty(v) + if type(v) == 'table' then return v end + return {} +end + +function M.list_or_empty(v) + if type(v) == 'table' then return v end + return {} +end + +function M.first_non_nil(...) + for i = 1, select('#', ...) do + local v = select(i, ...) + if v ~= nil then return v end + end + return nil +end + +function M.is_array(t) + if type(t) ~= 'table' then return false end + local n = #t + for k in pairs(t) do + if type(k) ~= 'number' or k < 1 or k % 1 ~= 0 or k > n then + return false + end + end + return true +end + +local function deep_copy_impl(v, seen) + if type(v) ~= 'table' then return v end + seen = seen or {} + if seen[v] then return seen[v] end + local out = {} + seen[v] = out + for k, val in pairs(v) do + out[deep_copy_impl(k, seen)] = deep_copy_impl(val, seen) + end + return out +end + +function M.deep_copy(v) + return deep_copy_impl(v, {}) +end + +local function deep_equal_impl(a, b, seen) + if a == b then return true end + if type(a) ~= 'table' or type(b) ~= 'table' then return false end + seen = seen or {} + seen[a] = seen[a] or {} + if seen[a][b] then return true end + seen[a][b] = true + for k, av in pairs(a) do + if not deep_equal_impl(av, b[k], seen) then return false end + end + for k in pairs(b) do + if a[k] == nil then return false end + end + return true +end + +function M.deep_equal(a, b) + return deep_equal_impl(a, b, {}) +end + +function M.keys(t) + local out = {} + for k in pairs(t or {}) do out[#out + 1] = k end + return out +end + +function M.sorted_keys(t, cmp) + local out = M.keys(t) + table.sort(out, cmp or function(a, b) return tostring(a) < tostring(b) end) + return out +end + +return M diff --git a/src/shared/topic.lua b/src/shared/topic.lua new file mode 100644 index 00000000..60dbf370 --- /dev/null +++ b/src/shared/topic.lua @@ -0,0 +1,73 @@ +-- shared/topic.lua +-- +-- Pure dense topic-array helpers. + +local tablex = require 'shared.table' + +local M = {} + +function M.copy(t) + return tablex.array_copy(t) +end + +function M.append(base, ...) + local out = M.copy(base) + for i = 1, select('#', ...) do + local v = select(i, ...) + if type(v) == 'table' then + for j = 1, #v do out[#out + 1] = v[j] end + else + out[#out + 1] = v + end + end + return out +end + +function M.starts_with(topic, prefix) + if type(topic) ~= 'table' or type(prefix) ~= 'table' then return false end + if #prefix > #topic then return false end + for i = 1, #prefix do + if topic[i] ~= prefix[i] then return false end + end + return true +end + +function M.replace_prefix(topic, from, to) + if not M.starts_with(topic, from) then return nil, 'topic does not start with prefix' end + local out = M.copy(to) + for i = #from + 1, #topic do out[#out + 1] = topic[i] end + return out, nil +end + +function M.key(topic, sep) + sep = sep or '/' + local parts = {} + for i = 1, #(topic or {}) do parts[i] = tostring(topic[i]) end + return table.concat(parts, sep) +end + +function M.to_string(topic) + return M.key(topic, '/') +end + +function M.validate_dense(t, opts) + opts = opts or {} + local name = opts.name or 'topic' + if type(t) ~= 'table' then return nil, name .. ' must be a table' end + local n = #t + for k, v in pairs(t) do + if type(k) ~= 'number' or k < 1 or k % 1 ~= 0 or k > n then + return nil, name .. ' must be a dense array' + end + local tv = type(v) + if tv ~= 'string' and tv ~= 'number' then + return nil, name .. '[' .. tostring(k) .. '] must be a string or number' + end + if opts.non_empty ~= false and tv == 'string' and v == '' then + return nil, name .. '[' .. tostring(k) .. '] must be non-empty' + end + end + return true, nil +end + +return M diff --git a/src/shared/validate.lua b/src/shared/validate.lua new file mode 100644 index 00000000..00b8ad32 --- /dev/null +++ b/src/shared/validate.lua @@ -0,0 +1,95 @@ +-- shared/validate.lua +-- +-- Pure validation helpers. This module must not depend on fibers, bus or services. + +local M = {} + +local function fail(name, msg, level) + error((name or 'value') .. ' ' .. msg, (level or 1) + 1) +end + +function M.table(v, name, level) + if type(v) ~= 'table' then fail(name, 'must be a table', (level or 1) + 1) end + return v +end + +function M.function_(v, name, level) + if type(v) ~= 'function' then fail(name, 'must be a function', (level or 1) + 1) end + return v +end + +M.func = M.function_ + +function M.string(v, name, level) + if type(v) ~= 'string' then fail(name, 'must be a string', (level or 1) + 1) end + return v +end + +function M.non_empty_string(v, name, level) + if type(v) ~= 'string' or v == '' then fail(name, 'must be a non-empty string', (level or 1) + 1) end + return v +end + +function M.non_empty_string_or_nil(v, name, level) + if v == nil then return nil end + return M.non_empty_string(v, name, (level or 1) + 1) +end + +function M.boolean_or_nil(v, name, level) + if v ~= nil and type(v) ~= 'boolean' then fail(name, 'must be a boolean or nil', (level or 1) + 1) end + return v +end + +function M.number(v, name, level) + if type(v) ~= 'number' then fail(name, 'must be a number', (level or 1) + 1) end + return v +end + +function M.positive_number(v, name, level) + if type(v) ~= 'number' or v <= 0 then fail(name, 'must be a positive number', (level or 1) + 1) end + return v +end + +function M.non_negative_number(v, name, level) + if type(v) ~= 'number' or v < 0 then fail(name, 'must be a non-negative number', (level or 1) + 1) end + return v +end + +function M.integer(v, name, level) + if type(v) ~= 'number' or v % 1 ~= 0 then fail(name, 'must be an integer', (level or 1) + 1) end + return v +end + +function M.positive_integer(v, name, level) + if type(v) ~= 'number' or v <= 0 or v % 1 ~= 0 then fail(name, 'must be a positive integer', (level or 1) + 1) end + return v +end + +function M.non_negative_integer(v, name, level) + if type(v) ~= 'number' or v < 0 or v % 1 ~= 0 then fail(name, 'must be a non-negative integer', (level or 1) + 1) end + return v +end + +function M.table_or_empty(v, name, level) + if v == nil then return {} end + return M.table(v, name, (level or 1) + 1) +end + +function M.list_or_empty(v, name, level) + if v == nil then return {} end + return M.table(v, name, (level or 1) + 1) +end + +function M.only_fields(t, allowed, name, level) + M.table(t, name, (level or 1) + 1) + local ok = {} + for _, k in ipairs(allowed or {}) do ok[k] = true end + for k in pairs(t) do + if not ok[k] then + fail((name or 'table') .. '.' .. tostring(k), 'is not permitted', (level or 1) + 1) + end + end + return t +end + +return M diff --git a/src/ubus_scripts/etc/hotplug.d/block/block b/src/ubus_scripts/etc/hotplug.d/block/block deleted file mode 100644 index 05f7c58d..00000000 --- a/src/ubus_scripts/etc/hotplug.d/block/block +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh - -# Define the UBUS event name -EVENT_NAME="hotplug.block" - -# Create JSON payload for UBUS event -JSON_PAYLOAD="{ - \"action\": \"$ACTION\", - \"devicename\": \"$DEVICENAME\", - \"devname\": \"$DEVNAME\", - \"devpath\": \"$DEVPATH\", - \"devtype\": \"$DEVTYPE\", - \"major\": \"$MAJOR\", - \"minor\": \"$MINOR\", - \"seqnum\": \"$SEQNUM\", - \"subsystem\": \"$SUBSYSTEM\" -}" - -# Send event to UBUS -ubus send "$EVENT_NAME" "$JSON_PAYLOAD" diff --git a/src/ubus_scripts/etc/hotplug.d/iface/iface b/src/ubus_scripts/etc/hotplug.d/iface/iface deleted file mode 100644 index ce3c2c32..00000000 --- a/src/ubus_scripts/etc/hotplug.d/iface/iface +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -# Define the UBUS event name -EVENT_NAME="hotplug.iface" - -# Create JSON payload for UBUS event -JSON_PAYLOAD="{ - \"action\": \"$ACTION\", - \"interface\": \"$INTERFACE\", - \"device\": \"$DEVICE\", - \"ifupdate_addresses\": \"$IFUPDATE_ADDRESSES\", - \"ifupdate_routes\": \"$IFUPDATE_ROUTES\", - \"ifupdate_prefixes\": \"$IFUPDATE_PREFIXES\", - \"ifupdate_data\": \"$IFUPDATE_DATA\" -}" - -# Send event to UBUS -ubus send "$EVENT_NAME" "$JSON_PAYLOAD" diff --git a/src/ubus_scripts/etc/hotplug.d/net/net b/src/ubus_scripts/etc/hotplug.d/net/net deleted file mode 100644 index 9d4f2293..00000000 --- a/src/ubus_scripts/etc/hotplug.d/net/net +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh - -# Define the UBUS event name -EVENT_NAME="hotplug.net" - -# Create JSON payload for UBUS event -JSON_PAYLOAD="{ - \"action\": \"$ACTION\", - \"devicename\": \"$DEVICENAME\", - \"path\": \"$PATH\", - \"devpath\": \"$DEVPATH\", - \"devtype\": \"$DEVTYPE\", - \"interface\": \"$INTERFACE\", - \"seqnum\": \"$SEQNUM\", - \"subsystem\": \"$SUBSYSTEM\", - \"ifindex\": \"$IFINDEX\" -}" - -# Send event to UBUS -ubus send "$EVENT_NAME" "$JSON_PAYLOAD" diff --git a/src/ubus_scripts/etc/hotplug.d/tty/tty b/src/ubus_scripts/etc/hotplug.d/tty/tty deleted file mode 100644 index 1ddc8739..00000000 --- a/src/ubus_scripts/etc/hotplug.d/tty/tty +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -# Define the UBUS event name -EVENT_NAME="hotplug.tty" - -# Create JSON payload for UBUS event -JSON_PAYLOAD="{ - \"action\": \"$ACTION\", - \"devicename\": \"$DEVICENAME\", - \"devname\": \"$DEVNAME\", - \"devpath\": \"$DEVPATH\", - \"seqnum\": \"$SEQNUM\", - \"subsystem\": \"$SUBSYSTEM\", - \"major\": \"$MAJOR\", - \"minor\": \"$MINOR\" -}" - -# Send event to UBUS -ubus send "$EVENT_NAME" "$JSON_PAYLOAD" diff --git a/src/ubus_scripts/etc/hotplug.d/usb/usb b/src/ubus_scripts/etc/hotplug.d/usb/usb deleted file mode 100644 index 2ca390cd..00000000 --- a/src/ubus_scripts/etc/hotplug.d/usb/usb +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -# Define the UBUS event name -EVENT_NAME="hotplug.usb" - -# Create JSON payload for UBUS event -JSON_PAYLOAD="{ - \"action\": \"$ACTION\", - \"devicename\": \"$DEVICENAME\", - \"devname\": \"$DEVNAME\", - \"devnum\": \"$DEVNUM\", - \"devpath\": \"$DEVPATH\", - \"devtype\": \"$DEVTYPE\", - \"type\": \"$TYPE\", - \"product\": \"$PRODUCT\", - \"seqnum\": \"$SEQNUM\", - \"busnum\": \"$BUSNUM\", - \"major\": \"$MAJOR\", - \"minor\": \"$MINOR\" -}" - -# Send event to UBUS -ubus send "$EVENT_NAME" "$JSON_PAYLOAD" diff --git a/src/ubus_scripts/etc/hotplug.d/usb/usbmisc b/src/ubus_scripts/etc/hotplug.d/usb/usbmisc deleted file mode 100644 index f026c18a..00000000 --- a/src/ubus_scripts/etc/hotplug.d/usb/usbmisc +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -# Define the UBUS event name -EVENT_NAME="hotplug.usbmisc" - -# Create JSON payload for UBUS event -JSON_PAYLOAD="{ - \"action\": \"$ACTION\", - \"devicename\": \"$DEVICENAME\", - \"devname\": \"$DEVNAME\", - \"devpath\": \"$DEVPATH\", - \"seqnum\": \"$SEQNUM\", - \"major\": \"$MAJOR\", - \"minor\": \"$MINOR\" -}" - -# Send event to UBUS -ubus send "$EVENT_NAME" "$JSON_PAYLOAD" diff --git a/src/ubus_scripts/etc/mwan3.user b/src/ubus_scripts/etc/mwan3.user deleted file mode 100644 index b1d13298..00000000 --- a/src/ubus_scripts/etc/mwan3.user +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -# Define the UBUS event name -EVENT_NAME="hotplug.mwan3" - -# Create JSON payload for UBUS event -JSON_PAYLOAD="{ - \"action\": \"$ACTION\", - \"interface\": \"$INTERFACE\", - \"device\": \"$DEVICE\" -}" - -# Send event to UBUS -ubus send "$EVENT_NAME" "$JSON_PAYLOAD" - -# Default mwan3 action -if [ "${ACTION}" = "connected" ] ; then - /etc/init.d/sysntpd restart -fi diff --git a/src/wraperr.lua b/src/wraperr.lua deleted file mode 100644 index 48c50755..00000000 --- a/src/wraperr.lua +++ /dev/null @@ -1,63 +0,0 @@ --- module wraperr - inspired by golang's errors - -local wraperr = {} -wraperr.__index = wraperr - -function wraperr:__tostring() - return tostring(self.content) .. (self.wrapped and ": "..tostring(self.wrapped) or "") -end - -function wraperr:wrap(err) - self.wrapped = err - return self -end - -function wraperr:unwrap() - return self.wrapped -end - -function wraperr:is(err) - -- Check if the current error object is the same as the given error - if self == err then - return true - end - -- If there is a wrapped error, recursively check it - if self.wrapped then - return self.wrapped:is(err) - end - -- If no match found, return false - return false -end - -function wraperr:as(target_err) - -- Check if the current error's metatable matches the target metatable - if getmetatable(self) == getmetatable(target_err) then - return true, self - elseif self.wrapped then - -- Recursively check the wrapped error - return self.wrapped:as(target_err) - end - - return false, nil -end - -local function custom(to_string) - local mt = { - __tostring = to_string, - __index = wraperr, -- Allows for using wraperr methods like wrap, unwrap, etc. - } - mt.new = function(error_tab) - setmetatable(error_tab, mt) - return error_tab - end - return mt -end - -local function new(content) - return setmetatable({content = content}, wraperr) -end - -return { - new = new, - custom = custom, -} \ No newline at end of file diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index 4358d3ca..00000000 --- a/tests/README.md +++ /dev/null @@ -1 +0,0 @@ -This folder will contain tests for `devicecode` \ No newline at end of file diff --git a/tests/integration/devhost/config_recovery_spec.lua b/tests/integration/devhost/config_recovery_spec.lua new file mode 100644 index 00000000..dbdce647 --- /dev/null +++ b/tests/integration/devhost/config_recovery_spec.lua @@ -0,0 +1,126 @@ +local cjson = require 'cjson.safe' +local busmod = require 'bus' +local safe = require 'coxpcall' + +local runfibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' +local fake_hal_mod = require 'tests.support.fake_hal' +local test_diag = require 'tests.support.test_diag' + +local config_service = require 'services.config' + +local T = {} + +local function decode_json(s) + local v, err = cjson.decode(s) + assert(v ~= nil, tostring(err)) + return v +end + +local function config_timings() + return { + hal_wait_timeout_s = 0.25, + heartbeat_s = 60.0, + persist_debounce_s = 0.02, + persist_max_delay_s = 0.05, + persist_retry_initial_s = 0.02, + persist_retry_max_s = 0.05, + } +end + +function T.devhost_config_recovers_after_failed_persist_retry() + runfibers.run(function(scope) + local bus = busmod.new() + local fake_hal = fake_hal_mod.new({ + backend = 'fakehal', + caps = { read_state = true, write_state = true }, + scripted = { + read_state = { { ok = true, found = false } }, + write_state = { + { ok = false, err = 'disk full' }, + { ok = true }, + }, + }, + }) + fake_hal:start(bus:connect(), { name = 'hal' }) + + local status_events = {} + local diag = test_diag.for_stack(scope, bus, { config = true, obs = true, max_records = 300, fake_hal = fake_hal }) + test_diag.add_calls(diag, 'status_events', status_events) + test_diag.add_subsystem(diag, 'device', { + summary_fn = test_diag.retained_fn(bus:connect(), { 'state', 'device' }), + }) + + local conn = bus:connect() + local sub = conn:subscribe({ 'svc', 'config', 'status' }, { queue_len = 32 }) + local ok_collector, cerr = scope:spawn(function() + while true do + local msg = sub:recv() + if not msg then return end + status_events[#status_events + 1] = msg.payload + end + end) + assert(ok_collector, tostring(cerr)) + + local ok_spawn, err = scope:spawn(function() + config_service.start(bus:connect(), { name = 'config', env = 'dev', timings = config_timings() }) + end) + assert(ok_spawn, tostring(err)) + + local ready = probe.wait_until(function() + for i = 1, #status_events do + local p = status_events[i] + if type(p) == 'table' and p.state == 'running' then return true end + end + return false + end, { timeout = 0.75, interval = 0.01 }) + if not ready then diag:fail('expected config to reach running') end + + local req_conn = bus:connect() + local reply, rerr = req_conn:call({ 'cmd', 'config', 'set' }, { + service = 'net', + data = { schema = 'devicecode.net/1', answer = 42 }, + }, { timeout = 0.25 }) + assert(reply ~= nil, tostring(rerr)) + assert(reply.ok == true) + + local degraded_index = nil + local saw_degraded = probe.wait_until(function() + for i = 1, #status_events do + local p = status_events[i] + if type(p) == 'table' and p.state == 'degraded' and tostring(p.reason) == 'persist_failed' then + degraded_index = i + return true + end + end + return false + end, { timeout = 0.75, interval = 0.01 }) + if not saw_degraded then diag:fail('expected degraded after failed persist') end + + local saw_recovered = probe.wait_until(function() + if degraded_index == nil then return false end + for i = degraded_index + 1, #status_events do + local p = status_events[i] + if type(p) == 'table' and p.state == 'running' then return true end + end + return false + end, { timeout = 0.75, interval = 0.01 }) + if not saw_recovered then diag:fail('expected recovery to running after retry') end + + local writes = {} + for i = 1, #fake_hal.calls do + local c = fake_hal.calls[i] + if c.method == 'write_state' then writes[#writes + 1] = c end + end + assert(#writes >= 2, 'expected at least two write_state attempts') + + local blob1 = decode_json(writes[1].req.data) + local blob2 = decode_json(writes[2].req.data) + assert(blob1.net.rev == 1) + assert(blob2.net.rev == 1) + assert(blob1.net.data.answer == 42) + assert(blob2.net.data.answer == 42) + end, { timeout = 1.5 }) +end + +return T diff --git a/tests/integration/devhost/dependency_uniformity_spec.lua b/tests/integration/devhost/dependency_uniformity_spec.lua new file mode 100644 index 00000000..91a013d9 --- /dev/null +++ b/tests/integration/devhost/dependency_uniformity_spec.lua @@ -0,0 +1,359 @@ +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local busmod = require 'bus' + +local runfibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' +local mainmod = require 'devicecode.main' + +local net_service = require 'services.net.service' +local net_config = require 'services.net.config' +local net_topics = require 'services.net.topics' +local update_service = require 'services.update.service' +local update_topics = require 'services.update.topics' +local ui_service = require 'services.ui.service' +local fabric = require 'services.fabric' +local fabric_topics = require 'services.fabric.topics' + +local T = {} + +local function assert_true(v, msg) if v ~= true then error(msg or ('expected true, got ' .. tostring(v)), 2) end end +local function assert_eq(a, b, msg) if a ~= b then error(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a)), 2) end end +local function assert_not_nil(v, msg) if v == nil then error(msg or 'expected non-nil', 2) end return v end + +local function net_cfg() + return { + schema = net_config.SCHEMA, + version = 1, + segments = { + lan = { kind = 'lan', vlan = 10, addressing = { ipv4 = { mode = 'static', cidr = '172.28.10.1/24' } } }, + }, + interfaces = { + lan_bridge = { kind = 'bridge', role = 'lan', segment = 'lan', members = { 'eth0' } }, + }, + wan = { members = {} }, + } +end + +local function ui_cfg() + return { + schema = 'devicecode.config/ui/1', + enabled = true, + http = { enabled = true, cap_id = 'main', host = '127.0.0.1', port = 8080 }, + sessions = { prune_interval = false }, + updates = { + upload = { + enabled = true, + max_bytes = 1024 * 1024, + require_auth = false, + component = 'mcu', + create_job = true, + start_job = true, + }, + commit = { require_auth = false }, + }, + } +end + +local function fabric_cfg() + return { + schema = fabric.config.SCHEMA, + local_node = 'node-a', + links = { + { + id = 'uart0', + peer_id = 'node-b', + transport = { source = 'uart', class = 'uart', id = 'main' }, + session = { hello_interval_s = 60, ping_interval_s = 60, liveness_timeout_s = 120 }, + bridge = {}, + transfer = {}, + }, + }, + } +end + +local function make_blocking_line_session() + return { + read_line_op = function () return sleep.sleep_op(3600):wrap(function () return nil, 'closed' end) end, + write_line_op = function () return fibers.always(true, nil) end, + flush_op = function () return fibers.always(true, nil) end, + terminate = function () return true, nil end, + } +end + +local function bind_http_listen(scope, bus, calls) + local conn = bus:connect() + local ep = assert(conn:bind({ 'cap', 'http', 'main', 'rpc', 'listen' }, { queue_len = 8 })) + local ok, err = scope:spawn(function () + while true do + local req = fibers.perform(ep:recv_op()) + if req == nil then return end + calls[#calls + 1] = req.payload or {} + req:reply({ listener = { + accept_op = function () return sleep.sleep_op(3600):wrap(function () return nil, 'closed' end) end, + terminate = function () return true, nil end, + } }) + end + end) + assert_true(ok, err) + conn:retain({ 'cap', 'http', 'main', 'status' }, { state = 'available', available = true }) + return conn +end + +local function bind_raw_uart_open(scope, bus, opts) + opts = opts or {} + local conn = bus:connect() + local ep = assert(conn:bind({ 'raw', 'host', 'uart', 'cap', 'uart', 'main', 'rpc', 'open' }, { queue_len = 8 })) + local calls = opts.calls or {} + local ok, err = scope:spawn(function () + while true do + local req = fibers.perform(ep:recv_op()) + if req == nil then return end + calls[#calls + 1] = req.payload or {} + if opts.no_route_once then + opts.no_route_once = false + req:reply({ ok = false, reason = { err = 'no_route' } }) + else + req:reply({ ok = true, reason = { session = make_blocking_line_session() } }) + end + end + end) + assert_true(ok, err) + conn:retain({ 'raw', 'host', 'uart', 'cap', 'uart', 'main', 'status' }, { state = 'available', available = true }) + return conn, calls +end + +local function bind_network_config_apply(scope, bus, calls) + local conn = bus:connect() + local ep = assert(conn:bind({ 'cap', 'network-config', 'main', 'rpc', 'apply' }, { queue_len = 8 })) + local ok, err = scope:spawn(function () + while true do + local req = fibers.perform(ep:recv_op()) + if req == nil then return end + local payload = req.payload or {} + calls[#calls + 1] = payload + req:reply({ ok = true, reason = { ok = true, applied = true, changed = true } }) + end + end) + assert_true(ok, err) + conn:retain({ 'cap', 'network-config', 'main', 'status' }, { state = 'available', available = true }) + return conn +end + +local function bind_fake_control_store(scope, bus, backing) + backing = backing or {} + local conn = bus:connect() + for _, method in ipairs({ 'list', 'get', 'put', 'delete' }) do + local ep = assert(conn:bind({ 'cap', 'control-store', 'update', 'rpc', method }, { queue_len = 8 })) + local loop_method = method + local ok, err = scope:spawn(function () + while true do + local req = fibers.perform(ep:recv_op()) + if req == nil then return end + local p = req.payload or {} + if loop_method == 'list' then + local keys, prefix = {}, p.prefix or '' + for k in pairs(backing) do if k:sub(1, #prefix) == prefix then keys[#keys + 1] = k end end + table.sort(keys) + req:reply({ ok = true, reason = keys }) + elseif loop_method == 'get' then + req:reply(backing[p.key] == nil and { ok = false, reason = 'not found' } or { ok = true, reason = backing[p.key] }) + elseif loop_method == 'put' then + backing[p.key] = p.data; req:reply({ ok = true, reason = nil }) + else + backing[p.key] = nil; req:reply({ ok = true, reason = nil }) + end + end + end) + assert_true(ok, err) + end + conn:retain({ 'cap', 'control-store', 'update', 'status' }, { state = 'available', available = true }) + return conn +end + + +local function retained_payload(conn, topic) + local view = conn:retained_view(topic) + local msg = view:get(topic) + view:close() + return msg and msg.payload or nil +end + +local function retain_artifact_store_available(bus) + local conn = bus:connect() + conn:retain({ 'cap', 'artifact-store', 'main', 'status' }, { state = 'available', available = true }) + return conn +end + +function T.ui_waits_for_http_capability_before_starting_listener() + runfibers.run(function (scope) + local bus = busmod.new() + local conn = bus:connect() + local reader = bus:connect() + local calls = {} + local child = assert(scope:child()) + assert_true(child:spawn(function () ui_service.start(conn, { config = ui_cfg() }) end)) + + local waiting = probe.wait_retained_payload(reader, { 'svc', 'ui', 'status' }, { timeout = 0.8, view_topic = { 'svc', 'ui', 'status' } }) + assert_not_nil(waiting) + assert_true(probe.wait_until(function () + local p = retained_payload(reader, { 'svc', 'ui', 'status' }) + return p and p.ready == false and p.listener_status == 'waiting_for_http' + end, { timeout = 0.8, interval = 0.01 }), 'ui should wait for HTTP') + + local cap_scope = assert(scope:child()) + bind_http_listen(cap_scope, bus, calls) + assert_true(probe.wait_until(function () + local view = reader:retained_view({ 'svc', 'ui', 'status' }) + local msg = view:get({ 'svc', 'ui', 'status' }) + view:close() + local p = msg and msg.payload + return p and p.ready == true and p.listener_status == 'running' + end, { timeout = 0.8, interval = 0.01 }), 'ui should start listener after HTTP appears') + assert_true(probe.wait_until(function () + return #calls == 1 + end, { timeout = 0.8, interval = 0.01 }), 'expected HTTP listen to be called after HTTP appears') + + child:cancel('test complete') + cap_scope:cancel('test complete') + fibers.perform(child:join_op()) + fibers.perform(cap_scope:join_op()) + end, { timeout = 2.0 }) +end + +function T.fabric_waits_for_raw_transport_and_recovers_after_route_missing() + runfibers.run(function (scope) + local bus = busmod.new() + local conn = bus:connect() + local reader = bus:connect() + local child = assert(scope:child()) + assert_true(child:spawn(function () fabric.start(conn, { config = fabric_cfg() }) end)) + + assert_true(probe.wait_until(function () + local view = reader:retained_view(fabric_topics.svc_status()) + local msg = view:get(fabric_topics.svc_status()) + view:close() + local p = msg and msg.payload + return p and p.state == 'waiting_for_dependency' + and p.dependencies and p.dependencies['transport:uart0'] + and p.dependencies['transport:uart0'].available == false + end, { timeout = 0.8, interval = 0.01 }), 'fabric should wait for raw transport') + + local cap_scope = assert(scope:child()) + local calls = {} + bind_raw_uart_open(cap_scope, bus, { calls = calls, no_route_once = true }) + + assert_true(probe.wait_until(function () + local view = reader:retained_view(fabric_topics.svc_status()) + local msg = view:get(fabric_topics.svc_status()) + view:close() + local p = msg and msg.payload + local dep = p and p.dependencies and p.dependencies['transport:uart0'] + return p and p.state == 'waiting_for_dependency' + and p.reason == 'transport_route_missing' + and dep and dep.route_missing == true + end, { timeout = 0.8, interval = 0.01 }), 'fabric should attribute no_route to transport dependency') + + -- Retaining a fresh available status clears route_missing and admits the pending generation again. + bus:connect():retain({ 'raw', 'host', 'uart', 'cap', 'uart', 'main', 'status' }, { state = 'available', available = true }) + assert_true(probe.wait_until(function () + local view = reader:retained_view(fabric_topics.svc_status()) + local msg = view:get(fabric_topics.svc_status()) + view:close() + local p = msg and msg.payload + return p and p.state == 'running' and p.ready == true + end, { timeout = 0.8, interval = 0.01 }), 'fabric should retry after transport route returns') + assert_true(probe.wait_until(function () + return #calls >= 2 + end, { timeout = 0.8, interval = 0.01 }), 'expected transport open to be retried') + + child:cancel('test complete') + cap_scope:cancel('test complete') + fibers.perform(child:join_op()) + fibers.perform(cap_scope:join_op()) + end, { timeout = 2.0 }) +end + +local function fake_config_service(configs) + return { + start = function (conn) + for service_id, payload in pairs(configs) do + conn:retain({ 'cfg', service_id }, payload) + end + while true do sleep.sleep(3600) end + end, + } +end + +local function fake_hal_service() + return { + start = function (conn) + while true do sleep.sleep(3600) end + end, + } +end + +local function make_loader(configs) + return function (name) + if name == 'config' then return fake_config_service(configs) end + if name == 'hal' then return fake_hal_service() end + return require('services.' .. name) + end +end + +function T.real_main_launches_modern_services_with_fake_hal_and_config_dependencies_delayed() + runfibers.run(function (scope) + local bus = busmod.new() + local conn = bus:connect() + local configs = { + net = net_cfg(), + update = { schema = 'devicecode.update/1', components = { { component = 'cm5' } } }, + ui = ui_cfg(), + fabric = fabric_cfg(), + } + local main_child = assert(scope:child()) + assert_true(scope:spawn(function () + mainmod.run(main_child, { + env = 'dev', + services_csv = 'hal,config,net,update,ui,fabric', + bus = bus, + service_loader = make_loader(configs), + }) + end)) + + -- Give config replay time to put dependent services into waiting states. + assert_true(probe.wait_until(function () + local net = retained_payload(conn, net_topics.summary()) + local upd = conn:call(update_topics.update_manager_rpc('status'), {}, { timeout = 0.05 }) + return net and net.state == 'waiting_for_hal' + and upd and upd.snapshot and upd.snapshot.state == 'waiting_for_job_store' + end, { timeout = 0.8, interval = 0.01 }), 'expected real launch to reach dependency waiting states') + + local cap_scope = assert(scope:child()) + local net_calls = {} + bind_network_config_apply(cap_scope, bus, net_calls) + bind_fake_control_store(cap_scope, bus, {}) + retain_artifact_store_available(bus) + bind_http_listen(cap_scope, bus, {}) + bind_raw_uart_open(cap_scope, bus, {}) + + assert_true(probe.wait_until(function () + local net = retained_payload(conn, net_topics.summary()) + local upd = conn:call(update_topics.update_manager_rpc('status'), {}, { timeout = 0.05 }) + local ui = retained_payload(conn, { 'svc', 'ui', 'status' }) + local fab = retained_payload(conn, fabric_topics.svc_status()) + return net and net.state == 'running' + and upd and upd.snapshot and upd.snapshot.state == 'running' + and ui and ui.ready == true + and fab and fab.state == 'running' + end, { timeout = 1.2, interval = 0.01 }), 'expected delayed dependencies to admit modern services') + assert_eq(#net_calls, 1) + + main_child:cancel('test complete') + cap_scope:cancel('test complete') + fibers.perform(main_child:join_op()) + fibers.perform(cap_scope:join_op()) + end, { timeout = 3.0 }) +end + +return T diff --git a/tests/integration/devhost/device_public_seams_spec.lua b/tests/integration/devhost/device_public_seams_spec.lua new file mode 100644 index 00000000..d1c11739 --- /dev/null +++ b/tests/integration/devhost/device_public_seams_spec.lua @@ -0,0 +1,228 @@ +-- tests/integration/devhost/device_public_seams_spec.lua +-- +-- Device public seam tests. These drive the service through retained raw truth +-- and public component capability RPCs, without touching coordinator internals. + +local busmod = require 'bus' +local fibers = require 'fibers' + +local runfibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' + +local device_service = require 'services.device.service' +local topics = require 'services.device.topics' +local device_config = require 'services.device.config' +local fabric_topics = require 'services.fabric.topics' + +local T = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end + +local function wait_retained_payload_where(conn, topic, label, pred, opts) + opts = opts or {} + local view = conn:retained_view(topic) + local value = probe.wait_versioned_until(label, function () + return view:version() + end, function (seen) + return view:changed_op(seen) + end, function () + local msg = view:get(topic) + local payload = msg and msg.payload or nil + if pred(payload) then return payload end + return nil + end, opts) + view:close() + return value +end + + +local function mcu_config(extra) + extra = extra or {} + local actions = extra.actions or { + ['stage-update'] = { + kind = 'fabric_stage', + target = 'updater/main', + }, + } + return { + schema = device_config.SCHEMA, + components = { + mcu = { + class = 'member', + subtype = 'mcu', + member = 'mcu', + facts = { + software = topics.raw_member_state('mcu', 'software'), + updater = topics.raw_member_state('mcu', 'updater'), + }, + actions = actions, + }, + }, + } +end + +local function start_device(scope, params) + params = params or {} + local bus = params.bus or busmod.new() + local svc_conn = bus:connect() + local caller = bus:connect() + local child = assert(scope:child()) + local ok, err = child:spawn(function () + device_service.start(svc_conn, { + watch_config = false, + initial_config = params.config or mcu_config(params.config_extra), + enable_observers = true, + enable_actions = true, + auto_publish = true, + fabric_client = params.fabric_client, + action_timeout = params.action_timeout or 1.0, + }) + end) + assert_true(ok, err) + + probe.wait_retained_payload(caller, topics.components(), { timeout = 1.0 }) + return { + bus = bus, + child = child, + caller = caller, + svc_conn = svc_conn, + } +end + +local function transfer_client_for(conn) + return { + send_blob_op = function (_, params, opts) + assert(params and params.source_owner, 'source_owner required') + return conn:call_op(fabric_topics.transfer_manager_rpc('send-blob'), { + source_owner = params.source_owner, + meta = params.meta, + target = params.target, + link_id = params.link_id, + size = params.size, + digest_alg = params.digest_alg, + digest = params.digest, + }, { timeout = opts and opts.timeout or 1.0 }):wrap(function (reply, err) + if reply == nil then return nil, err end + return reply.result or reply + end) + end, + } +end + +function T.device_observes_raw_member_truth_and_publishes_canonical_component_state() + runfibers.run(function(scope) + local h = start_device(scope) + local conn = h.caller + + conn:retain(topics.raw_member_state('mcu', 'software'), { + version = '1.2.3', + build_id = 'abc123', + image_id = 'mcu-image-1', + boot_id = 'mcu-boot-1', + }) + conn:retain(topics.raw_member_state('mcu', 'updater'), { + state = 'ready', + last_error = nil, + pending_version = nil, + pending_image_id = nil, + staged_image_id = nil, + job_id = nil, + }) + + local component = wait_retained_payload_where(conn, topics.component('mcu'), 'mcu component software observed', function (p) + return p and p.software and p.software.version == '1.2.3' and p.updater and p.updater.state == 'ready' + end, { timeout = 1.0 }) + assert_eq(component.kind, 'device.component') + assert_eq(component.component, 'mcu') + assert_eq(component.software.version, '1.2.3') + assert_eq(component.updater.state, 'ready') + + local software = wait_retained_payload_where(conn, topics.component_software('mcu'), 'mcu software projection observed', function (p) + return p and p.version == '1.2.3' + end, { timeout = 1.0 }) + assert_eq(software.kind, 'device.component.software') + assert_eq(software.component, 'mcu') + assert_eq(software.version, '1.2.3') + + local meta = probe.wait_retained_payload(conn, topics.component_cap_meta('mcu'), { timeout = 1.0 }) + assert_eq(meta.owner, 'device') + assert_eq(meta.canonical_state[1], 'state') + assert_eq(meta.canonical_state[2], 'device') + assert_eq(meta.canonical_state[3], 'component') + assert_eq(meta.canonical_state[4], 'mcu') + assert_eq(meta.backing.facts.software[1], 'raw') + assert_eq(meta.backing.facts.software[2], 'member') + assert_eq(meta.backing.facts.software[3], 'mcu') + + h.child:cancel('test complete') + end, { timeout = 5.0 }) +end + +function T.device_stage_update_calls_public_transfer_manager_capability() + runfibers.run(function(scope) + local bus = busmod.new() + local caller = bus:connect() + local source_terminated = 0 + local seen_request + + local transfer_ep = caller:bind(fabric_topics.transfer_manager_rpc('send-blob'), { + queue_len = 4, + }) + local ok, err = scope:spawn(function () + local req = transfer_ep:recv() + seen_request = req and req.payload + if seen_request and seen_request.source_owner then + local source, herr = seen_request.source_owner:detach() + assert_not_nil(source, herr) + end + if req then + req:reply({ + ok = true, + result = { + ok = true, + reply_payload = { staged = true, target = seen_request and seen_request.target }, + staged = true, + }, + }) + end + end) + assert_true(ok, err) + + local h = start_device(scope, { + bus = bus, + fabric_client = transfer_client_for(caller), + }) + local conn = h.caller + probe.wait_retained_payload(conn, topics.component_cap_meta('mcu'), { timeout = 1.0 }) + local source = { + read_chunk_op = function () return fibers.always(nil, nil) end, + terminate = function () source_terminated = source_terminated + 1; return true, nil end, + } + + local reply, call_err = conn:call(topics.component_cap_rpc('mcu', 'stage-update'), { + source = source, + }, { timeout = 1.0 }) + assert_not_nil(reply, call_err) + assert_eq(reply.staged, true) + assert_eq(reply.target, 'updater/main') + assert_not_nil(seen_request) + assert_eq(seen_request.target, 'updater/main') + assert_eq(source_terminated, 0, 'device must release cleanup after source-owner transfer') + + local component = wait_retained_payload_where(conn, topics.component('mcu'), 'mcu stage action recorded', function (p) + return p and p.last_action and p.last_action.action == 'stage-update' + end, { timeout = 1.0 }) + assert_not_nil(component.last_action) + assert_eq(component.last_action.action, 'stage-update') + assert_eq(component.last_action.ok, true) + assert_nil(component.last_action.result.source_owner) + + h.child:cancel('test complete') + end, { timeout = 5.0 }) +end + +return T diff --git a/tests/integration/devhost/fabric_hal_uart_smoke_spec.lua b/tests/integration/devhost/fabric_hal_uart_smoke_spec.lua new file mode 100644 index 00000000..52757dce --- /dev/null +++ b/tests/integration/devhost/fabric_hal_uart_smoke_spec.lua @@ -0,0 +1,318 @@ +-- tests/integration/devhost/fabric_hal_uart_smoke_spec.lua +-- +-- Devhost smoke test for Fabric over the real HAL UART manager/driver path. +-- +-- This deliberately follows integration.devhost.hal_uart_spec for the UART +-- manager setup. The only extra layer is a small in-test raw-host bus adapter +-- that exposes the emitted manager capability at the endpoint Fabric already +-- knows how to call: +-- +-- raw/host//cap/uart//rpc/open +-- +-- This keeps the smoke test narrow: +-- +-- fabric.start -> raw-host bus call -> real HAL UART session -> PTY hello +-- +-- It does not try to test the full HAL service config/publication path. + +local busmod = require 'bus' +local fibers = require 'fibers' +local channel = require 'fibers.channel' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local safe = require 'coxpcall' + +local hal_types = require 'services.hal.types.core' +local cap_args = require 'services.hal.types.capability_args' + +local runfibers = require 'tests.support.run_fibers' +local pty = require 'tests.support.pty' + +local fabric = require 'services.fabric' +local protocol = require 'services.fabric.protocol' +local hal_transport = require 'services.fabric.hal_transport' + +local perform = fibers.perform +local T = {} + +local function assert_eq(a, b, msg) + assert(a == b, msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) +end + +local function fresh_manager() + package.loaded['services.hal.managers.uart'] = nil + package.loaded['services.hal.drivers.uart'] = nil + return require('services.hal.managers.uart') +end + +local function dummy_logger() + local logger = {} + for _, k in ipairs({ 'debug', 'info', 'warn', 'error' }) do + logger[k] = function() end + end + function logger:child() + return self + end + return logger +end + +local function wait_channel_get(ch, timeout_s, what) + local which, a, b = perform(op.named_choice({ + item = ch:get_op(), + timeout = sleep.sleep_op(timeout_s or 1.0):wrap(function() + return true + end), + })) + + if which == 'timeout' then + error(('timed out waiting for %s'):format(what or 'channel item'), 0) + end + + if a == nil then + error(('channel closed while waiting for %s: %s'):format(what or 'channel item', tostring(b)), 0) + end + + return a +end + +local function wait_device_event(dev_ev_ch, event_type, class, id, timeout_s) + local deadline = fibers.now() + (timeout_s or 1.0) + + while fibers.now() < deadline do + local ev = wait_channel_get(dev_ev_ch, deadline - fibers.now(), 'device event') + if ev.event_type == event_type and ev.class == class and ev.id == id then + return ev + end + end + + error(('timed out waiting for device event %s %s/%s'):format( + tostring(event_type), tostring(class), tostring(id) + ), 0) +end + +local function start_manager(scope) + local uart_mgr = fresh_manager() + + local dev_ev_ch = channel.new(16) + local cap_emit_ch = channel.new(32) + + local ok, err = perform(uart_mgr.start_op(dummy_logger(), dev_ev_ch, cap_emit_ch)) + assert(ok == true, tostring(err)) + + -- This mirrors the existing devhost UART tests. The UART manager currently + -- exposes only an Op-based stop path, so the test follows the established + -- integration-test cleanup style rather than pretending this is a Fabric + -- service finaliser pattern. + scope:finally(function() + safe.pcall(function() + perform(uart_mgr.shutdown_op()) + end) + end) + + return uart_mgr, dev_ev_ch, cap_emit_ch +end + +local function apply_uart_config(uart_mgr, port) + local ok_cfg, err_cfg = perform(uart_mgr.apply_config_op({ + serial_ports = { + { + id = 'uart0', + path = port.slave_name, + baud = 115200, + mode = '8N1', + }, + }, + })) + assert(ok_cfg == true, tostring(err_cfg)) +end + +local function wait_uart_cap(dev_ev_ch) + local added = wait_device_event(dev_ev_ch, 'added', 'uart', 'uart0', 1.5) + assert(type(added.capabilities) == 'table' and #added.capabilities == 1) + local cap = added.capabilities[1] + assert(cap.class == 'uart') + assert(cap.id == 'uart0') + assert(type(cap.control_ch) == 'table', 'UART capability should expose control_ch') + return cap +end + +local function normalise_uart_open_opts(opts) + if opts == nil or getmetatable(opts) ~= cap_args.UARTOpenOpts then + local open_opts, err = cap_args.new.UARTOpenOpts(opts) + assert(open_opts, tostring(err)) + return open_opts + end + return opts +end + +local function call_hal_control(cap, verb, opts) + local reply_ch = channel.new(1) + local req, err = hal_types.new.ControlRequest(verb, opts or {}, reply_ch) + assert(req, tostring(err)) + + perform(cap.control_ch:put_op(req)) + + local reply = wait_channel_get(reply_ch, 1.0, 'HAL control reply') + assert(type(reply) == 'table', 'HAL control reply must be a table') + assert(type(reply.ok) == 'boolean', 'HAL control reply ok must be boolean') + + return reply +end + +local function expose_raw_host_uart_open(scope, conn, cap, source) + source = source or 'uart_manager' + + local ep = conn:bind({ 'raw', 'host', source, 'cap', 'uart', cap.id, 'rpc', 'open' }, { + queue_len = 4, + }) + + local ok_spawn, spawn_err = scope:spawn(function() + while true do + local req = ep:recv() + if req == nil then + return + end + + local open_opts = normalise_uart_open_opts(req.payload) + local reply = call_hal_control(cap, 'open', open_opts) + + local replied = req:reply(reply) + if not replied then + -- The Fabric side may have timed out or been cancelled. The HAL + -- session returned by a successful open is owned by that reply path + -- only if the reply is accepted; close it immediately otherwise. + if reply.ok == true + and type(reply.reason) == 'table' + and type(reply.reason.session) == 'table' + and type(reply.reason.session.terminate) == 'function' + then + reply.reason.session:terminate('fabric request abandoned') + end + end + end + end) + assert(ok_spawn, tostring(spawn_err)) + + scope:finally(function() + safe.pcall(function() + ep:unbind() + end) + end) + + return { + source = source, + class = 'uart', + id = cap.id, + } +end + +local function fabric_uart_config(raw_cap) + return { + schema = fabric.config.SCHEMA, + local_node = 'devhost-node', + links = { + { + id = 'uart-link', + peer_id = 'peer-node', + transport = { + source = raw_cap.source, + class = raw_cap.class, + id = raw_cap.id, + }, + session = { + hello_interval_s = 0.10, + ping_interval_s = 5.0, + liveness_timeout_s = 5.0, + }, + bridge = {}, + }, + }, + } +end + +local function wait_decoded_line(port, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 2.0) + local buf = '' + + while fibers.now() < deadline do + local remain = deadline - fibers.now() + local which, chunk, err = perform(op.named_choice{ + data = port.master:read_some_op(4096), + timeout = sleep.sleep_op(remain), + }) + + if which == 'timeout' then + break + end + + if chunk == nil and err ~= nil then + error(('PTY read failed while waiting for %s: %s'):format( + tostring(label or 'Fabric frame'), + tostring(err) + ), 0) + end + + if chunk ~= nil then + buf = buf .. chunk + while true do + local line, rest = buf:match('^(.-)\n(.*)$') + if not line then break end + buf = rest + + if line ~= '' then + local frame, derr = protocol.decode_line(line) + if frame then + return frame + end + error(('invalid Fabric line while waiting for %s: %s; line=%q'):format( + tostring(label or 'frame'), + tostring(derr), + tostring(line) + ), 0) + end + end + end + end + + error(('timed out waiting for %s; buffered=%q'):format( + tostring(label or 'Fabric frame'), + buf + ), 0) +end + +function T.fabric_start_opens_real_hal_uart_manager_session_and_writes_hello() + runfibers.run(function(scope) + local uart_mgr, dev_ev_ch = start_manager(scope) + local port = pty.open(scope) + + apply_uart_config(uart_mgr, port) + local cap = wait_uart_cap(dev_ev_ch) + + local bus = busmod.new() + local raw_cap = expose_raw_host_uart_open(scope, bus:connect(), cap, 'uart_manager') + + local ok_fabric, fabric_err = scope:spawn(function() + local fabric_conn = bus:connect() + fabric.start(fabric_conn, { + name = 'fabric', + env = 'test', + config = fabric_uart_config(raw_cap), + link_overrides = { + ['uart-link'] = { + open_transport_op = function () + return hal_transport.open_transport_op(fabric_conn, raw_cap) + end, + }, + }, + }) + end) + assert(ok_fabric, tostring(fabric_err)) + + local frame = wait_decoded_line(port, 2.0, 'Fabric hello on HAL UART PTY') + assert_eq(frame.type, 'hello') + assert_eq(frame.node, 'devhost-node') + assert(type(frame.sid) == 'string' and frame.sid ~= '', 'hello should include a sid') + end, { timeout = 5.0 }) +end + +return T diff --git a/tests/integration/devhost/fabric_public_service_path_spec.lua b/tests/integration/devhost/fabric_public_service_path_spec.lua new file mode 100644 index 00000000..3362161e --- /dev/null +++ b/tests/integration/devhost/fabric_public_service_path_spec.lua @@ -0,0 +1,709 @@ +-- tests/integration/devhost/fabric_public_service_path_spec.lua +-- +-- Whole public Fabric service-path integration tests on devhost fakes. +-- +-- These deliberately exercise the public configured-link entry point: +-- +-- fabric.start(conn, opts) +-- -> service shell +-- -> compiled generation +-- -> composed link +-- -> local bus bridge runtime +-- -> transfer manager +-- -> JSONL line transport +-- +-- They use a deterministic in-memory JSONL line transport rather than the HAL UART +-- driver. The separate fabric_hal_uart_smoke_spec covers the current HAL UART +-- open path. + +local busmod = require 'bus' +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' + +local probe = require 'tests.support.bus_probe' +local runfibers = require 'tests.support.run_fibers' + +local fabric = require 'services.fabric' +local protocol = require 'services.fabric.protocol' +local topics = require 'services.fabric.topics' +local hal_transport = require 'services.fabric.hal_transport' + +local T = {} + +local function assert_eq(a, b, msg) + assert(a == b, msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) +end + +local function assert_true(v, msg) + assert(v == true, msg or ('expected true, got ' .. tostring(v))) +end + + +local function join_topic(topic) + if type(topic) ~= 'table' then return tostring(topic) end + local parts = {} + for i = 1, #topic do parts[#parts + 1] = tostring(topic[i]) end + return table.concat(parts, '/') +end + +local function compact_value(v, depth) + depth = depth or 0 + local tv = type(v) + if tv == 'string' then + return string.format('%q', v) + end + if tv ~= 'table' then + return tostring(v) + end + if depth >= 2 then return '{...}' end + local parts = {} + local n = 0 + for i = 1, #v do + n = n + 1 + parts[#parts + 1] = compact_value(v[i], depth + 1) + if n >= 8 then break end + end + for k, val in pairs(v) do + if type(k) ~= 'number' then + n = n + 1 + parts[#parts + 1] = tostring(k) .. '=' .. compact_value(val, depth + 1) + if n >= 12 then break end + end + end + return '{' .. table.concat(parts, ', ') .. '}' +end + +local function frame_summary(frame) + if type(frame) ~= 'table' then return tostring(frame) end + local parts = { tostring(frame.type or '?') } + if frame.sid then parts[#parts + 1] = 'sid=' .. tostring(frame.sid) end + if frame.node then parts[#parts + 1] = 'node=' .. tostring(frame.node) end + if frame.id then parts[#parts + 1] = 'id=' .. tostring(frame.id) end + if frame.xfer_id then parts[#parts + 1] = 'xfer_id=' .. tostring(frame.xfer_id) end + if frame.target then parts[#parts + 1] = 'target=' .. tostring(frame.target) end + if frame.offset ~= nil then parts[#parts + 1] = 'offset=' .. tostring(frame.offset) end + if frame.next ~= nil then parts[#parts + 1] = 'next=' .. tostring(frame.next) end + if frame.size ~= nil then parts[#parts + 1] = 'size=' .. tostring(frame.size) end + if frame.digest then parts[#parts + 1] = 'digest=' .. tostring(frame.digest) end + if frame.chunk_digest then parts[#parts + 1] = 'chunk_digest=' .. tostring(frame.chunk_digest) end + if frame.topic then parts[#parts + 1] = 'topic=' .. join_topic(frame.topic) end + if frame.ok ~= nil then parts[#parts + 1] = 'ok=' .. tostring(frame.ok) end + if frame.err then parts[#parts + 1] = 'err=' .. tostring(frame.err) end + return table.concat(parts, ' ') +end + +local function chat(label, value) + local msg = '[fabric-pair-send-blob] ' .. tostring(label) + if value ~= nil then + if type(value) == 'table' and value.type then + msg = msg .. ': ' .. frame_summary(value) + else + msg = msg .. ': ' .. compact_value(value) + end + end + io.stderr:write(msg .. '\n') +end + +local function dump_transport(label, transport) + chat(label .. ' written_count', #transport.written) + for i, frame in ipairs(transport.written) do + io.stderr:write(('[fabric-pair-send-blob] %s[%d] %s\n'):format(label, i, frame_summary(frame))) + end + chat(label .. ' flushes', transport.flushes and transport.flushes() or '?') + chat(label .. ' closes', transport.closes and transport.closes() or '?') + chat(label .. ' close_reason', transport.close_reason and transport.close_reason() or nil) +end + +local function trace_transport(label, transport) + local orig = transport.session.write_line_op + function transport.session:write_line_op(line) + local frame = protocol.decode_line(line) + if frame then + chat(label .. ' write_line', frame) + else + chat(label .. ' write_line undecodable', line) + end + return orig(self, line) + end +end + +local function copy_frame(frame) + local out = {} + for k, v in pairs(frame or {}) do + if type(v) == 'table' then + local t = {} + for k2, v2 in pairs(v) do t[k2] = v2 end + out[k] = t + else + out[k] = v + end + end + return out +end + +local function new_line_transport() + local in_tx, in_rx = mailbox.new(64, { full = 'reject_newest' }) + local written = {} + local flushes = 0 + local closes = 0 + local close_reason + + local session = {} + + function session:read_line_op() + return in_rx:recv_op():wrap(function(frame) + if frame == nil then + return nil, in_rx:why() or 'closed' + end + + local line, err = protocol.encode_line(frame) + if not line then + return nil, err + end + return line, nil + end) + end + + function session:write_line_op(line) + local frame, err = protocol.decode_line(line) + if not frame then + return fibers.always(nil, err) + end + written[#written + 1] = copy_frame(frame) + return fibers.always(true, nil) + end + + function session:flush_op() + flushes = flushes + 1 + return fibers.always(true, nil) + end + + function session:terminate(reason) + closes = closes + 1 + close_reason = reason + in_tx:close(reason or 'transport terminated') + return true, nil + end + + return { + session = session, + in_tx = in_tx, + written = written, + flushes = function() return flushes end, + closes = function() return closes end, + close_reason = function() return close_reason end, + } +end + + +local function wait_retained_payload_where(conn, topic, label, pred, opts) + opts = opts or {} + local view = conn:retained_view(topic) + local value = probe.wait_versioned_until(label, function () + return view:version() + end, function (seen) + return view:changed_op(seen) + end, function () + local msg = view:get(topic) + local payload = msg and msg.payload or nil + if pred(payload) then return payload end + return nil + end, opts) + view:close() + return value +end + +local function wait_written(label, transport, predicate, opts) + opts = opts or {} + local last + local ok = probe.wait_until(function() + for i = 1, #transport.written do + local frame = transport.written[i] + if predicate(frame, i) then + last = frame + return true + end + end + return false + end, { timeout = opts.timeout or 1.5, interval = opts.interval or 0.002 }) + + if not ok then + error('timed out waiting for written frame: ' .. tostring(label), 0) + end + + return last +end + +local function send_inbound(transport, frame) + local ok, err = fibers.perform(transport.in_tx:send_op(frame)) + assert_true(ok, tostring(err)) +end + +local function base_config(extra_link) + extra_link = extra_link or {} + + local link = { + id = 'link-a', + peer_id = 'node-b', + transport = { + source = 'test', + class = 'jsonl', + id = 'link-a', + }, + session = { + hello_interval_s = 5.0, + ping_interval_s = 5.0, + liveness_timeout_s = 5.0, + }, + bridge = extra_link.bridge or {}, + transfer = extra_link.transfer, + queues = extra_link.queues, + } + + return { + schema = fabric.config.SCHEMA, + local_node = 'node-a', + links = { link }, + } +end + +local function wrap_session_op(session) + local wrapped, err = hal_transport.wrap_transport(session) + return fibers.always(wrapped, err) +end + +local function default_link_overrides(transport) + if not transport then return nil end + return { + ['link-a'] = { + open_transport_op = function () + return wrap_session_op(transport.session) + end, + }, + } +end + +local function start_public_fabric(scope, conn, cfg, transport, opts) + opts = opts or {} + local ok, err = scope:spawn(function() + fabric.start(conn, { + name = opts.name or 'fabric', + env = 'test', + config = cfg, + link_overrides = opts.link_overrides or default_link_overrides(transport), + config_queue_len = opts.config_queue_len, + }) + end) + assert_true(ok, tostring(err)) +end + +local function link_config(params) + params = params or {} + return { + schema = fabric.config.SCHEMA, + local_node = params.local_node or 'node-a', + links = { + { + id = params.id or 'link-a', + peer_id = params.peer_id or 'node-b', + transport = { + source = 'test', + class = 'jsonl', + id = params.id or 'link-a', + }, + session = params.session or { + hello_interval_s = 5.0, + ping_interval_s = 5.0, + liveness_timeout_s = 5.0, + }, + bridge = params.bridge or {}, + transfer = params.transfer, + queues = params.queues, + }, + }, + } +end + +local function connect_line_transports(a, b) + local write_a = a.session.write_line_op + function a.session:write_line_op(line) + local frame = assert(protocol.decode_line(line)) + write_a(self, line) + return b.in_tx:send_op(copy_frame(frame)) + end + + local write_b = b.session.write_line_op + function b.session:write_line_op(line) + local frame = assert(protocol.decode_line(line)) + write_b(self, line) + return a.in_tx:send_op(copy_frame(frame)) + end +end + +function T.fabric_start_bridges_local_and_remote_retained_state_over_composed_link() + runfibers.run(function(scope) + local bus = busmod.new() + local conn = bus:connect() + local transport = new_line_transport() + + conn:retain({ 'local', 'retained', 'flag' }, { enabled = true }) + + local cfg = base_config({ + bridge = { + imports = { + { ['local'] = { 'mirror' }, remote = { 'remote', 'in' } }, + }, + exports = { + { + ['local'] = { 'local', 'retained' }, + remote = { 'remote', 'out' }, + publish = false, + retain = true, + }, + }, + }, + }) + + start_public_fabric(scope, conn, cfg, transport) + + wait_written('initial hello', transport, function(frame) + return frame.type == 'hello' and frame.node == 'node-a' + end) + + -- Retained exports are now session-scoped. They are replayed only after + -- fabric.session has established a peer session. + send_inbound(transport, assert(protocol.hello_ack('peer-sid', 'node-b'))) + + local retained_pub = wait_written('exported retained local bus state', transport, function(frame) + return frame.type == 'pub' + and frame.retain == true + and frame.topic[1] == 'remote' + and frame.topic[2] == 'out' + and frame.topic[3] == 'flag' + end) + assert_eq(retained_pub.payload.enabled, true) + + send_inbound(transport, assert(protocol.pub({ 'remote', 'in', 'status' }, { online = true }, true))) + + local mirrored = probe.wait_retained_payload(conn, { 'mirror', 'status' }, { + timeout = 1.5, + }) + assert_eq(mirrored.online, true) + end, { timeout = 4.0 }) +end + +function T.fabric_start_pair_bridges_publish_retained_and_rpc_over_composed_links() + runfibers.run(function(scope) + local bus_a = busmod.new() + local bus_b = busmod.new() + local conn_a = bus_a:connect() + local conn_b = bus_b:connect() + local transport_a = new_line_transport() + local transport_b = new_line_transport() + connect_line_transports(transport_a, transport_b) + + local cfg_a = link_config({ + local_node = 'node-a', + id = 'link-a', + peer_id = 'node-b', + bridge = { + exports = { + { + ['local'] = { 'a', 'pub' }, + remote = { 'wire', 'pub' }, + publish = true, + retain = true, + }, + }, + rpc = { + outbound = { + { + ['local'] = { 'a', 'rpc', 'echo' }, + remote = { 'svc', 'echo' }, + timeout_s = 1.0, + }, + }, + }, + }, + }) + + local cfg_b = link_config({ + local_node = 'node-b', + id = 'link-a', + peer_id = 'node-a', + bridge = { + imports = { + { + ['local'] = { 'b', 'pub' }, + remote = { 'wire', 'pub' }, + }, + }, + rpc = { + inbound = { + { + ['local'] = { 'b', 'rpc', 'echo' }, + remote = { 'svc', 'echo' }, + timeout_s = 1.0, + }, + }, + }, + }, + }) + + start_public_fabric(scope, conn_a, cfg_a, transport_a) + start_public_fabric(scope, conn_b, cfg_b, transport_b) + + wait_written('node-a hello', transport_a, function(frame) + return frame.type == 'hello' and frame.node == 'node-a' + end) + wait_written('node-b hello', transport_b, function(frame) + return frame.type == 'hello' and frame.node == 'node-b' + end) + wait_written('node-a hello_ack', transport_a, function(frame) + return frame.type == 'hello_ack' and frame.node == 'node-a' + end) + wait_written('node-b hello_ack', transport_b, function(frame) + return frame.type == 'hello_ack' and frame.node == 'node-b' + end) + + local ep_b = conn_b:bind({ 'b', 'rpc', 'echo' }) + local ok_handler, handler_err = scope:spawn(function() + while true do + local req = ep_b:recv() + if not req then return end + req:reply({ echoed = req.payload }) + end + end) + assert_true(ok_handler, tostring(handler_err)) + + local sub_b = conn_b:subscribe({ 'b', 'pub', '#' }, { + queue_len = 8, + full = 'reject_newest', + }) + conn_a:publish({ 'a', 'pub', 'one' }, { value = 11 }) + local msg = probe.wait_message(conn_b, { 'b', 'pub', '#' }, { + timeout = 1.5, + sub = sub_b, + }) + sub_b:unsubscribe() + assert_eq(msg.topic[1], 'b') + assert_eq(msg.topic[2], 'pub') + assert_eq(msg.topic[3], 'one') + assert_eq(msg.payload.value, 11) + + conn_a:retain({ 'a', 'pub', 'answer' }, { value = 42 }) + local retained = probe.wait_retained_payload(conn_b, { 'b', 'pub', 'answer' }, { + timeout = 1.5, + }) + assert_eq(retained.value, 42) + + local value, err = conn_a:call({ 'a', 'rpc', 'echo' }, { hello = 'world' }, { + timeout = 1.5, + }) + assert(value ~= nil, tostring(err)) + assert_eq(value.echoed.hello, 'world') + end, { timeout = 5.0 }) +end + +function T.fabric_start_config_replacement_restarts_generation_and_closes_old_transport() + runfibers.run(function(scope) + local bus = busmod.new() + local conn = bus:connect() + local transport_a = new_line_transport() + local transport_b = new_line_transport() + + start_public_fabric(scope, conn, nil, nil, { + link_overrides = { + ['link-a'] = { open_transport_op = function () return wrap_session_op(transport_a.session) end }, + ['link-b'] = { open_transport_op = function () return wrap_session_op(transport_b.session) end }, + }, + }) + + conn:retain(topics.cfg(), link_config({ + id = 'link-a', + peer_id = 'node-b', + })) + + wait_written('first generation hello', transport_a, function(frame) + return frame.type == 'hello' and frame.node == 'node-a' + end) + + conn:retain(topics.cfg(), link_config({ + id = 'link-b', + peer_id = 'node-c', + })) + + wait_written('replacement generation hello', transport_b, function(frame) + return frame.type == 'hello' and frame.node == 'node-a' + end) + + local ok_closed = probe.wait_until(function() + return transport_a.closes() >= 1 + end, { timeout = 1.5, interval = 0.002 }) + assert_true(ok_closed, 'old generation transport should be closed after config replacement') + end, { timeout = 4.0 }) +end + +function T.fabric_exposes_public_transfer_manager_capability() + runfibers.run(function(scope) + local bus = busmod.new() + local conn = bus:connect() + local transport = new_line_transport() + + start_public_fabric(scope, conn, link_config({ + id = 'link-a', + peer_id = 'node-b', + }), transport) + + local meta = probe.wait_retained_payload(conn, topics.transfer_manager_meta(), { + timeout = 1.5, + }) + assert_eq(meta.owner, 'fabric') + assert_eq(meta.class, 'transfer-manager') + assert_eq(meta.methods[1], 'send-blob') + + local status = wait_retained_payload_where(conn, topics.transfer_manager_status(), 'fabric transfer manager available', function (p) + return p and p.available == true + end, { timeout = 1.5 }) + assert_eq(status.available, true) + assert_eq(status.links[1], 'link-a') + + local reply, err = conn:call(topics.transfer_manager_rpc('send-blob'), { + link_id = 'missing-link', + request_id = 'xfer-missing-link', + target = 'mcu', + size = 0, + digest = protocol.digest_hex(''), + data = '', + }, { timeout = 1.0 }) + assert_eq(reply, nil) + assert_eq(err, 'link_not_ready') + end, { timeout = 4.0 }) +end + + +function T.fabric_pair_sends_blob_to_registered_remote_transfer_target() + runfibers.run(function(scope) + local bus_a = busmod.new() + local bus_b = busmod.new() + local conn_a = bus_a:connect() + local conn_b = bus_b:connect() + local transport_a = new_line_transport() + local transport_b = new_line_transport() + connect_line_transports(transport_a, transport_b) + trace_transport('node-a transport', transport_a) + trace_transport('node-b transport', transport_b) + chat('created paired in-memory line transports') + + local received = {} + local committed = false + local target = {} + function target:open_sink_op(req) + chat('remote target open_sink_op', req) + assert_eq(req.target, 'updater/main') + assert_eq(req.size, 6) + assert_eq(req.digest, protocol.digest_hex('abcdef')) + local sink = {} + function sink:append_op(chunk) + chat('remote target append_op', { chunk = chunk, received_before = table.concat(received) }) + received[#received + 1] = chunk + return fibers.always(true, nil) + end + function sink:commit_op(req2) + chat('remote target commit_op', req2) + committed = true + return fibers.always({ staged = true, digest = req2.digest }, nil) + end + function sink:abort(reason) + chat('remote target abort', reason) + return true, nil + end + return fibers.always(sink, nil) + end + + local cfg_a = link_config({ + local_node = 'node-a', + id = 'link-a', + peer_id = 'node-b', + transfer = { chunk_size = 3, timeout_s = 1.0 }, + }) + local cfg_b = link_config({ + local_node = 'node-b', + id = 'link-a', + peer_id = 'node-a', + transfer = { chunk_size = 3, timeout_s = 1.0 }, + }) + + chat('starting node-a fabric service') + start_public_fabric(scope, conn_a, cfg_a, transport_a) + chat('starting node-b fabric service with receive target updater/main') + start_public_fabric(scope, conn_b, cfg_b, transport_b, { + link_overrides = { + ['link-a'] = { + open_transport_op = function () return wrap_session_op(transport_b.session) end, + transfer = { + chunk_size = 3, + timeout_s = 1.0, + receive_targets = { ['updater/main'] = target }, + }, + }, + }, + }) + + chat('waiting for fabric handshakes') + chat('saw handshake', wait_written('node-a hello', transport_a, function(frame) + return frame.type == 'hello' and frame.node == 'node-a' + end)) + chat('saw handshake', wait_written('node-b hello', transport_b, function(frame) + return frame.type == 'hello' and frame.node == 'node-b' + end)) + chat('saw handshake', wait_written('node-a hello_ack', transport_a, function(frame) + return frame.type == 'hello_ack' and frame.node == 'node-a' + end)) + chat('saw handshake', wait_written('node-b hello_ack', transport_b, function(frame) + return frame.type == 'hello_ack' and frame.node == 'node-b' + end)) + + chat('waiting for node-a transfer manager status') + chat('node-a transfer manager status', wait_retained_payload_where(conn_a, topics.transfer_manager_status(), 'node-a transfer manager available', function (p) + return p and p.available == true + end, { timeout = 1.5 })) + chat('waiting for node-b transfer manager status') + chat('node-b transfer manager status', wait_retained_payload_where(conn_b, topics.transfer_manager_status(), 'node-b transfer manager available', function (p) + return p and p.available == true + end, { timeout = 1.5 })) + + chat('calling node-a transfer manager send-blob') + local reply, err = conn_a:call(topics.transfer_manager_rpc('send-blob'), { + link_id = 'link-a', + request_id = 'xfer-real-fabric', + xfer_id = 'xfer-real-fabric', + target = 'updater/main', + size = 6, + digest = protocol.digest_hex('abcdef'), + data = 'abcdef', + chunk_size = 3, + timeout_s = 2.0, + meta = { kind = 'firmware', component = 'mcu' }, + }, { timeout = 2.0 }) + chat('send-blob returned', { reply = reply, err = err, received = table.concat(received), committed = committed }) + if reply == nil then + dump_transport('node-a transport', transport_a) + dump_transport('node-b transport', transport_b) + local status_a = conn_a:retained_view(topics.transfer_manager_status()) + local status_b = conn_b:retained_view(topics.transfer_manager_status()) + local msg_a = status_a:get(topics.transfer_manager_status()) + local msg_b = status_b:get(topics.transfer_manager_status()) + chat('node-a retained transfer status at failure', msg_a and msg_a.payload or nil) + chat('node-b retained transfer status at failure', msg_b and msg_b.payload or nil) + status_a:close() + status_b:close() + end + assert(reply ~= nil, tostring(err)) + assert_eq(reply.ok, true) + assert_eq(table.concat(received), 'abcdef') + assert_true(committed) + end, { timeout = 6.0 }) +end + +return T diff --git a/tests/integration/devhost/hal_uart_spec.lua b/tests/integration/devhost/hal_uart_spec.lua new file mode 100644 index 00000000..7c13416b --- /dev/null +++ b/tests/integration/devhost/hal_uart_spec.lua @@ -0,0 +1,387 @@ +local fibers = require 'fibers' +local channel = require 'fibers.channel' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local exec = require 'fibers.io.exec' + +local hal_types = require 'services.hal.types.core' +local cap_args = require 'services.hal.types.capability_args' + +local runfibers = require 'tests.support.run_fibers' +local pty = require 'tests.support.pty' +local safe = require 'coxpcall' + +local perform = fibers.perform +local T = {} + +local function fresh_manager() + package.loaded['services.hal.managers.uart'] = nil + package.loaded['services.hal.drivers.uart'] = nil + return require('services.hal.managers.uart') +end + +local function dummy_logger() + local logger = {} + for _, k in ipairs({ 'debug', 'info', 'warn', 'error' }) do + logger[k] = function() end + end + function logger:child() + return self + end + return logger +end + +local function wait_channel_get(ch, timeout_s, what) + local which, a, b = perform(op.named_choice({ + item = ch:get_op(), + timeout = sleep.sleep_op(timeout_s or 1.0):wrap(function() + return true + end), + })) + + if which == 'timeout' then + error(('timed out waiting for %s'):format(what or 'channel item'), 0) + end + + if a == nil then + error(('channel closed while waiting for %s: %s'):format(what or 'channel item', tostring(b)), 0) + end + + return a +end + +local function wait_device_event(dev_ev_ch, event_type, class, id, timeout_s) + local deadline = fibers.now() + (timeout_s or 1.0) + + while fibers.now() < deadline do + local ev = wait_channel_get(dev_ev_ch, deadline - fibers.now(), 'device event') + if ev.event_type == event_type and ev.class == class and ev.id == id then + return ev + end + end + + error(('timed out waiting for device event %s %s/%s'):format( + tostring(event_type), tostring(class), tostring(id) + ), 0) +end + +local function call_control(cap, verb, opts) + local reply_ch = channel.new(1) + local req, err = hal_types.new.ControlRequest(verb, opts or {}, reply_ch) + assert(req, tostring(err)) + + -- channel:put_op() returns no values on success + perform(cap.control_ch:put_op(req)) + + local reply = wait_channel_get(reply_ch, 1.0, 'control reply') + assert(type(reply) == 'table', 'control reply must be a table') + assert(type(reply.ok) == 'boolean', 'control reply ok must be boolean') + + return reply +end + +local function open_uart_session(cap) + local open_opts, err = cap_args.new.UARTOpenOpts() + assert(open_opts, tostring(err)) + + local reply = call_control(cap, 'open', open_opts) + assert(reply.ok == true, tostring(reply.reason)) + assert(type(reply.reason) == 'table') + assert(type(reply.reason.session) == 'table') + return reply.reason.session, reply.reason +end + +local function stty_output(path) + local cmd = exec.command('stty', '-F', tostring(path), '-a') + local out, status, code, _sig, err = perform(cmd:combined_output_op()) + assert(status == 'exited' and code == 0, tostring(err or out)) + return tostring(out or '') +end + +local function assert_stty_contains(raw, token) + if not raw:find(token, 1, true) then + error(('expected stty output to contain %q, got:\n%s'):format(token, raw), 0) + end +end + +local function start_manager(scope) + local uart_mgr = fresh_manager() + + local dev_ev_ch = channel.new(16) + local cap_emit_ch = channel.new(32) + + local ok, err = perform(uart_mgr.start_op(dummy_logger(), dev_ev_ch, cap_emit_ch)) + assert(ok == true, tostring(err)) + + scope:finally(function() + safe.pcall(function() + perform(uart_mgr.shutdown_op()) + end) + end) + + return uart_mgr, dev_ev_ch, cap_emit_ch +end + +function T.devhost_uart_open_returns_wrapped_session_and_allows_reopen() + runfibers.run(function(scope) + local uart_mgr, dev_ev_ch = start_manager(scope) + local port = pty.open(scope) + + local ok_cfg, err_cfg = perform(uart_mgr.apply_config_op({ + serial_ports = { + { + id = 'uart0', + path = port.slave_name, + baud = 115200, + mode = '8N1', + }, + }, + })) + assert(ok_cfg == true, tostring(err_cfg)) + + local added = wait_device_event(dev_ev_ch, 'added', 'uart', 'uart0', 1.5) + assert(type(added.capabilities) == 'table' and #added.capabilities == 1) + local cap = added.capabilities[1] + assert(cap.class == 'uart') + assert(cap.id == 'uart0') + + local session1 = open_uart_session(cap) + local ok_w1, err_w1 = port:write('abc') + assert(ok_w1 == true, tostring(err_w1)) + + local got1, rerr1 = perform(session1:read_some_op(3)) + assert(got1 ~= nil, tostring(rerr1)) + assert(got1 == 'abc', ('expected "abc", got %q'):format(tostring(got1))) + + local n1, swerr1 = perform(session1:write_op('xyz')) + assert(n1 ~= nil, tostring(swerr1)) + local got2 = port:expect_some(3, 1.0, 'read from PTY master') + assert(got2 == 'xyz', ('expected "xyz", got %q'):format(tostring(got2))) + + local ok_c1, cerr1 = perform(session1:close_op()) + assert(ok_c1 ~= nil, tostring(cerr1)) + + local session2 = open_uart_session(cap) + local n2, swerr2 = perform(session2:write_op('q')) + assert(n2 ~= nil, tostring(swerr2)) + local got3 = port:expect_some(1, 1.0, 'read from reopened UART session') + assert(got3 == 'q', ('expected "q", got %q'):format(tostring(got3))) + + local ok_c2, cerr2 = perform(session2:close_op()) + assert(ok_c2 ~= nil, tostring(cerr2)) + end, { timeout = 4.0 }) +end + +function T.devhost_uart_reconfigures_when_the_port_changes() + runfibers.run(function(scope) + local uart_mgr, dev_ev_ch = start_manager(scope) + local port1 = pty.open(scope) + local port2 = pty.open(scope) + + local ok1, err1 = perform(uart_mgr.apply_config_op({ + serial_ports = { + { + id = 'uart0', + path = port1.slave_name, + baud = 115200, + mode = '8N1', + }, + }, + })) + assert(ok1 == true, tostring(err1)) + + local added1 = wait_device_event(dev_ev_ch, 'added', 'uart', 'uart0', 1.5) + local cap1 = added1.capabilities[1] + local session1 = open_uart_session(cap1) + + local n1, errw1 = perform(session1:write_op('first')) + assert(n1 ~= nil, tostring(errw1)) + local got1 = port1:expect_some(5, 1.0, 'read from first PTY master') + assert(got1 == 'first', ('expected "first", got %q'):format(tostring(got1))) + + local okc1, cerr1 = perform(session1:close_op()) + assert(okc1 ~= nil, tostring(cerr1)) + + local ok2, err2 = perform(uart_mgr.apply_config_op({ + serial_ports = { + { + id = 'uart0', + path = port2.slave_name, + baud = 115200, + mode = '8N1', + }, + }, + })) + assert(ok2 == true, tostring(err2)) + + local removed = wait_device_event(dev_ev_ch, 'removed', 'uart', 'uart0', 1.5) + assert(removed ~= nil) + local added2 = wait_device_event(dev_ev_ch, 'added', 'uart', 'uart0', 1.5) + local cap2 = added2.capabilities[1] + + local session2 = open_uart_session(cap2) + local n2, errw2 = perform(session2:write_op('second')) + assert(n2 ~= nil, tostring(errw2)) + local got2 = port2:expect_some(6, 1.0, 'read from second PTY master') + assert(got2 == 'second', ('expected "second", got %q'):format(tostring(got2))) + + local okc2, cerr2 = perform(session2:close_op()) + assert(okc2 ~= nil, tostring(cerr2)) + end, { timeout = 4.0 }) +end + + +function T.reconfigure_poisons_old_session_wrapper() + runfibers.run(function(scope) + local uart_mgr, dev_ev_ch = start_manager(scope) + local port1 = pty.open(scope) + local port2 = pty.open(scope) + + local ok1, err1 = perform(uart_mgr.apply_config_op({ + serial_ports = { + { + id = 'uart0', + path = port1.slave_name, + baud = 115200, + mode = '8N1', + }, + }, + })) + assert(ok1 == true, tostring(err1)) + + local added1 = wait_device_event(dev_ev_ch, 'added', 'uart', 'uart0', 1.5) + local cap1 = added1.capabilities[1] + local session1 = open_uart_session(cap1) + + local n1, errw1 = perform(session1:write_op('old')) + assert(n1 ~= nil, tostring(errw1)) + local got1 = port1:expect_some(3, 1.0, 'read from old PTY master') + assert(got1 == 'old', ('expected "old", got %q'):format(tostring(got1))) + + local ok2, err2 = perform(uart_mgr.apply_config_op({ + serial_ports = { + { + id = 'uart0', + path = port2.slave_name, + baud = 115200, + mode = '8N1', + }, + }, + })) + assert(ok2 == true, tostring(err2)) + + wait_device_event(dev_ev_ch, 'removed', 'uart', 'uart0', 1.5) + local added2 = wait_device_event(dev_ev_ch, 'added', 'uart', 'uart0', 1.5) + local cap2 = added2.capabilities[1] + + local old_n, old_werr = perform(session1:write_op('x')) + assert(old_n == nil) + assert(old_werr ~= nil) + local old_chunk, old_rerr = perform(session1:read_some_op(1)) + assert(old_chunk == nil) + assert(old_rerr ~= nil) + + local session2 = open_uart_session(cap2) + local n2, errw2 = perform(session2:write_op('new')) + assert(n2 ~= nil, tostring(errw2)) + local got2 = port2:expect_some(3, 1.0, 'read from new PTY master') + assert(got2 == 'new', ('expected "new", got %q'):format(tostring(got2))) + + local okc2, cerr2 = perform(session2:close_op()) + assert(okc2 ~= nil, tostring(cerr2)) + end, { timeout = 4.0 }) +end + +function T.pending_read_loses_to_timeout_cleanly() + runfibers.run(function(scope) + local uart_mgr, dev_ev_ch = start_manager(scope) + local port = pty.open(scope) + + local ok_cfg, err_cfg = perform(uart_mgr.apply_config_op({ + serial_ports = { + { + id = 'uart0', + path = port.slave_name, + baud = 115200, + mode = '8N1', + }, + }, + })) + assert(ok_cfg == true, tostring(err_cfg)) + + local added = wait_device_event(dev_ev_ch, 'added', 'uart', 'uart0', 1.5) + local cap = added.capabilities[1] + local session = open_uart_session(cap) + + local which = perform(fibers.named_choice{ + data = session:read_some_op(16):wrap(function(chunk, err) + return 'data', chunk, err + end), + timeout = sleep.sleep_op(0.05):wrap(function() + return 'timeout' + end), + }) + + assert(which == 'timeout') + + local ok_c, cerr = perform(session:close_op()) + assert(ok_c ~= nil, tostring(cerr)) + end, { timeout = 3.0 }) +end + + +function T.devhost_uart_start_applies_strict_stty_settings() + runfibers.run(function(scope) + local uart_mgr, dev_ev_ch = start_manager(scope) + local port = pty.open(scope) + + local ok_cfg, err_cfg = perform(uart_mgr.apply_config_op({ + serial_ports = { + { + id = 'uart0', + path = port.slave_name, + baud = 115200, + mode = '8N1', + }, + }, + })) + assert(ok_cfg == true, tostring(err_cfg)) + wait_device_event(dev_ev_ch, 'added', 'uart', 'uart0', 1.5) + + local raw = stty_output(port.slave_name) + assert_stty_contains(raw, 'speed 115200') + for _, token in ipairs({ + 'cs8', 'cread', 'clocal', + '-cstopb', '-parenb', '-crtscts', + '-ixon', '-ixoff', '-icrnl', + '-icanon', '-echo', '-isig', '-iexten', + '-opost', '-onlcr', + }) do + assert_stty_contains(raw, token) + end + assert(raw:find('min%s*=%s*1'), 'expected VMIN/min = 1 in stty output:\n' .. raw) + assert(raw:find('time%s*=%s*0'), 'expected VTIME/time = 0 in stty output:\n' .. raw) + end, { timeout = 4.0 }) +end + +function T.devhost_uart_config_fails_when_stty_fails() + runfibers.run(function(scope) + local uart_mgr, _dev_ev_ch = start_manager(scope) + + local ok_cfg, err_cfg = perform(uart_mgr.apply_config_op({ + serial_ports = { + { + id = 'uart0', + path = '/dev/devicecode-test-missing-uart', + baud = 115200, + mode = '8N1', + }, + }, + })) + + assert(ok_cfg == false, 'expected missing UART path to fail stty configuration') + assert(tostring(err_cfg):find('stty', 1, true), 'expected stty error, got: ' .. tostring(err_cfg)) + end, { timeout = 3.0 }) +end + +return T diff --git a/tests/integration/devhost/local_ui_http_spec.lua b/tests/integration/devhost/local_ui_http_spec.lua new file mode 100644 index 00000000..63f93603 --- /dev/null +++ b/tests/integration/devhost/local_ui_http_spec.lua @@ -0,0 +1,119 @@ +-- tests/integration/devhost/local_ui_http_spec.lua +-- +-- Devhost coverage for the initial local UI port. These tests deliberately go +-- through curl and the real HTTP/UI/GSM services. Only the HAL control-store is +-- faked so that APN persistence can be exercised without hardware. + +local cjson = require 'cjson.safe' + +local runfibers = require 'tests.support.run_fibers' +local harness = require 'tests.support.local_ui_devhost' + +local T = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) + if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end +end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end return v end +local function assert_nil(v, msg) if v ~= nil then fail((msg or 'expected nil') .. ': got ' .. tostring(v)) end end + +local function devhost_port(offset) + local base = tonumber(os.getenv('LOCAL_UI_DEVHOST_PORT')) or 18120 + return base + (offset or 0) +end + +local function json_body(status, body) + assert_eq(status, '200', body) + local decoded, err = cjson.decode(body or '') + return assert_not_nil(decoded, 'expected JSON body, got: ' .. tostring(body) .. ' decode_err=' .. tostring(err)) +end + +function T.devhost_local_ui_serves_curated_bootstrap_over_real_http() + runfibers.run(function (scope) + local inst = harness.start(scope, { port = devhost_port(1), static_root = 'src/services/ui/www' }) + harness.wait_http_ready(inst.base_url, { timeout = 5 }) + + local status, body = harness.curl({ + '--silent', '--show-error', '--max-time', '5', + '--write-out', '\n__HTTP_STATUS__:%{http_code}', + inst.base_url .. '/api/local-ui/bootstrap', + }) + local payload = json_body(status, body) + assert_eq(payload.schema, 'devicecode.ui.local-bootstrap/1') + assert_not_nil(payload.items['state/net/summary'], 'bootstrap should include curated network state') + assert_not_nil(payload.items['state/device/components'], 'bootstrap should include curated device state') + assert_not_nil(payload.items['state/system/stats'], + 'bootstrap should include curated system stats') + assert_nil(payload.items['raw/host/secret'], 'bootstrap must not include raw HAL topics') + assert_nil(payload.items['cfg/secret'], 'bootstrap must not include cfg topics') + end, { timeout = 12 }) +end + +function T.devhost_local_ui_apns_round_trip_through_gsm_and_fake_control_store() + runfibers.run(function (scope) + local inst = harness.start(scope, { port = devhost_port(2), static_root = 'src/services/ui/www' }) + harness.wait_http_ready(inst.base_url, { timeout = 5 }) + + local apns = { + { + carrier = 'Demo Carrier', + mcc = '234', + mnc = '10', + apn = 'custom-apn', + authtype = 'none', + user = 'demo-user', + password = 'demo-password', + }, + } + + local status, body = harness.curl_json('PUT', inst.base_url .. '/api/gsm/apns/custom', apns) + assert_eq(status, '200') + assert_true(body.ok, 'APN PUT should succeed') + assert_eq(body.apns[1].apn, 'custom-apn') + assert_eq(body.apns[1].authtype, 'none') + + status, body = harness.curl_json('GET', inst.base_url .. '/api/gsm/apns/custom') + assert_eq(status, '200') + assert_eq(body[1].carrier, 'Demo Carrier') + assert_eq(body[1].password, 'demo-password') + + local stored = assert_not_nil(inst.control_store:get('custom-apns-v1'), + 'APN list should be persisted in fake control-store') + assert_true(stored:find('custom-apn', 1, true) ~= nil, 'persisted APN JSON should contain the APN') + + status, body = harness.curl_json('GET', inst.base_url .. '/api/local-ui/bootstrap') + assert_eq(status, '200') + local apn_state = assert_not_nil(body.items['state/gsm/apns/custom'], + 'bootstrap should include GSM APN retained state after PUT') + assert_eq(apn_state.payload.count, 1) + assert_eq(apn_state.payload.records[1].apn, 'custom-apn') + assert_eq(apn_state.payload.records[1].password, nil) + assert_eq(apn_state.payload.records[1].user, nil) + assert_eq(apn_state.payload.records[1].has_password, true) + assert_eq(apn_state.payload.records[1].has_user, true) + + local saw_put = false + for _, call in ipairs(inst.control_store.calls) do + if call.method == 'put' and call.payload and call.payload.key == 'custom-apns-v1' then saw_put = true end + end + assert_true(saw_put, 'UI APN PUT should have reached GSM and then the control-store capability') + end, { timeout = 12 }) +end + +function T.devhost_local_ui_diagnostics_route_is_stubbed_for_now() + runfibers.run(function (scope) + local inst = harness.start(scope, { port = devhost_port(3), static_root = 'src/services/ui/www' }) + harness.wait_http_ready(inst.base_url, { timeout = 5 }) + + local status, body = harness.curl_json('GET', inst.base_url .. '/api/diagnostics') + assert_eq(status, '200') + assert_eq(body.schema, 'devicecode.diagnostics.stub/1') + assert_true(body.stub, 'diagnostics should remain explicitly stubbed in this pass') + assert_not_nil(body.diagnostics, 'stub should preserve old diagnostics body shape') + assert_not_nil(body.diagnostics_logs, 'stub should preserve old diagnostics logs shape') + end, { timeout = 12 }) +end + +return T diff --git a/tests/integration/devhost/main_failure_spec.lua b/tests/integration/devhost/main_failure_spec.lua new file mode 100644 index 00000000..dd79f5a4 --- /dev/null +++ b/tests/integration/devhost/main_failure_spec.lua @@ -0,0 +1,178 @@ +-- integration/devhost/main_failure_spec.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local busmod = require 'bus' + +local safe = require 'coxpcall' + +local runfibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' +local fake_hal_mod = require 'tests.support.fake_hal' +local test_diag = require 'tests.support.test_diag' + +local mainmod = require 'devicecode.main' + +local T = {} + +local function wait_retained_payload_matching(conn, topic, pred, opts) + opts = opts or {} + + local found = nil + local ok = probe.wait_until(function() + local ok2, payload = safe.pcall(function() + return probe.wait_payload(conn, topic, { timeout = 0.02 }) + end) + + if ok2 and pred(payload) then + found = payload + return true + end + + return false + end, { + timeout = opts.timeout or 0.75, + interval = opts.interval or 0.01, + }) + + if ok then + return found + end + + return nil +end + +local function make_service_loader(fake_hal, linger_box) + return function(name) + if name == 'hal' then + return { + start = function(conn, opts) + fake_hal:start(conn, { + name = opts and opts.name or 'hal', + env = opts and opts.env or 'dev', + }) + + while true do + sleep.sleep(3600.0) + end + end, + } + elseif name == 'linger' then + return { + start = function(conn, opts) + local scope = fibers.current_scope() + + scope:finally(function(aborted, st, primary) + linger_box.finalised = true + linger_box.aborted = aborted + linger_box.status = st + linger_box.primary = primary + end) + + while true do + sleep.sleep(3600.0) + end + end, + } + elseif name == 'boom' then + return { + start = function(conn, opts) + sleep.sleep(0.02) + error('boom exploded', 0) + end, + } + end + + error('unexpected service name: ' .. tostring(name), 0) + end +end + +local function spawn_main(scope, bus, fake_hal, linger_box, services_csv) + local child, cerr = scope:child() + assert(child ~= nil, tostring(cerr)) + + local ok_spawn, err = scope:spawn(function() + mainmod.run(child, { + env = 'dev', + services_csv = services_csv, + bus = bus, + service_loader = make_service_loader(fake_hal, linger_box), + exit_grace_period = 0.05, + }) + end) + assert(ok_spawn, tostring(err)) + + return child +end + +function T.devhost_main_fails_fast_when_child_service_errors() + runfibers.run(function(scope) + local bus = busmod.new() + local conn = bus:connect() + + local linger_box = { + finalised = false, + aborted = nil, + status = nil, + primary = nil, + } + + local fake_hal = fake_hal_mod.new({ + backend = 'fakehal', + caps = {}, + scripted = {}, + }) + + local diag = test_diag.for_stack(scope, bus, { obs = true, max_records = 300, fake_hal = fake_hal }) + test_diag.add_table(diag, 'linger_box', function() return linger_box end) + + local main_scope = spawn_main(scope, bus, fake_hal, linger_box, 'hal,linger,boom') + + local main_failed = wait_retained_payload_matching(conn, { 'obs', 'state', 'main' }, function(payload) + return type(payload) == 'table' + and payload.status == 'failed' + and payload.what == 'service_not_ok' + and payload.service == 'boom' + end, { timeout = 0.75 }) + + if main_failed == nil then + diag:fail('expected main to fail because boom service errored') + end + + local boom_failed = wait_retained_payload_matching(conn, { 'obs', 'state', 'service', 'boom' }, function(payload) + return type(payload) == 'table' + and payload.service == 'boom' + and payload.status == 'failed' + and tostring(payload.primary):match('boom exploded') ~= nil + end, { timeout = 0.75 }) + + if boom_failed == nil then + diag:fail('expected failing boom service state to be retained') + end + + -- Cancellation is not join. Join the main child scope so that cancelled + -- siblings are finalised and their finalisers run. + local jst, jrep, jprimary = fibers.perform(main_scope:join_op()) + assert(jst == 'cancelled' or jst == 'failed') + + local linger_final = probe.wait_until(function() + return linger_box.finalised == true + and linger_box.aborted == true + and (linger_box.status == 'cancelled' or linger_box.status == 'failed') + end, { timeout = 0.75, interval = 0.01 }) + + if not linger_final then + diag:fail(('expected sibling linger service to be cancelled and finalised' + .. ' after joining main scope' + .. ' (join_status=%s finalised=%s aborted=%s status=%s primary=%s)'):format( + tostring(jst), + tostring(linger_box.finalised), + tostring(linger_box.aborted), + tostring(linger_box.status), + tostring(linger_box.primary) + )) + end + end, { timeout = 1.25 }) +end + +return T diff --git a/tests/integration/devhost/mcu_update_full_path_spec.lua b/tests/integration/devhost/mcu_update_full_path_spec.lua new file mode 100644 index 00000000..d345f3bd --- /dev/null +++ b/tests/integration/devhost/mcu_update_full_path_spec.lua @@ -0,0 +1,790 @@ +-- tests/integration/devhost/mcu_update_full_path_spec.lua +-- +-- Realistic devhost MCU update flow: +-- browser HTTP upload -> UI -> Update -> Device -> real paired Fabric -> fake MCU, +-- then fake reboot of both Lua instances and Update reconciliation from the +-- control-store-backed durable job plus post-boot Device canonical state. + +local busmod = require 'bus' +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' + +local cjson_ok, cjson = pcall(require, 'cjson.safe') +if not cjson_ok then cjson = require 'cjson' end +local posix_ok, stdlib = pcall(require, 'posix.stdlib') + +local http_request = require 'http.request' + +local runfibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' +local dcmcu_fixture = require 'tests.support.dcmcu_fixture' + +local bus_cleanup = require 'devicecode.support.bus_cleanup' + +local hal_service = require 'services.hal' +local http_service = require 'services.http.service' +local ui_service = require 'services.ui.service' +local update_service = require 'services.update.service' +local device_service = require 'services.device.service' +local fabric = require 'services.fabric' + +local http_driver_mod = require 'services.http.transport.cqueues_driver' +local hal_transport = require 'services.fabric.hal_transport' +local fabric_protocol = require 'services.fabric.protocol' +local fabric_topics = require 'services.fabric.topics' +local update_topics = require 'services.update.topics' +local device_topics = require 'services.device.topics' +local http_topics = require 'services.http.topics' + +local T = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end + +local function log(msg) + io.stderr:write('[mcu-full-path] ' .. tostring(msg) .. '\n') +end + +local function shquote(s) + return "'" .. tostring(s):gsub("'", "'\\''") .. "'" +end + +local function mkdir_p(path) + local ok = os.execute('mkdir -p ' .. shquote(path)) + if ok ~= true and ok ~= 0 then error('mkdir failed: ' .. tostring(path), 0) end +end + +local function rm_rf(path) + os.execute('rm -rf ' .. shquote(path)) +end + +local function write_file(path, body) + local f = assert(io.open(path, 'wb')) + assert(f:write(body)) + assert(f:close()) +end + +local function temp_roots() + local base = ('/tmp/devicecode-mcu-full-%d-%d'):format(os.time(), math.random(100000, 999999)) + rm_rf(base) + local roots = { + base = base, + config = base .. '/config', + static = base .. '/static', + artifact = { + transient = base .. '/cm5/artifacts/transient', + durable = base .. '/cm5/artifacts/durable', + import = base .. '/cm5/artifacts/import', + }, + control = base .. '/cm5/control/update', + mcu = base .. '/mcu', + } + mkdir_p(roots.config) + mkdir_p(roots.static) + mkdir_p(roots.artifact.transient) + mkdir_p(roots.artifact.durable) + mkdir_p(roots.artifact.import) + mkdir_p(roots.control) + mkdir_p(roots.mcu) + write_file(roots.static .. '/index.html', 'ok') + return roots +end + +local function copy_frame(frame) + local out = {} + for k, v in pairs(frame or {}) do + if type(v) == 'table' then + local t = {} + for k2, v2 in pairs(v) do t[k2] = v2 end + out[k] = t + else + out[k] = v + end + end + return out +end + +local function new_line_transport() + local in_tx, in_rx = mailbox.new(128, { full = 'reject_newest' }) + local written = {} + local session = {} + + function session:read_line_op() + return in_rx:recv_op():wrap(function(frame) + if frame == nil then return nil, in_rx:why() or 'closed' end + local line, err = fabric_protocol.encode_line(frame) + if not line then return nil, err end + return line, nil + end) + end + + function session:write_line_op(line) + local frame, err = fabric_protocol.decode_line(line) + if not frame then return fibers.always(nil, err) end + written[#written + 1] = copy_frame(frame) + return fibers.always(true, nil) + end + + function session:flush_op() + return fibers.always(true, nil) + end + + function session:terminate(reason) + in_tx:close(reason or 'transport terminated') + return true, nil + end + + return { session = session, in_tx = in_tx, written = written } +end + +local function connect_line_transports(a, b) + local write_a = a.session.write_line_op + function a.session:write_line_op(line) + local frame = assert(fabric_protocol.decode_line(line)) + write_a(self, line) + return b.in_tx:send_op(copy_frame(frame)) + end + + local write_b = b.session.write_line_op + function b.session:write_line_op(line) + local frame = assert(fabric_protocol.decode_line(line)) + write_b(self, line) + return a.in_tx:send_op(copy_frame(frame)) + end +end + +local function wrap_session_op(session) + local wrapped, err = hal_transport.wrap_transport(session) + return fibers.always(wrapped, err) +end + +local function wait_retained_payload_where(conn, topic, label, pred, opts) + opts = opts or {} + local view = conn:retained_view(topic) + local value = probe.wait_versioned_until(label, function () + return view:version() + end, function (seen) + return view:changed_op(seen) + end, function () + local msg = view:get(topic) + local payload = msg and msg.payload or nil + if pred(payload) then return payload end + return nil + end, opts) + view:close() + return value +end + +local function wait_job(conn, job_id, state, timeout) + return wait_retained_payload_where(conn, update_topics.update_component('mcu'), 'update job ' .. tostring(job_id) .. ' ' .. tostring(state), function (p) + local job = p and (p.current_job or p.last_job) or nil + if job and job.job_id == job_id and job.state == state then return job end + return nil + end, { timeout = timeout or 4.0 }) +end + +local function wait_component_software(conn, image_id, boot_id) + return wait_retained_payload_where(conn, device_topics.component_software('mcu'), 'mcu software canonical state', function (p) + if p and (image_id == nil or p.image_id == image_id) and (boot_id == nil or p.boot_id == boot_id) then return p end + return nil + end, { timeout = 4.0 }) +end + +local function start_public_fabric(scope, conn, cfg, transport, opts) + opts = opts or {} + local link_overrides = opts.link_overrides or { + ['link-a'] = { + open_transport_op = function () return wrap_session_op(transport.session) end, + transfer = opts.transfer, + }, + } + local ok, err = scope:spawn(function () + fabric.start(conn, { + name = opts.name or 'fabric', + env = 'test', + config = cfg, + link_overrides = link_overrides, + }) + end) + assert_true(ok, tostring(err)) +end + +local function fabric_config(local_node, peer_node, bridge, transfer) + return { + schema = fabric.config.SCHEMA, + local_node = local_node, + links = { + { + id = 'link-a', + peer_id = peer_node, + transport = { source = 'test', class = 'jsonl', id = 'link-a' }, + session = { hello_interval_s = 5.0, ping_interval_s = 5.0, liveness_timeout_s = 5.0 }, + bridge = bridge or {}, + transfer = transfer or { chunk_size = 2048, timeout_s = 3.0 }, + }, + }, + } +end + +local function cm5_fabric_config() + return fabric_config('cm5', 'mcu', { + imports = { + { id = 'mcu-state', remote = { 'state', 'self' }, ['local'] = { 'raw', 'member', 'mcu', 'state' } }, + { id = 'mcu-event', remote = { 'event', 'self' }, ['local'] = { 'raw', 'member', 'mcu', 'cap', 'telemetry', 'main', 'event' } }, + { id = 'mcu-cap', remote = { 'cap', 'self' }, ['local'] = { 'raw', 'member', 'mcu', 'cap' } }, + }, + rpc = { + outbound = { + { + id = 'mcu-prepare', + ['local'] = { 'raw', 'member', 'mcu', 'cap', 'updater', 'main', 'rpc', 'prepare-update' }, + remote = { 'cap', 'self', 'updater', 'main', 'rpc', 'prepare-update' }, + timeout_s = 2.0, + }, + { + id = 'mcu-commit', + ['local'] = { 'raw', 'member', 'mcu', 'cap', 'updater', 'main', 'rpc', 'commit-update' }, + remote = { 'cap', 'self', 'updater', 'main', 'rpc', 'commit-update' }, + timeout_s = 2.0, + }, + }, + }, + }, { chunk_size = 2048, timeout_s = 4.0 }) +end + +local function mcu_fabric_config() + return fabric_config('mcu', 'cm5', { + exports = { + { id = 'mcu-state-export', ['local'] = { 'state', 'self' }, remote = { 'state', 'self' }, publish = true, retain = true }, + { id = 'mcu-cap-export', ['local'] = { 'cap', 'self' }, remote = { 'cap', 'self' }, publish = true, retain = true }, + }, + rpc = { + inbound = { + { + id = 'mcu-prepare-in', + ['local'] = { 'cap', 'self', 'updater', 'main', 'rpc', 'prepare-update' }, + remote = { 'cap', 'self', 'updater', 'main', 'rpc', 'prepare-update' }, + timeout_s = 2.0, + }, + { + id = 'mcu-commit-in', + ['local'] = { 'cap', 'self', 'updater', 'main', 'rpc', 'commit-update' }, + remote = { 'cap', 'self', 'updater', 'main', 'rpc', 'commit-update' }, + timeout_s = 2.0, + }, + }, + }, + }, { chunk_size = 2048, timeout_s = 4.0 }) +end + +local function publish_mcu_facts(conn, fake) + local image = fake.committed_image_id or fake.old_image_id + local boot = 'mcu-boot-' .. tostring(fake.boot_seq) + conn:retain({ 'state', 'self', 'software' }, { + image_id = image, + boot_id = boot, + version = image, + }) + conn:retain({ 'state', 'self', 'updater' }, { + state = 'ready', + last_error = nil, + staged_image_id = fake.staged and fake.staged.image_id or nil, + pending_image_id = fake.committed_image_id, + job_id = fake.job_id, + }) + conn:retain({ 'cap', 'self', 'updater', 'main', 'meta' }, { + class = 'updater', + id = 'main', + methods = { 'prepare-update', 'commit-update' }, + }) + conn:retain({ 'cap', 'self', 'updater', 'main', 'status' }, { + available = true, + state = 'available', + }) +end + +local function new_mcu_receive_target(fake) + local target = {} + function target:open_sink_op(req) + fake.transfer_begin = req + assert_eq(req.target, 'updater/main') + local chunks = {} + local sink = {} + function sink:append_op(chunk) + chunks[#chunks + 1] = chunk + return fibers.always(true, nil) + end + function sink:commit_op(req2) + local bytes = table.concat(chunks) + fake.staged = { + bytes = bytes, + digest = req2.digest, + size = req2.size, + image_id = req.meta and (req.meta.image_id or req.meta.expected_image_id), + job_id = req.meta and req.meta.job_id, + } + fake.staged_signal = (fake.staged_signal or 0) + 1 + return fibers.always({ staged = true, digest = req2.digest }, nil) + end + function sink:abort(reason) + fake.abort_reason = reason + return true, nil + end + return fibers.always(sink, nil) + end + return target +end + +local function start_fake_mcu(scope, bus, fake) + local conn = bus:connect({ origin_base = { service = 'fake-mcu' } }) + fake.boot_seq = (fake.boot_seq or 0) + 1 + publish_mcu_facts(conn, fake) + + local eps = {} + local function bind(topic) + local ep, err = bus_cleanup.bind(conn, topic, { queue_len = 8 }) + assert_not_nil(ep, err) + eps[#eps + 1] = ep + return ep + end + + local prepare_ep = bind({ 'cap', 'self', 'updater', 'main', 'rpc', 'prepare-update' }) + local commit_ep = bind({ 'cap', 'self', 'updater', 'main', 'rpc', 'commit-update' }) + + scope:finally(function () + for _, ep in ipairs(eps) do bus_cleanup.unbind(conn, ep) end + bus_cleanup.disconnect(conn) + end) + + assert_true(scope:spawn(function () + while true do + local req = fibers.perform(prepare_ep:recv_op()) + if req == nil then return end + fake.prepare_payload = req.payload + assert_eq(type(req.payload) == 'table' and req.payload.target or nil, 'mcu') + fake.job_id = type(req.payload) == 'table' and req.payload.job_id or nil + conn:retain({ 'state', 'self', 'updater' }, { state = 'ready', last_error = nil, job_id = fake.job_id }) + req:reply({ ready = true, target = 'updater/main', max_chunk_size = 2048 }) + end + end)) + + assert_true(scope:spawn(function () + while true do + local req = fibers.perform(commit_ep:recv_op()) + if req == nil then return end + fake.commit_payload = req.payload + assert_eq(type(req.payload), 'table') + assert_eq(req.payload.job_id, fake.job_id) + assert_eq(req.payload.metadata, nil) + fake.commit_seen = true + fake.committed_image_id = (fake.staged and fake.staged.image_id) + or (type(req.payload) == 'table' and req.payload.expected_image_id) + conn:retain({ 'state', 'self', 'updater' }, { + state = 'rebooting', + last_error = nil, + pending_image_id = fake.committed_image_id, + staged_image_id = fake.staged and fake.staged.image_id or nil, + job_id = fake.job_id, + }) + req:reply({ accepted = true, reboot_required = true }) + end + end)) + + return conn +end + +local function new_fabric_client(conn) + local client = {} + function client:send_blob_op(params, opts) + params = params or {} + opts = opts or {} + assert(params.source_owner, 'fabric client source_owner required') + return conn:call_op(fabric_topics.transfer_manager_rpc('send-blob'), { + link_id = params.link_id or 'link-a', + request_id = params.request_id or ('device-stage-' .. tostring(params.job_id or os.clock())), + xfer_id = params.xfer_id, + target = assert(params.target, 'fabric client params.target required'), + source_owner = params.source_owner, + size = params.size, + digest_alg = params.digest_alg, + digest = params.digest, + chunk_size = params.chunk_size or 2048, + meta = params.meta, + timeout_s = opts.timeout or params.timeout or 4.0, + }, { timeout = opts.timeout or params.timeout or 4.0 }):wrap(function (reply, err) + if reply == nil then return nil, err end + return { + ok = reply.ok, + committed = reply.committed, + transfer = reply, + }, nil + end) + end + return client +end + +local function update_config() + return { + schema = 'devicecode.update/1', + components = { + { component = 'mcu' }, + }, + } +end + +local function start_hal(scope, bus, roots) + local conn = bus:connect({ origin_base = { service = 'hal' } }) + if posix_ok and stdlib and stdlib.setenv then + stdlib.setenv('DEVICECODE_CONFIG_DIR', roots.config, true) + end + assert_true(scope:spawn(function () + hal_service.start(conn, { name = 'hal', env = 'test', heartbeat_s = false }) + end)) + conn:retain({ 'cfg', 'hal' }, { data = { + schema = 'devicecode.config/hal/1', + artifact_store = { + stores = { + { + id = 'main', + transient_root = roots.artifact.transient, + durable_root = roots.artifact.durable, + import_root = roots.artifact.import, + durable_enabled = true, + }, + }, + }, + control_store = { + { name = 'update', root = roots.control }, + }, + } }) + wait_retained_payload_where(conn, { 'cap', 'artifact-store', 'main', 'status' }, 'artifact-store available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + wait_retained_payload_where(conn, { 'cap', 'control-store', 'update', 'status' }, 'control-store available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + return conn +end + +local function start_http(scope, bus) + local conn = bus:connect({ origin_base = { service = 'http' } }) + assert_true(scope:spawn(function (s) + http_service.run(s, { + conn = conn, + id = 'main', + config = { + schema = 'devicecode.config/http/1', + id = 'main', + policy = { allow_loopback = true, max_request_body = 16 * 1024 * 1024, max_response_body = 16 * 1024 * 1024 }, + }, + backend_timeout = 3.0, + connection_setup_timeout = 3.0, + intra_stream_timeout = 3.0, + }) + end)) + wait_retained_payload_where(conn, { 'cap', 'http', 'main', 'status' }, 'http service available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + return conn +end + +local function start_device(scope, bus, fabric_client) + local conn = bus:connect({ origin_base = { service = 'device' } }) + assert_true(scope:spawn(function (s) + device_service.run(s, { + conn = conn, + initial_config = { schema = 'devicecode.config/device/1' }, + fabric_client = fabric_client, + watch_config = false, + }) + end)) + wait_retained_payload_where(conn, device_topics.component_cap_status('mcu'), 'mcu component cap available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + return conn +end + +local function start_update(scope, bus) + local conn = bus:connect({ origin_base = { service = 'update' } }) + assert_true(scope:spawn(function (s) + update_service.run(s, { + conn = conn, + service_id = 'update', + watch_config = false, + config = update_config(), + job_store_call_opts = { timeout = 3.0 }, + }) + end)) + wait_retained_payload_where(conn, update_topics.update_manager_status(), 'update manager available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + wait_retained_payload_where(conn, update_topics.artifact_ingest_status(), 'artifact ingest available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + return conn +end + +local function start_ui(scope, bus, port, roots) + local conn = bus:connect({ origin_base = { service = 'ui' } }) + assert_true(scope:spawn(function (s) + ui_service.run(s, { + conn = conn, + service_id = 'ui', + auth_opts = { + users = { + tester = { password = 'test-password', principal = { kind = 'user', id = 'tester' } }, + }, + }, + bus = bus, + connect = function (principal) + return bus:connect({ principal = principal or { kind = 'ui-test' } }) + end, + encode_json = function (v) return assert(cjson.encode(v)) end, + update = { + bus = bus, + component = 'mcu', + ingest_id = 'ing-mcu-full-path', + job_id = 'job-mcu-full-path', + create_job = true, + start_job = true, + timeout = 8.0, + chunk_size = 4096, + metadata = { + source = 'browser', + format = 'dcmcu-v1', + }, + }, + updates = { upload = { enabled = true, max_bytes = 1024 * 1024, require_auth = false, component = 'mcu', create_job = true, start_job = true }, commit = { require_auth = false } }, + }) + end)) + conn:retain({ 'cfg', 'ui' }, { data = { + schema = 'devicecode.config/ui/1', + enabled = true, + http = { enabled = true, cap_id = 'main', host = '127.0.0.1', port = port, max_active_requests = 8 }, + static = { root = roots.static, index = 'index.html' }, + updates = { upload = { enabled = true, max_bytes = 1024 * 1024, require_auth = false, component = 'mcu', create_job = true, start_job = true }, commit = { require_auth = false } }, + sse = { enabled = false }, + sessions = { prune_interval = false }, + } }) + wait_retained_payload_where(conn, http_topics.state('main', 'stats'), 'ui listener active', function (p) + return p and type(p.active_listeners) == 'number' and p.active_listeners > 0 and p + end, { timeout = 4.0 }) + return conn +end + +local function start_fabric_pair(parent_scope, cm5_bus, mcu_bus, fake) + local pair_scope = assert(parent_scope:child()) + local transport_cm5 = new_line_transport() + local transport_mcu = new_line_transport() + connect_line_transports(transport_cm5, transport_mcu) + + local cm5_conn = cm5_bus:connect({ origin_base = { service = 'fabric-cm5' } }) + local mcu_conn = mcu_bus:connect({ origin_base = { service = 'fabric-mcu' } }) + + start_public_fabric(pair_scope, cm5_conn, cm5_fabric_config(), transport_cm5, { name = 'fabric-cm5' }) + start_public_fabric(pair_scope, mcu_conn, mcu_fabric_config(), transport_mcu, { + name = 'fabric-mcu', + link_overrides = { + ['link-a'] = { + open_transport_op = function () return wrap_session_op(transport_mcu.session) end, + transfer = { + chunk_size = 2048, + timeout_s = 4.0, + receive_targets = { ['updater/main'] = new_mcu_receive_target(fake) }, + }, + }, + }, + }) + + wait_retained_payload_where(cm5_conn, fabric_topics.transfer_manager_status(), 'cm5 fabric transfer manager available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + + return { + scope = pair_scope, + cm5_conn = cm5_conn, + mcu_conn = mcu_conn, + } +end + +local function stop_fabric_pair(pair, reason) + if not pair then return end + if pair.scope then + pair.scope:cancel(reason or 'fabric pair stop') + fibers.perform(pair.scope:join_op()) + end + bus_cleanup.disconnect(pair.cm5_conn) + bus_cleanup.disconnect(pair.mcu_conn) +end + +local function start_cm5_instance(parent_scope, roots, port) + local scope = assert(parent_scope:child()) + local bus = busmod.new() + local conn = bus:connect({ origin_base = { service = 'test-cm5' } }) + + start_hal(scope, bus, roots) + start_http(scope, bus) + + return { + scope = scope, + bus = bus, + conn = conn, + start_services_after_fabric = function (fabric_client) + start_device(scope, bus, fabric_client) + start_update(scope, bus) + if port then start_ui(scope, bus, port, roots) end + end, + } +end + +local function start_mcu_instance(parent_scope, fake) + local scope = assert(parent_scope:child()) + local bus = busmod.new() + local conn = bus:connect({ origin_base = { service = 'test-mcu' } }) + start_fake_mcu(scope, bus, fake) + return { scope = scope, bus = bus, conn = conn } +end + +local function stop_instance(inst, reason) + if inst and inst.scope then + inst.scope:cancel(reason or 'test reboot') + fibers.perform(inst.scope:join_op()) + end +end + +local function run_http_upload(scope, port, body) + local driver = assert(http_driver_mod.new({ label = 'mcu-full-path-http-client' })) + assert_true(driver:start(scope), 'HTTP upload client driver should start') + local status, resp_body = fibers.perform(driver:run_op('mcu-full-path-upload', function () + local req = http_request.new_from_uri(('http://127.0.0.1:%d/api/update/upload'):format(port)) + req.headers:upsert(':method', 'POST') + req.headers:upsert('content-type', 'application/octet-stream') + req.headers:upsert('content-length', tostring(#body)) + req:set_body(body) + local headers, stream = assert(req:go(8)) + local status = headers:get(':status') + local response_body, body_err = stream:get_body_as_string(8) + if response_body == nil then + response_body = (''):format(tostring(body_err)) + end + return status, response_body + end)) + driver:terminate('upload complete') + return status, resp_body +end + + +local function run_http_json(scope, port, path, payload, headers) + headers = headers or {} + local driver = assert(http_driver_mod.new({ label = 'mcu-full-path-http-json-client' })) + assert_true(driver:start(scope), 'HTTP JSON client driver should start') + local status, resp_body = fibers.perform(driver:run_op('mcu-full-path-json', function () + local req = http_request.new_from_uri(('http://127.0.0.1:%d%s'):format(port, path)) + req.headers:upsert(':method', 'POST') + req.headers:upsert('content-type', 'application/json') + for k, v in pairs(headers) do + req.headers:upsert(k, tostring(v)) + end + req:set_body(assert(cjson.encode(payload or {}))) + local resp_headers, stream = assert(req:go(8)) + local status = resp_headers:get(':status') + local response_body, body_err = stream:get_body_as_string(8) + if response_body == nil then + response_body = (''):format(tostring(body_err)) + end + return status, response_body + end)) + driver:terminate('json complete') + return status, resp_body, resp_body and cjson.decode(resp_body) or nil +end + +function T.ui_http_mcu_update_survives_fake_reboot_and_reconciles() + runfibers.run(function (root_scope) + local roots = temp_roots() + local blob = dcmcu_fixture.make('mcu-image-new') + local port = 30000 + math.random(0, 20000) + local fake = { old_image_id = 'mcu-image-old' } + + log('booting initial CM5 instance') + local cm5 = start_cm5_instance(root_scope, roots, port) + log('booting initial fake MCU instance') + local mcu = start_mcu_instance(root_scope, fake) + log('starting initial Fabric pair') + local pair = start_fabric_pair(root_scope, cm5.bus, mcu.bus, fake) + log('starting CM5 Device/Update/UI services') + cm5.start_services_after_fabric(new_fabric_client(cm5.bus:connect({ origin_base = { service = 'device-fabric-client' } }))) + + log('waiting for initial canonical MCU software state') + wait_component_software(cm5.conn, 'mcu-image-old', 'mcu-boot-1') + + log('sending real HTTP upload') + local status, body = run_http_upload(root_scope, port, blob) + assert_eq(status, '200', 'upload HTTP status ' .. tostring(status) .. ': ' .. tostring(body)) + local decoded = assert(cjson.decode(body), body) + assert_eq(decoded.status, 'ok') + assert_eq(decoded.job_id, 'job-mcu-full-path') + assert_eq(decoded.job, nil) + + log('waiting for job awaiting_commit') + wait_job(cm5.conn, 'job-mcu-full-path', 'awaiting_commit', 8.0) + assert_true(probe.wait_until(function () + return fake.staged and fake.staged.bytes == blob + end, { timeout = 4.0 }), 'fake MCU should stage transferred artifact') + + log('committing job through real HTTP update commit route') + local commit_status, commit_body, commit_decoded = run_http_json( + root_scope, + port, + '/api/update/commit', + { job_id = 'job-mcu-full-path' }, + nil + ) + assert_eq(commit_status, '200', commit_body) + assert_not_nil(commit_decoded and commit_decoded.value, commit_body) + assert_eq(commit_decoded.value.ok, true) + log('waiting for job awaiting_return') + wait_job(cm5.conn, 'job-mcu-full-path', 'awaiting_return', 6.0) + assert_true(probe.wait_until(function () return fake.commit_seen == true end, { timeout = 2.0 }), 'fake MCU should see commit') + assert_eq(fake.committed_image_id, 'mcu-image-new') + + log('fake reboot: stopping Fabric pair') + stop_fabric_pair(pair, 'fake fabric link reboot') + log('fake reboot: stopping CM5 instance') + stop_instance(cm5, 'fake cm5 reboot') + log('fake reboot: stopping MCU instance') + stop_instance(mcu, 'fake mcu reboot') + + log('reboot: starting fresh CM5 instance') + local cm5b = start_cm5_instance(root_scope, roots, nil) + log('reboot: starting fresh MCU instance') + local mcub = start_mcu_instance(root_scope, fake) + log('reboot: starting fresh Fabric pair') + local pair_b = start_fabric_pair(root_scope, cm5b.bus, mcub.bus, fake) + log('reboot: starting fresh CM5 Device/Update services') + cm5b.start_services_after_fabric(new_fabric_client(cm5b.bus:connect({ origin_base = { service = 'device-fabric-client-boot2' } }))) + + log('reboot: waiting for post-boot canonical MCU software state') + wait_component_software(cm5b.conn, 'mcu-image-new', 'mcu-boot-2') + log('reboot: waiting for job succeeded') + local final_job = wait_job(cm5b.conn, 'job-mcu-full-path', 'succeeded', 8.0) + assert_eq(final_job.component, 'mcu') + assert_eq(final_job.job_id, 'job-mcu-full-path') + assert_not_nil(final_job.commit_attempt, 'job should carry commit attempt details') + if final_job.commit_attempt and final_job.commit_attempt.pre_commit then + assert_eq(final_job.commit_attempt.pre_commit.pre_commit_boot_id, 'mcu-boot-1') + end + + log('cleanup: stopping second Fabric pair') + stop_fabric_pair(pair_b, 'test complete') + log('cleanup: stopping second CM5 instance') + stop_instance(cm5b, 'test complete') + log('cleanup: stopping second MCU instance') + stop_instance(mcub, 'test complete') + log('cleanup: removing temporary roots') + rm_rf(roots.base) + end, { timeout = 30.0 }) +end + +return T diff --git a/tests/integration/devhost/mcu_update_http_uart_fault_spec.lua b/tests/integration/devhost/mcu_update_http_uart_fault_spec.lua new file mode 100644 index 00000000..2e5ffd59 --- /dev/null +++ b/tests/integration/devhost/mcu_update_http_uart_fault_spec.lua @@ -0,0 +1,197 @@ +-- tests/integration/devhost/mcu_update_http_uart_fault_spec.lua +-- +-- Go MCU devhost PTY fault/chaos matrix. This deliberately runs through the +-- public HTTP update path and the real Lua service stack, then pushes byte-level +-- PTY faults through the UART middleware. It skips cleanly when the Go devhost +-- binary cannot be supplied or built. + +local base = require 'integration.devhost.mcu_update_http_uart_spec' +local H = assert(base.__helpers, 'mcu_update_http_uart_spec helpers unavailable') + +local T = {} + +local function log(msg) + H.log('[fault-matrix] ' .. tostring(msg)) +end + +local function run_case(case) + log('case start: ' .. tostring(case.name)) + local result = H.run_go_devhost_pty_cycle({ + label = case.name, + blob_bytes = case.blob_bytes, + uart = case.uart, + reboot_uart = case.reboot_uart, + timeout_s = case.timeout_s, + stage_timeout_s = case.stage_timeout_s, + }) + if result and result.skipped then + log('case skipped: ' .. tostring(result.reason)) + return result + end + assert(result and result.ok, 'case did not return ok: ' .. tostring(case.name)) + if type(case.assert_stats) == 'function' then + case.assert_stats(result.first_uart_stats or {}, result.second_uart_stats or {}) + end + log('case ok: ' .. tostring(case.name)) + return result +end + +local function assert_at_least(v, n, what) + if not ((tonumber(v) or 0) >= n) then + error(('expected %s >= %d, got %s'):format(what, n, tostring(v)), 0) + end +end + +local function env_bool(name, default_value) + local raw = os.getenv(name) + if raw == nil or raw == '' then return default_value end + raw = tostring(raw):lower():match('^%s*(.-)%s*$') + if raw == '1' or raw == 'true' or raw == 'yes' or raw == 'on' then return true end + if raw == '0' or raw == 'false' or raw == 'no' or raw == 'off' then return false end + error(('%s must be boolean-like'):format(name), 0) +end + +local function env_size_bytes(name, default_value) + local raw = os.getenv(name) + if raw == nil or raw == '' then return default_value end + raw = tostring(raw):match('^%s*(.-)%s*$') + local number, suffix = raw:match('^(%d+)%s*([kKmMgG]?[iI]?[bB]?)$') + if number == nil then error(('invalid %s=%q'):format(name, raw), 0) end + local n = tonumber(number) + suffix = suffix:lower() + if suffix == 'k' or suffix == 'kb' or suffix == 'kib' then n = n * 1024 + elseif suffix == 'm' or suffix == 'mb' or suffix == 'mib' then n = n * 1024 * 1024 + elseif suffix == 'g' or suffix == 'gb' or suffix == 'gib' then n = n * 1024 * 1024 * 1024 + elseif suffix ~= '' and suffix ~= 'b' then error(('invalid %s=%q'):format(name, raw), 0) end + return n +end + +local FULL_FAULT_MATRIX = env_bool('MCU_HTTP_UART_FULL_FAULT_MATRIX', false) +local FAULT_BLOB_BYTES = env_size_bytes('MCU_HTTP_UART_FAULT_BLOB_BYTES', 8 * 1024) + + +local function cases() + return { + { + name = 'standalone-bad-json-both-directions', + blob_bytes = FAULT_BLOB_BYTES, + uart = { + malformed_line_count = 1, + faults = { + cm5_to_mcu = { malformed_line_count = 2, malformed_line_first_at = 1536, malformed_line_every_bytes = 4096 }, + mcu_to_cm5 = { malformed_line_count = 2, malformed_line_first_at = 512, malformed_line_every_bytes = 4096 }, + }, + }, + assert_stats = function (st) + assert_at_least(st.malformed_lines, 2, 'malformed lines') + end, + }, + { + name = 'single-byte-loss-in-cm5-bulk-frame', + blob_bytes = FAULT_BLOB_BYTES, + uart = { + malformed_line_count = 0, + faults = { + cm5_to_mcu = { disable_malformed = true, drop_byte_after_bytes = 3072 }, + mcu_to_cm5 = { disable_malformed = true }, + }, + }, + assert_stats = function (st) + assert_at_least(st.dropped_bytes, 1, 'dropped bytes') + end, + }, + { + name = 'single-byte-loss-in-mcu-control-frame', + blob_bytes = FAULT_BLOB_BYTES, + uart = { + malformed_line_count = 0, + faults = { + cm5_to_mcu = { disable_malformed = true }, + mcu_to_cm5 = { disable_malformed = true, drop_byte_in_frame_type = 'xfer_need' }, + }, + }, + assert_stats = function (st) + assert_at_least(st.dropped_bytes, 1, 'dropped bytes') + end, + }, + { + name = 'long-pauses-both-directions', + blob_bytes = FAULT_BLOB_BYTES, + uart = { + malformed_line_count = 0, + faults = { + cm5_to_mcu = { disable_malformed = true, pause_once_after_bytes = 4096, pause_s = 0.20 }, + mcu_to_cm5 = { disable_malformed = true, pause_once_after_bytes = 2048, pause_s = 0.20 }, + }, + }, + assert_stats = function (st) + assert_at_least(st.fault_pauses, 2, 'fault pauses') + end, + }, + { + name = 'combined-bad-json-byte-loss-and-pauses', + blob_bytes = FAULT_BLOB_BYTES, + uart = { + malformed_line_count = 1, + faults = { + cm5_to_mcu = { + malformed_line_count = 1, + malformed_line_first_at = 1536, + drop_byte_after_bytes = 8192, + pause_once_after_bytes = 4096, + pause_s = 0.20, + }, + mcu_to_cm5 = { + malformed_line_count = 1, + malformed_line_first_at = 1024, + drop_byte_in_frame_type = 'xfer_need', + pause_once_after_bytes = 1024, + pause_s = 0.20, + }, + }, + }, + assert_stats = function (st) + assert_at_least(st.malformed_lines, 1, 'malformed lines') + assert_at_least(st.dropped_bytes, 1, 'dropped bytes') + assert_at_least(st.fault_pauses, 2, 'fault pauses') + end, + }, + } +end + +function T.go_devhost_pty_fiendish_transport_recovery_matrix() + for i, case in ipairs(cases()) do + if FULL_FAULT_MATRIX or i <= 3 then + local result = run_case(case) + if result and result.skipped then return end + else + log('case skipped in normal CI: ' .. tostring(case.name) .. ' (set MCU_HTTP_UART_FULL_FAULT_MATRIX=1)') + end + end +end + +function T.go_devhost_pty_destructive_newline_loss_documents_stop_wait_failure() + local enabled = os.getenv('MCU_HTTP_UART_DESTRUCTIVE_FAULTS') + if enabled ~= '1' and enabled ~= 'true' then + log('skipping destructive newline-loss case; set MCU_HTTP_UART_DESTRUCTIVE_FAULTS=1') + return + end + run_case({ + name = 'destructive-newline-loss-mcu-to-cm5', + blob_bytes = FAULT_BLOB_BYTES, + stage_timeout_s = 25.0, + timeout_s = 80.0, + uart = { + malformed_line_count = 0, + faults = { + cm5_to_mcu = { disable_malformed = true }, + mcu_to_cm5 = { disable_malformed = true, drop_next_newline_after_bytes = 3072 }, + }, + }, + assert_stats = function (st) + assert_at_least(st.dropped_newlines, 1, 'dropped newlines') + end, + }) +end + +return T diff --git a/tests/integration/devhost/mcu_update_http_uart_spec.lua b/tests/integration/devhost/mcu_update_http_uart_spec.lua new file mode 100644 index 00000000..c52f8630 --- /dev/null +++ b/tests/integration/devhost/mcu_update_http_uart_spec.lua @@ -0,0 +1,2333 @@ +-- tests/integration/devhost/mcu_update_http_uart_spec.lua +-- +-- Release-gate devhost MCU update flow over the real CM5 HAL UART boundary: +-- curl HTTP upload -> UI -> Update -> Device -> Fabric -> HAL UART -> PTY +-- middleware -> MCU-side Fabric -> fake MCU updater. +-- +-- The middleware deliberately preserves the base Fabric protocol semantics. It +-- fragments and paces bytes like a modestly imperfect 115200 bps UART, +-- includes occasional non-corrupt stalls, a small fake flash-write delay on +-- the MCU side, and a small number of standalone malformed JSONL lines +-- between valid frames. It does not corrupt bytes inside a valid frame, +-- reorder frames, or ask the sender for future transfer offsets. + +local busmod = require 'bus' +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local channel = require 'fibers.channel' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local exec = require 'fibers.io.exec' +local socket = require 'fibers.io.socket' + +local cjson_ok, cjson = pcall(require, 'cjson.safe') +if not cjson_ok then cjson = require 'cjson' end +local posix_ok, stdlib = pcall(require, 'posix.stdlib') +local safe = require 'coxpcall' + +local http_request = require 'http.request' + +local runfibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' +local pty = require 'tests.support.pty' +local dcmcu_fixture = require 'tests.support.dcmcu_fixture' + +local bus_cleanup = require 'devicecode.support.bus_cleanup' + +local hal_types = require 'services.hal.types.core' +local cap_args = require 'services.hal.types.capability_args' + +local hal_service = require 'services.hal' +local http_service = require 'services.http.service' +local ui_service = require 'services.ui.service' +local update_service = require 'services.update.service' +local device_service = require 'services.device.service' +local fabric = require 'services.fabric' + +local http_driver_mod = require 'services.http.transport.cqueues_driver' +local hal_transport = require 'services.fabric.hal_transport' +local fabric_protocol = require 'services.fabric.protocol' +local fabric_topics = require 'services.fabric.topics' +local update_topics = require 'services.update.topics' +local device_topics = require 'services.device.topics' +local mcu_schema = require 'services.device.schemas.mcu' +local http_topics = require 'services.http.topics' +local xxhash32 = require 'shared.hash.xxhash32' + +local T = {} +local unpack = rawget(table, 'unpack') or _G.unpack + +local DEFAULT_CI_MCU_BLOB_BYTES = 6 * 1024 +local LARGE_MCU_BLOB_ENV = 'MCU_HTTP_UART_BLOB_BYTES' + +local function env_size_bytes(name, default_value) + local raw = os.getenv(name) + if raw == nil or raw == '' then return default_value end + raw = tostring(raw):match('^%s*(.-)%s*$') + local number, suffix = raw:match('^(%d+)%s*([kKmMgG]?[iI]?[bB]?)$') + if number == nil then + error(('invalid %s=%q; expected bytes, KiB, MiB or GiB, for example 204800 or 200K'):format(name, raw), 0) + end + local n = tonumber(number) + suffix = suffix:lower() + if suffix == 'k' or suffix == 'kb' or suffix == 'kib' then + n = n * 1024 + elseif suffix == 'm' or suffix == 'mb' or suffix == 'mib' then + n = n * 1024 * 1024 + elseif suffix == 'g' or suffix == 'gb' or suffix == 'gib' then + n = n * 1024 * 1024 * 1024 + elseif suffix ~= '' and suffix ~= 'b' then + error(('invalid %s=%q; expected bytes, KiB, MiB or GiB, for example 204800 or 200K'):format(name, raw), 0) + end + if n <= 0 then + error(('%s must be positive'):format(name), 0) + end + return n +end + + +local function env_number(name, default_value) + local raw = os.getenv(name) + if raw == nil or raw == '' then return default_value end + raw = tostring(raw):match('^%s*(.-)%s*$') + local n = tonumber(raw) + if n == nil or n <= 0 then + error(('%s must be a positive number'):format(name), 0) + end + return n +end + +local function env_bool(name, default_value) + local raw = os.getenv(name) + if raw == nil or raw == '' then return default_value end + raw = tostring(raw):lower():match('^%s*(.-)%s*$') + if raw == '1' or raw == 'true' or raw == 'yes' or raw == 'on' then return true end + if raw == '0' or raw == 'false' or raw == 'no' or raw == 'off' then return false end + error(('%s must be boolean-like'):format(name), 0) +end + +local UART_BYTES_PER_SEC = env_number('MCU_HTTP_UART_BYTES_PER_SEC', env_number('MCU_HTTP_UART_BAUD', 921600) / 10) +local REALISTIC_MCU_BLOB_BYTES = env_size_bytes(LARGE_MCU_BLOB_ENV, DEFAULT_CI_MCU_BLOB_BYTES) +local REALISTIC_TRANSFER_TIMEOUT_S = 300.0 +local SHORT_STAGE_TIMEOUT_S = 2.0 +local REALISTIC_TEST_TIMEOUT_S = 240.0 +local MCU_PROGRESS_LOG_BYTES = 16 * 1024 +local WAIT_PROGRESS_LOG_S = env_number('MCU_HTTP_UART_WAIT_LOG_S', 10.0) +local UART_MAX_READ_BYTES = 128 +local UART_MAX_FRAGMENT_BYTES = 16 +local UART_LONG_PAUSE_EVERY_BYTES = 64 * 1024 +local UART_LONG_PAUSE_BASE_S = 0.20 +local UART_LONG_PAUSE_JITTER_S = 0.15 +local UART_MALFORMED_LINE_COUNT = 2 +local UART_MALFORMED_LINE_FIRST_AT = 1536 +local UART_MALFORMED_LINE_EVERY_BYTES = 128 * 1024 +local MCU_FLASH_WRITE_DELAY_S = 0.010 +local MCU_HTTP_UART_VERBOSE = env_bool('MCU_HTTP_UART_VERBOSE', false) + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end +local function assert_contains(haystack, needle, msg) + haystack = tostring(haystack or '') + needle = tostring(needle or '') + if haystack:find(needle, 1, true) == nil then + fail(msg or ('expected ' .. haystack .. ' to contain ' .. needle)) + end +end + +local function log(msg) + if not MCU_HTTP_UART_VERBOSE then return end + io.stderr:write('[mcu-http-uart] ' .. tostring(msg) .. '\n') +end + +local function shquote(s) + return "'" .. tostring(s):gsub("'", "'\\''") .. "'" +end + +local function env_non_empty(name) + local v = os.getenv(name) + if v == nil or v == '' then return nil end + v = tostring(v):match('^%s*(.-)%s*$') + if v == '' then return nil end + return v +end + + +local function command_available(name) + local cmd = exec.command('sh', '-c', 'command -v ' .. shquote(name) .. ' >/dev/null 2>&1') + local _out, st, code = fibers.perform(cmd:combined_output_op()) + return st == 'exited' and code == 0 +end + +local function mkdir_p(path) + local ok = os.execute('mkdir -p ' .. shquote(path)) + if ok ~= true and ok ~= 0 then error('mkdir failed: ' .. tostring(path), 0) end +end + +local function rm_rf(path) + os.execute('rm -rf ' .. shquote(path)) +end + +local function write_file(path, body) + local f = assert(io.open(path, 'wb')) + assert(f:write(body)) + assert(f:close()) +end + + +local function sha256_hex_string(data) + local path = os.tmpname() + write_file(path, data) + local script = table.concat({ + 'set -eu', + 'if command -v sha256sum >/dev/null 2>&1; then', + ' sha256sum ' .. shquote(path) .. " | awk '{ print $1 }'", + 'elif command -v shasum >/dev/null 2>&1; then', + ' shasum -a 256 ' .. shquote(path) .. " | awk '{ print $1 }'", + 'else', + " echo 'sha256sum_or_shasum_required' >&2", + ' exit 127', + 'fi', + }, '\n') + local cmd = exec.command('sh', '-c', script) + local out, st, code, sig, err = fibers.perform(cmd:combined_output_op()) + rm_rf(path) + if not (st == 'exited' and code == 0) then + error(('sha256 helper failed: status=%s code=%s signal=%s err=%s output=%s'):format( + tostring(st), tostring(code), tostring(sig), tostring(err), tostring(out) + ), 0) + end + local hex = tostring(out or ''):match('([0-9a-fA-F]+)') + if hex == nil or #hex ~= 64 then + error('sha256 helper returned invalid digest: ' .. tostring(out), 0) + end + return hex:lower() +end + +local function temp_roots() + local base = ('/tmp/devicecode-mcu-http-uart-%d-%d'):format(os.time(), math.random(100000, 999999)) + rm_rf(base) + local roots = { + base = base, + config = base .. '/config', + static = base .. '/static', + artifact = { + transient = base .. '/cm5/artifacts/transient', + durable = base .. '/cm5/artifacts/durable', + import = base .. '/cm5/artifacts/import', + }, + control = base .. '/cm5/control/update', + mcu = base .. '/mcu', + } + mkdir_p(roots.config) + mkdir_p(roots.static) + mkdir_p(roots.artifact.transient) + mkdir_p(roots.artifact.durable) + mkdir_p(roots.artifact.import) + mkdir_p(roots.control) + mkdir_p(roots.mcu) + write_file(roots.static .. '/index.html', 'ok') + return roots +end + +local function copy_frame(frame) + local out = {} + for k, v in pairs(frame or {}) do + if type(v) == 'table' then + local t = {} + for k2, v2 in pairs(v) do t[k2] = v2 end + out[k] = t + else + out[k] = v + end + end + return out +end + +local function new_line_transport() + local in_tx, in_rx = mailbox.new(128, { full = 'reject_newest' }) + local written = {} + local session = {} + + function session:read_line_op() + return in_rx:recv_op():wrap(function(frame) + if frame == nil then return nil, in_rx:why() or 'closed' end + local line, err = fabric_protocol.encode_line(frame) + if not line then return nil, err end + return line, nil + end) + end + + function session:write_line_op(line) + local frame, err = fabric_protocol.decode_line(line) + if not frame then return fibers.always(nil, err) end + written[#written + 1] = copy_frame(frame) + return fibers.always(true, nil) + end + + function session:flush_op() + return fibers.always(true, nil) + end + + function session:terminate(reason) + in_tx:close(reason or 'transport terminated') + return true, nil + end + + return { session = session, in_tx = in_tx, written = written } +end + +local function connect_line_transports(a, b) + local write_a = a.session.write_line_op + function a.session:write_line_op(line) + local frame = assert(fabric_protocol.decode_line(line)) + write_a(self, line) + return b.in_tx:send_op(copy_frame(frame)) + end + + local write_b = b.session.write_line_op + function b.session:write_line_op(line) + local frame = assert(fabric_protocol.decode_line(line)) + write_b(self, line) + return a.in_tx:send_op(copy_frame(frame)) + end +end + +local function wrap_session_op(session) + local wrapped, err = hal_transport.wrap_transport(session) + return fibers.always(wrapped, err) +end + +local function wait_retained_payload_where(conn, topic, label, pred, opts) + opts = opts or {} + local view = conn:retained_view(topic) + local value = probe.wait_versioned_until(label, function () + return view:version() + end, function (seen) + return view:changed_op(seen) + end, function () + local msg = view:get(topic) + local payload = msg and msg.payload or nil + if pred(payload) then return payload end + return nil + end, opts) + view:close() + return value +end + + +local function topic_string(topic) + if type(topic) ~= 'table' then return tostring(topic) end + local out = {} + for i = 1, #topic do out[i] = tostring(topic[i]) end + return table.concat(out, '/') +end + +local function fabric_payload_snapshot(payload) + if type(payload) ~= 'table' then return nil end + return type(payload.snapshot) == 'table' and payload.snapshot or payload +end + +local function compact_fabric_session(s) + s = type(s) == 'table' and s or {} + return ('phase=%s established=%s local=%s peer_node=%s peer_sid=%s gen=%s wire_errors=%s bad_frames=%s last_wire_error=%s why=%s'):format( + tostring(s.phase), tostring(s.established), tostring(s.local_node), tostring(s.peer_node), + tostring(s.peer_sid), tostring(s.session_generation), tostring(s.wire_errors or 0), + tostring(s.bad_frame_count or 0), tostring(s.last_wire_error), tostring(s.why) + ) +end + +local function compact_fabric_bridge(s) + s = type(s) == 'table' and s or {} + return ('state=%s imported=%s pending=%s inbound=%s frames_sent=%s frames_recv=%s session_peer=%s drop=%s err=%s'):format( + tostring(s.state), tostring(s.imported_topics), tostring(s.pending_calls), tostring(s.inbound_calls), + tostring(s.frames_sent), tostring(s.frames_received), + tostring(type(s.session) == 'table' and s.session.peer_sid or nil), + tostring(s.session_drop_reason), tostring(s.last_err) + ) +end + +local function compact_fabric_transfer(s) + s = type(s) == 'table' and s or {} + local stats = type(s.stats) == 'table' and s.stats or {} + local active = type(s.active) == 'table' and s.active or nil + local last = type(s.last) == 'table' and s.last or nil + return ('active=%s active_status=%s last_status=%s completed=%s failed=%s cancelled=%s stale=%s'):format( + tostring(active ~= nil), tostring(active and active.status), tostring(last and last.status), + tostring(stats.completed), tostring(stats.failed), tostring(stats.cancelled), tostring(stats.stale) + ) +end + +local function compact_fabric_link(s) + s = type(s) == 'table' and s or {} + local comps = {} + if type(s.components) == 'table' then + for name, rec in pairs(s.components) do + comps[#comps + 1] = tostring(name) .. '=' .. tostring(type(rec) == 'table' and rec.status or rec) + end + table.sort(comps) + end + return ('state=%s completed=%s/%s reason=%s components=[%s]'):format( + tostring(s.state), tostring(s.completed), tostring(s.total), tostring(s.reason), table.concat(comps, ',') + ) +end + +local function describe_fabric_status_payload(payload, component) + local s = fabric_payload_snapshot(payload) + if component == 'session' then return compact_fabric_session(s) end + if component == 'rpc_bridge' then return compact_fabric_bridge(s) end + if component == 'transfer_manager' or component == 'transfer' then return compact_fabric_transfer(s) end + return compact_fabric_link(s) +end + +local function retained_payload_now(conn, topic) + local view = conn:retained_view(topic) + local msg = view:get(topic) + view:close() + return msg and msg.payload or nil +end + +local function log_fabric_status(conn, prefix) + prefix = prefix or 'fabric' + local topics = { + { label = 'link', topic = fabric_topics.state_link('link-a') }, + { label = 'session', component = 'session', topic = fabric_topics.state_link_component('link-a', 'session') }, + { label = 'rpc_bridge', component = 'rpc_bridge', topic = fabric_topics.state_link_component('link-a', 'rpc_bridge') }, + { label = 'transfer_manager', component = 'transfer_manager', topic = fabric_topics.state_link_component('link-a', 'transfer_manager') }, + } + for _, item in ipairs(topics) do + local payload = retained_payload_now(conn, item.topic) + log(('%s %s %s -> %s'):format(prefix, item.label, topic_string(item.topic), describe_fabric_status_payload(payload, item.component))) + end +end + +local function wait_fabric_session_established(conn, label, opts) + opts = opts or {} + local topic = fabric_topics.state_link_component('link-a', 'session') + local payload = wait_retained_payload_where(conn, topic, label or 'fabric session established', function (p) + local s = fabric_payload_snapshot(p) + if type(s) == 'table' and s.established == true and type(s.peer_sid) == 'string' and s.peer_sid ~= '' then + return p + end + return nil + end, { timeout = opts.timeout or 6.0 }) + log((label or 'fabric session established') .. ': ' .. describe_fabric_status_payload(payload, 'session')) + return payload +end + + +local function fabric_progress_fragment(conn) + if conn == nil then return '' end + local session = retained_payload_now(conn, fabric_topics.state_link_component('link-a', 'session')) + local bridge = retained_payload_now(conn, fabric_topics.state_link_component('link-a', 'rpc_bridge')) + local transfer = retained_payload_now(conn, fabric_topics.state_link_component('link-a', 'transfer_manager')) + return ('fabric_session=(%s) fabric_bridge=(%s) fabric_transfer=(%s)'):format( + describe_fabric_status_payload(session, 'session'), + describe_fabric_status_payload(bridge, 'rpc_bridge'), + describe_fabric_status_payload(transfer, 'transfer_manager') + ) +end + +local function fresh_uart_manager() + -- The UART manager is currently module-singleton state. The filtered test + -- runner still requires every spec module before it runs the selected test, + -- so make this test's direct manager instance explicit and isolated. + package.loaded['services.hal.managers.uart'] = nil + package.loaded['services.hal.drivers.uart'] = nil + return require 'services.hal.managers.uart' +end + +local function dummy_logger() + local logger = {} + for _, k in ipairs({ 'debug', 'info', 'warn', 'error' }) do + logger[k] = function () end + end + function logger:child() return self end + return logger +end + +local function wait_channel_get(ch, timeout_s, what) + local which, a, b = fibers.perform(op.named_choice({ + item = ch:get_op(), + timeout = sleep.sleep_op(timeout_s or 1.0), + })) + if which == 'timeout' then + error(('timed out waiting for %s'):format(what or 'channel item'), 0) + end + if a == nil then + error(('channel closed while waiting for %s: %s'):format(what or 'channel item', tostring(b)), 0) + end + return a +end + +local function wait_until_test(label, pred, timeout_s, interval_s) + local deadline = fibers.now() + (timeout_s or 2.0) + while fibers.now() < deadline do + if pred() then return true end + fibers.perform(sleep.sleep_op(interval_s or 0.05)) + end + error('timed out waiting for ' .. tostring(label), 0) +end + +local function wait_device_event(dev_ev_ch, event_type, class, id, timeout_s) + local deadline = fibers.now() + (timeout_s or 1.5) + while fibers.now() < deadline do + local ev = wait_channel_get(dev_ev_ch, deadline - fibers.now(), 'UART device event') + if ev.event_type == event_type and ev.class == class and ev.id == id then + return ev + end + end + error(('timed out waiting for UART device event %s %s/%s'):format( + tostring(event_type), tostring(class), tostring(id) + ), 0) +end + +local function wait_uart_cap(dev_ev_ch) + local added = wait_device_event(dev_ev_ch, 'added', 'uart', 'uart0', 1.5) + assert_true(type(added.capabilities) == 'table' and #added.capabilities == 1, 'UART added event missing capability') + local cap = added.capabilities[1] + assert_eq(cap.class, 'uart') + assert_eq(cap.id, 'uart0') + assert_true(type(cap.control_ch) == 'table', 'UART capability should expose control_ch') + return cap +end + +local function normalise_uart_open_opts(opts) + if opts == nil or getmetatable(opts) ~= cap_args.UARTOpenOpts then + local open_opts, err = cap_args.new.UARTOpenOpts(opts) + assert_not_nil(open_opts, tostring(err)) + return open_opts + end + return opts +end + +local function call_hal_control(cap, verb, opts) + local reply_ch = channel.new(1) + local req, err = hal_types.new.ControlRequest(verb, opts or {}, reply_ch) + assert_not_nil(req, tostring(err)) + + fibers.perform(cap.control_ch:put_op(req)) + + local reply = wait_channel_get(reply_ch, 1.0, 'HAL UART control reply') + assert_true(type(reply) == 'table', 'HAL control reply must be a table') + return reply +end + +local function expose_raw_host_uart_open(scope, bus, cap, source) + source = source or 'uart_manager' + local conn = bus:connect({ origin_base = { service = 'hal-uart-test-adapter' } }) + local cap_id = cap.id + local ep = conn:bind({ 'raw', 'host', source, 'cap', 'uart', cap_id, 'rpc', 'open' }, { + queue_len = 8, + }) + + conn:retain({ 'raw', 'host', source, 'status' }, { + state = 'available', + available = true, + source = source, + class = 'uart', + id = cap_id, + }) + conn:retain({ 'raw', 'host', source, 'meta' }, { + source = source, + class = 'uart', + id = cap_id, + }) + conn:retain({ 'raw', 'host', source, 'cap', 'uart', cap_id, 'status' }, { + state = 'available', + available = true, + source_kind = 'host', + source = source, + }) + conn:retain({ 'raw', 'host', source, 'cap', 'uart', cap_id, 'meta' }, { + source_kind = 'host', + source = source, + offerings = { open = true }, + }) + + -- Register cleanup before spawning work on this scope. lua-fibers only + -- allows adding finalisers to a started scope from within that same scope. + scope:finally(function () + safe.pcall(function () ep:unbind() end) + safe.pcall(function () conn:unretain({ 'raw', 'host', source, 'cap', 'uart', cap_id, 'meta' }) end) + safe.pcall(function () conn:retain({ 'raw', 'host', source, 'cap', 'uart', cap_id, 'status' }, { + state = 'removed', + available = false, + source_kind = 'host', + source = source, + }) end) + safe.pcall(function () conn:unretain({ 'raw', 'host', source, 'meta' }) end) + safe.pcall(function () conn:retain({ 'raw', 'host', source, 'status' }, { + state = 'removed', + available = false, + source = source, + class = 'uart', + id = cap_id, + }) end) + bus_cleanup.disconnect(conn) + end) + + local ok_spawn, spawn_err = scope:spawn(function () + while true do + local req = ep:recv() + if req == nil then return end + + local open_opts = normalise_uart_open_opts(req.payload) + local reply = call_hal_control(cap, 'open', open_opts) + + local replied = req:reply(reply) + if not replied + and reply.ok == true + and type(reply.reason) == 'table' + and type(reply.reason.session) == 'table' + and type(reply.reason.session.terminate) == 'function' + then + reply.reason.session:terminate('fabric request abandoned') + end + end + end) + assert_true(ok_spawn, tostring(spawn_err)) + + return { source = source, class = 'uart', id = cap_id } +end + +local function start_cm5_uart_manager(scope, bus, port) + local uart_mgr = fresh_uart_manager() + local dev_ev_ch = channel.new(16) + local cap_emit_ch = channel.new(32) + + local ok_start, start_err = fibers.perform(uart_mgr.start_op(dummy_logger(), dev_ev_ch, cap_emit_ch)) + assert_true(ok_start, tostring(start_err)) + scope:finally(function () + safe.pcall(function () fibers.perform(uart_mgr.shutdown_op()) end) + end) + + local ok_cfg, cfg_err = fibers.perform(uart_mgr.apply_config_op({ + serial_ports = { + { + id = 'uart0', + path = port.slave_name, + baud = 115200, + mode = '8N1', + }, + }, + })) + assert_true(ok_cfg, tostring(cfg_err)) + + local cap = wait_uart_cap(dev_ev_ch) + local raw_cap = expose_raw_host_uart_open(scope, bus, cap, 'uart_manager') + + local conn = bus:connect({ origin_base = { service = 'hal-uart-test-wait' } }) + wait_retained_payload_where(conn, { 'raw', 'host', raw_cap.source, 'cap', raw_cap.class, raw_cap.id, 'status' }, + 'HAL UART raw capability available', function (p) + return p and p.available == true and p + end, { timeout = 1.0 }) + bus_cleanup.disconnect(conn) + + return raw_cap +end + +local function wait_job(conn, job_id, state, timeout) + return wait_retained_payload_where(conn, update_topics.update_component('mcu'), 'update job ' .. tostring(job_id) .. ' ' .. tostring(state), function (p) + local job = p and (p.current_job or p.last_job) or nil + if job and job.job_id == job_id and job.state == state then return job end + return nil + end, { timeout = timeout or 4.0 }) +end + +local function describe_job_value(v, depth) + depth = depth or 0 + if type(v) ~= 'table' then return tostring(v) end + if depth >= 2 then return '
' end + local parts = {} + for k, vv in pairs(v) do + if type(vv) ~= 'function' then + parts[#parts + 1] = tostring(k) .. '=' .. describe_job_value(vv, depth + 1) + end + end + table.sort(parts) + return '{' .. table.concat(parts, ',') .. '}' +end + +local function describe_job_progress(p) + if type(p) ~= 'table' then return 'absent' end + local parts = { 'state=' .. tostring(p.state) } + if p.stage_attempt ~= nil then + local st = p.stage_attempt + parts[#parts + 1] = 'stage=' .. tostring(type(st) == 'table' and (st.status or st.state or st.phase) or st) + if type(st) == 'table' and st.err ~= nil then parts[#parts + 1] = 'stage_err=' .. tostring(st.err) end + end + if p.component ~= nil then parts[#parts + 1] = 'component=' .. tostring(p.component) end + if p.err ~= nil then parts[#parts + 1] = 'err=' .. describe_job_value(p.err) end + if p.error ~= nil then parts[#parts + 1] = 'error=' .. describe_job_value(p.error) end + if p.reason ~= nil then parts[#parts + 1] = 'reason=' .. describe_job_value(p.reason) end + return table.concat(parts, ' ') +end + +local function wait_job_chatty(conn, job_id, state, timeout, progress_fn) + timeout = timeout or 4.0 + local topic = update_topics.update_component('mcu') + local view = conn:retained_view(topic) + local deadline = fibers.now() + timeout + local last_log = -math.huge + + local function maybe_log(force) + local now = fibers.now() + if not force and (now - last_log) < WAIT_PROGRESS_LOG_S then return end + last_log = now + local msg = view:get(topic) + local payload = msg and msg.payload or nil + local extra = progress_fn and progress_fn() or '' + if extra ~= '' then extra = '; ' .. extra end + log(('waiting for job %s -> %s: %s%s'):format(job_id, state, describe_job_progress(payload), extra)) + end + + local payload = (view:get(topic) or {}).payload + local job = payload and (payload.current_job or payload.last_job) or nil + if job and job.job_id == job_id and job.state == state then + view:close() + return job + end + maybe_log(true) + + local seen = view:version() + while true do + local remaining = deadline - fibers.now() + if remaining <= 0 then + maybe_log(true) + view:close() + error('timed out waiting for update job ' .. tostring(job_id) .. ' ' .. tostring(state), 0) + end + + local which, version, reason = fibers.perform(op.named_choice({ + changed = view:changed_op(seen), + progress = sleep.sleep_op(math.min(WAIT_PROGRESS_LOG_S, remaining)), + })) + + if which == 'changed' then + if version == nil then + view:close() + error('update job ' .. tostring(job_id) .. ' closed: ' .. tostring(reason or 'closed'), 0) + end + seen = version + payload = (view:get(topic) or {}).payload + local job = payload and (payload.current_job or payload.last_job) or nil + if job and job.job_id == job_id and job.state == state then + maybe_log(true) + view:close() + return job + end + end + + maybe_log(which == 'changed') + end +end + +local function wait_component_software(conn, image_id, boot_id) + return wait_retained_payload_where(conn, device_topics.component_software('mcu'), 'mcu software canonical state', function (p) + if p and (image_id == nil or p.image_id == image_id) and (boot_id == nil or p.boot_id == boot_id) then return p end + return nil + end, { timeout = 4.0 }) +end + +local function start_public_fabric(scope, conn, cfg, transport, opts) + opts = opts or {} + local link_overrides = opts.link_overrides + if link_overrides == nil and transport ~= nil then + link_overrides = { + ['link-a'] = { + open_transport_op = function () return wrap_session_op(transport.session) end, + transfer = opts.transfer, + }, + } + end + local ok, err = scope:spawn(function () + fabric.start(conn, { + name = opts.name or 'fabric', + env = 'test', + config = cfg, + link_overrides = link_overrides, + }) + end) + assert_true(ok, tostring(err)) +end + +local function fabric_config(local_node, peer_node, bridge, transfer) + return { + schema = fabric.config.SCHEMA, + local_node = local_node, + links = { + { + id = 'link-a', + peer_id = peer_node, + transport = { source = 'test', class = 'jsonl', id = 'link-a' }, + session = { hello_interval_s = 5.0, ping_interval_s = 5.0, liveness_timeout_s = 5.0 }, + bridge = bridge or {}, + transfer = transfer or { chunk_size = 2048, timeout_s = 3.0 }, + }, + }, + } +end + +local function cm5_fabric_config() + return fabric_config('cm5', 'mcu', { + imports = { + { id = 'mcu-state', remote = { 'state', 'self' }, ['local'] = { 'raw', 'member', 'mcu', 'state' } }, + { id = 'mcu-event', remote = { 'event', 'self' }, ['local'] = { 'raw', 'member', 'mcu', 'cap', 'telemetry', 'main', 'event' } }, + { id = 'mcu-cap', remote = { 'cap', 'self' }, ['local'] = { 'raw', 'member', 'mcu', 'cap' } }, + }, + rpc = { + outbound = { + { + id = 'mcu-prepare', + ['local'] = { 'raw', 'member', 'mcu', 'cap', 'updater', 'main', 'rpc', 'prepare-update' }, + remote = { 'cap', 'self', 'updater', 'main', 'rpc', 'prepare-update' }, + timeout_s = 2.0, + }, + { + id = 'mcu-commit', + ['local'] = { 'raw', 'member', 'mcu', 'cap', 'updater', 'main', 'rpc', 'commit-update' }, + remote = { 'cap', 'self', 'updater', 'main', 'rpc', 'commit-update' }, + timeout_s = 2.0, + }, + }, + }, + }, { chunk_size = 2048, timeout_s = REALISTIC_TRANSFER_TIMEOUT_S }) +end + +local function mcu_fabric_config() + return fabric_config('mcu', 'cm5', { + exports = { + { id = 'mcu-state-export', ['local'] = { 'state', 'self' }, remote = { 'state', 'self' }, publish = true, retain = true }, + { id = 'mcu-cap-export', ['local'] = { 'cap', 'self' }, remote = { 'cap', 'self' }, publish = true, retain = true }, + }, + rpc = { + inbound = { + { + id = 'mcu-prepare-in', + ['local'] = { 'cap', 'self', 'updater', 'main', 'rpc', 'prepare-update' }, + remote = { 'cap', 'self', 'updater', 'main', 'rpc', 'prepare-update' }, + timeout_s = 2.0, + }, + { + id = 'mcu-commit-in', + ['local'] = { 'cap', 'self', 'updater', 'main', 'rpc', 'commit-update' }, + remote = { 'cap', 'self', 'updater', 'main', 'rpc', 'commit-update' }, + timeout_s = 2.0, + }, + }, + }, + }, { chunk_size = 2048, timeout_s = REALISTIC_TRANSFER_TIMEOUT_S }) +end + +local function publish_mcu_facts(conn, fake) + local image = fake.committed_image_id or fake.old_image_id + local boot = 'mcu-boot-' .. tostring(fake.boot_seq) + conn:retain({ 'state', 'self', 'software' }, { + image_id = image, + boot_id = boot, + version = image, + }) + conn:retain({ 'state', 'self', 'updater' }, { + state = 'ready', + last_error = nil, + staged_image_id = fake.staged and fake.staged.image_id or nil, + pending_image_id = fake.committed_image_id, + job_id = fake.job_id, + }) + conn:retain({ 'cap', 'self', 'updater', 'main', 'meta' }, { + class = 'updater', + id = 'main', + methods = { 'prepare-update', 'commit-update' }, + }) + conn:retain({ 'cap', 'self', 'updater', 'main', 'status' }, { + available = true, + state = 'available', + }) +end + +local function new_mcu_receive_target(fake) + local target = {} + function target:open_sink_op(req) + fake.transfer_begin = req + fake.receive_started_at = fibers.now() + fake.receive_bytes = 0 + fake.receive_chunks = 0 + fake.receive_next_log = MCU_PROGRESS_LOG_BYTES + assert_eq(req.target, 'updater/main') + log(('MCU receive target opened: target=%s size=%s job_id=%s image_id=%s'):format( + tostring(req.target), + tostring(req.size), + tostring(req.meta and req.meta.job_id), + tostring(req.meta and (req.meta.image_id or req.meta.expected_image_id)) + )) + local chunks = {} + local sink = {} + function sink:append_op(chunk) + return fibers.run_scope_op(function () + -- Simulate a small flash/programming delay on the MCU side. + -- This keeps the receiver honest without changing the transfer + -- protocol: it still only asks for the current offset. + fibers.perform(sleep.sleep_op(MCU_FLASH_WRITE_DELAY_S)) + chunks[#chunks + 1] = chunk + fake.receive_bytes = (fake.receive_bytes or 0) + #(chunk or '') + fake.receive_chunks = (fake.receive_chunks or 0) + 1 + if fake.receive_bytes >= (fake.receive_next_log or MCU_PROGRESS_LOG_BYTES) then + local elapsed = fibers.now() - (fake.receive_started_at or fibers.now()) + log(('MCU received %d bytes in %d chunks after %.1fs'):format( + fake.receive_bytes, fake.receive_chunks or 0, elapsed + )) + fake.receive_next_log = fake.receive_bytes + MCU_PROGRESS_LOG_BYTES + end + return true, nil + end):wrap(function (status, _report, ok, err) + if status ~= 'ok' then return nil, err or status end + return ok, err + end) + end + function sink:commit_op(req2) + local bytes = table.concat(chunks) + log(('MCU transfer commit: %d bytes in %d chunks; digest=%s'):format( + #bytes, fake.receive_chunks or 0, tostring(req2.digest) + )) + fake.staged = { + bytes = bytes, + digest = req2.digest, + size = req2.size, + image_id = req.meta and (req.meta.image_id or req.meta.expected_image_id), + job_id = req.meta and req.meta.job_id, + } + fake.staged_signal = (fake.staged_signal or 0) + 1 + return fibers.always({ staged = true, digest = req2.digest }, nil) + end + function sink:abort(reason) + fake.abort_reason = reason + fake.abort_count = (fake.abort_count or 0) + 1 + log(('MCU transfer abort: reason=%s after %d bytes in %d chunks'):format( + tostring(reason), fake.receive_bytes or 0, fake.receive_chunks or 0 + )) + return true, nil + end + return fibers.always(sink, nil) + end + return target +end + +local function start_fake_mcu(scope, bus, fake) + local conn = bus:connect({ origin_base = { service = 'fake-mcu' } }) + fake.boot_seq = (fake.boot_seq or 0) + 1 + publish_mcu_facts(conn, fake) + + local eps = {} + local function bind(topic) + local ep, err = bus_cleanup.bind(conn, topic, { queue_len = 8 }) + assert_not_nil(ep, err) + eps[#eps + 1] = ep + return ep + end + + local prepare_ep = bind({ 'cap', 'self', 'updater', 'main', 'rpc', 'prepare-update' }) + local commit_ep = bind({ 'cap', 'self', 'updater', 'main', 'rpc', 'commit-update' }) + + scope:finally(function () + for _, ep in ipairs(eps) do bus_cleanup.unbind(conn, ep) end + bus_cleanup.disconnect(conn) + end) + + assert_true(scope:spawn(function () + while true do + local req = fibers.perform(prepare_ep:recv_op()) + if req == nil then return end + fake.prepare_payload = req.payload + assert_eq(type(req.payload) == 'table' and req.payload.target or nil, 'mcu') + fake.job_id = type(req.payload) == 'table' and req.payload.job_id or nil + conn:retain({ 'state', 'self', 'updater' }, { state = 'ready', last_error = nil, job_id = fake.job_id }) + req:reply({ ready = true, target = 'updater/main', max_chunk_size = 2048 }) + end + end)) + + assert_true(scope:spawn(function () + while true do + local req = fibers.perform(commit_ep:recv_op()) + if req == nil then return end + fake.commit_payload = req.payload + assert_eq(type(req.payload), 'table') + assert_eq(req.payload.job_id, fake.job_id) + assert_eq(req.payload.metadata, nil) + fake.commit_seen = true + fake.committed_image_id = (fake.staged and fake.staged.image_id) + or (type(req.payload) == 'table' and req.payload.expected_image_id) + conn:retain({ 'state', 'self', 'updater' }, { + state = 'rebooting', + last_error = nil, + pending_image_id = fake.committed_image_id, + staged_image_id = fake.staged and fake.staged.image_id or nil, + job_id = fake.job_id, + }) + req:reply({ accepted = true, reboot_required = true }) + end + end)) + + return conn +end + +local function new_fabric_client(conn) + local client = {} + function client:send_blob_op(params, opts) + params = params or {} + opts = opts or {} + assert(params.source_owner, 'fabric client source_owner required') + return conn:call_op(fabric_topics.transfer_manager_rpc('send-blob'), { + link_id = params.link_id or 'link-a', + request_id = params.request_id or ('device-stage-' .. tostring(params.job_id or os.clock())), + xfer_id = params.xfer_id, + target = assert(params.target, 'fabric client params.target required'), + source_owner = params.source_owner, + size = params.size, + digest_alg = params.digest_alg, + digest = params.digest, + chunk_size = params.chunk_size or 2048, + meta = params.meta, + timeout_s = opts.timeout or params.timeout or REALISTIC_TRANSFER_TIMEOUT_S, + }, { timeout = opts.timeout or params.timeout or REALISTIC_TRANSFER_TIMEOUT_S }):wrap(function (reply, err) + if reply == nil then return nil, err end + return { + ok = reply.ok, + committed = reply.committed, + transfer = reply, + }, nil + end) + end + return client +end + +local function update_config(opts) + opts = opts or {} + return { + schema = 'devicecode.update/1', + components = { + { + component = 'mcu', + -- Active update owns the phase budget. The Update backend + -- opens the Device bus call without lua-bus' default timeout; + -- active_job.stage composes the backend Op with this deadline. + stage_timeout_s = opts.stage_timeout_s or REALISTIC_TRANSFER_TIMEOUT_S, + }, + }, + } +end + +local function device_config(opts) + opts = opts or {} + local stage_timeout_s = opts.stage_action_timeout_s or REALISTIC_TRANSFER_TIMEOUT_S + return { + schema = 'devicecode.config/device/1', + components = { + mcu = { + class = 'member', + subtype = 'mcu', + role = 'controller', + member = 'mcu', + required_facts = { 'software', 'updater' }, + facts = mcu_schema.member_fact_topics('mcu'), + events = mcu_schema.member_event_topics('mcu'), + actions = { + ['restart'] = { kind = 'rpc', call_topic = device_topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') }, + ['prepare-update'] = { + kind = 'rpc', + call_topic = device_topics.raw_member_cap_rpc('mcu', 'updater', 'main', 'prepare-update'), + timeout_s = 2.0, + }, + ['stage-update'] = { + kind = 'fabric_stage', + target = 'updater/main', + chunk_size = 2048, + artifact_store = 'main', + -- This is Device action policy, not a hidden bus timeout. + -- Keep it in the component config so the test does not + -- change Device's global default action timeout. + timeout_s = stage_timeout_s, + }, + ['commit-update'] = { + kind = 'rpc', + call_topic = device_topics.raw_member_cap_rpc('mcu', 'updater', 'main', 'commit-update'), + timeout_s = 2.0, + }, + }, + }, + }, + } +end + +local function hal_config(roots) + return { + schema = 'devicecode.config/hal/1', + artifact_store = { + stores = { + { + id = 'main', + transient_root = roots.artifact.transient, + durable_root = roots.artifact.durable, + import_root = roots.artifact.import, + durable_enabled = true, + }, + }, + }, + control_store = { + { name = 'update', root = roots.control }, + }, + } +end + +local function start_hal(scope, bus, roots) + local conn = bus:connect({ origin_base = { service = 'hal' } }) + if posix_ok and stdlib and stdlib.setenv then + stdlib.setenv('DEVICECODE_CONFIG_DIR', roots.config, true) + end + assert_true(scope:spawn(function () + hal_service.start(conn, { name = 'hal', env = 'test', heartbeat_s = false }) + end)) + conn:retain({ 'cfg', 'hal' }, { data = hal_config(roots) }) + wait_retained_payload_where(conn, { 'cap', 'artifact-store', 'main', 'status' }, 'artifact-store available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + wait_retained_payload_where(conn, { 'cap', 'control-store', 'update', 'status' }, 'control-store available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + return conn +end + +local function start_http(scope, bus) + local conn = bus:connect({ origin_base = { service = 'http' } }) + assert_true(scope:spawn(function (s) + http_service.run(s, { + conn = conn, + id = 'main', + config = { + schema = 'devicecode.config/http/1', + id = 'main', + policy = { allow_loopback = true, max_request_body = 16 * 1024 * 1024, max_response_body = 16 * 1024 * 1024 }, + }, + backend_timeout = 3.0, + connection_setup_timeout = 3.0, + intra_stream_timeout = 3.0, + }) + end)) + wait_retained_payload_where(conn, { 'cap', 'http', 'main', 'status' }, 'http service available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + return conn +end + +local function start_device(scope, bus, fabric_client, opts) + opts = opts or {} + local conn = bus:connect({ origin_base = { service = 'device' } }) + assert_true(scope:spawn(function (s) + device_service.run(s, { + conn = conn, + initial_config = device_config(opts), + fabric_client = fabric_client, + watch_config = false, + }) + end)) + wait_retained_payload_where(conn, device_topics.component_cap_status('mcu'), 'mcu component cap available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + return conn +end + +local function start_update(scope, bus, opts) + opts = opts or {} + local conn = bus:connect({ origin_base = { service = 'update' } }) + assert_true(scope:spawn(function (s) + update_service.run(s, { + conn = conn, + service_id = 'update', + watch_config = false, + config = update_config(opts), + job_store_call_opts = { timeout = 3.0 }, + }) + end)) + wait_retained_payload_where(conn, update_topics.update_manager_status(), 'update manager available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + wait_retained_payload_where(conn, update_topics.artifact_ingest_status(), 'artifact ingest available', function (p) + return p and p.available == true and p + end, { timeout = 4.0 }) + return conn +end + +local function start_ui(scope, bus, port, roots) + local conn = bus:connect({ origin_base = { service = 'ui' } }) + assert_true(scope:spawn(function (s) + ui_service.run(s, { + conn = conn, + service_id = 'ui', + auth_opts = { + users = { + tester = { password = 'test-password', principal = { kind = 'user', id = 'tester' } }, + }, + }, + bus = bus, + connect = function (principal) + return bus:connect({ principal = principal or { kind = 'ui-test' } }) + end, + encode_json = function (v) return assert(cjson.encode(v)) end, + update = { + bus = bus, + component = 'mcu', + ingest_id = 'ing-mcu-http-uart', + job_id = 'job-mcu-http-uart', + create_job = true, + start_job = true, + timeout = 8.0, + chunk_size = 4096, + metadata = { + source = 'browser', + format = 'dcmcu-v1', + }, + }, + updates = { upload = { enabled = true, max_bytes = 1024 * 1024, require_auth = false, component = 'mcu', create_job = true, start_job = true }, commit = { require_auth = false } }, + }) + end)) + conn:retain({ 'cfg', 'ui' }, { data = { + schema = 'devicecode.config/ui/1', + enabled = true, + http = { enabled = true, cap_id = 'main', host = '127.0.0.1', port = port, max_active_requests = 8 }, + static = { root = roots.static, index = 'index.html' }, + updates = { upload = { enabled = true, max_bytes = 1024 * 1024, require_auth = false, component = 'mcu', create_job = true, start_job = true }, commit = { require_auth = false } }, + sse = { enabled = false }, + sessions = { prune_interval = false }, + } }) + wait_retained_payload_where(conn, http_topics.state('main', 'stats'), 'ui listener active', function (p) + return p and type(p.active_listeners) == 'number' and p.active_listeners > 0 and p + end, { timeout = 4.0 }) + return conn +end + +local function make_realistic_mcu_blob(image_id, target_bytes) + target_bytes = target_bytes or REALISTIC_MCU_BLOB_BYTES + local payload = string.rep('payload-0123456789abcdef-', math.ceil(target_bytes / 24)):sub(1, target_bytes) + return dcmcu_fixture.make(image_id or 'mcu-image-new', payload, { + payload_sha256 = sha256_hex_string(payload), + }) +end + +local function write_all(stream, data) + local off = 1 + while off <= #data do + local n, err = fibers.perform(stream:write_op(data:sub(off))) + if n == nil then return nil, err or 'write_failed' end + if n <= 0 then return nil, 'zero_length_write' end + off = off + n + end + return true, nil +end + + +local function fault_for_direction(stats, direction) + local faults = type(stats.faults) == 'table' and stats.faults or {} + return type(faults[direction]) == 'table' and faults[direction] or {} +end + +local function maybe_inject_malformed_jsonl(direction, output, stats) + local f = fault_for_direction(stats, direction) + local default_direction = stats.default_malformed_direction or 'cm5_to_mcu' + if f.disable_malformed == true then return true, nil end + if f.malformed_line_count == nil and direction ~= default_direction then return true, nil end + local max_lines = f.malformed_line_count or stats.malformed_line_count or UART_MALFORMED_LINE_COUNT + local key = 'malformed_lines_' .. direction + if (stats[key] or 0) >= max_lines then return true, nil end + local next_key = 'next_malformed_at_' .. direction + local next_at = stats[next_key] or f.malformed_line_first_at or stats.malformed_line_first_at or UART_MALFORMED_LINE_FIRST_AT + if (stats.bytes or 0) < next_at then return true, nil end + + stats.malformed_lines = (stats.malformed_lines or 0) + 1 + stats[key] = (stats[key] or 0) + 1 + stats[next_key] = next_at + (f.malformed_line_every_bytes or stats.malformed_line_every_bytes or UART_MALFORMED_LINE_EVERY_BYTES) + local line = ('{ malformed json from mcu_http_uart test #%d on %s }\n'):format(stats.malformed_lines, direction) + local ok, err = write_all(output.master, line) + if ok ~= true then return nil, err or 'malformed_jsonl_inject_failed' end + + stats.bytes = (stats.bytes or 0) + #line + stats.fragments = (stats.fragments or 0) + 1 + stats.malformed_bytes = (stats.malformed_bytes or 0) + #line + log(('UART middleware injected standalone malformed JSONL line #%d on %s at %d bytes'):format( + stats.malformed_lines, direction, stats.bytes + )) + return true, nil +end + +local function fault_drop_line_pattern(f) + if type(f) ~= 'table' then return nil end + if type(f.drop_byte_when_line_contains) == 'string' and f.drop_byte_when_line_contains ~= '' then + return f.drop_byte_when_line_contains + end + if type(f.drop_byte_in_frame_type) == 'string' and f.drop_byte_in_frame_type ~= '' then + -- Fabric JSONL encoders in both Go and Lua emit compact JSON. Match + -- the type field rather than a byte-count so the fault remains + -- deterministic as retained telemetry volume changes. + return '"type":"' .. f.drop_byte_in_frame_type .. '"' + end + return nil +end + +local function remember_fault_line_tail(f, pattern, combined, piece) + if type(piece) ~= 'string' or piece == '' then return end + if piece:find('\n', 1, true) then + f.drop_line_buf = '' + return + end + local keep = math.max(1, math.min(#combined, #pattern - 1)) + f.drop_line_buf = combined:sub(#combined - keep + 1) +end + +local function maybe_drop_targeted_uart_byte(direction, stats, piece) + local f = fault_for_direction(stats, direction) + if f.drop_byte_done == true then return piece end + local pattern = fault_drop_line_pattern(f) + if not pattern then return piece end + + local prior = f.drop_line_buf or '' + local combined = prior .. piece + local _, match_end = combined:find(pattern, 1, true) + if not match_end then + remember_fault_line_tail(f, pattern, combined, piece) + return piece + end + + local rel = match_end - #prior + if rel < 1 then rel = 1 end + if rel > #piece then rel = #piece end + + f.drop_byte_done = true + f.drop_line_buf = nil + stats.dropped_bytes = (stats.dropped_bytes or 0) + 1 + log(('UART middleware dropped byte #%d on %s matching %q rel=%d'):format( + stats.dropped_bytes, direction, pattern, rel + )) + return piece:sub(1, rel - 1) .. piece:sub(rel + 1) +end + +local function apply_uart_faults(direction, stats, piece) + local f = fault_for_direction(stats, direction) + if type(piece) ~= 'string' or piece == '' then return piece end + + local dir_bytes = type(stats.bytes_by_direction) == 'table' and (stats.bytes_by_direction[direction] or 0) or 0 + local function crosses(after) + return type(after) == 'number' and dir_bytes < after and (dir_bytes + #piece) >= after + end + + if f.pause_once_after_bytes and not f.pause_once_done and crosses(f.pause_once_after_bytes) then + f.pause_once_done = true + local pause_s = f.pause_s or 0.75 + stats.fault_pauses = (stats.fault_pauses or 0) + 1 + log(('UART middleware fault pause #%d on %s for %.3fs at %d direction-bytes'):format( + stats.fault_pauses, direction, pause_s, dir_bytes + )) + fibers.perform(sleep.sleep_op(pause_s)) + end + + if f.drop_byte_after_bytes and not f.drop_byte_done and crosses(f.drop_byte_after_bytes) then + local rel = math.max(1, math.min(#piece, f.drop_byte_after_bytes - dir_bytes)) + f.drop_byte_done = true + stats.dropped_bytes = (stats.dropped_bytes or 0) + 1 + log(('UART middleware dropped byte #%d on %s at stream byte %d rel=%d'):format( + stats.dropped_bytes, direction, f.drop_byte_after_bytes, rel + )) + piece = piece:sub(1, rel - 1) .. piece:sub(rel + 1) + end + + piece = maybe_drop_targeted_uart_byte(direction, stats, piece) + + local newline_after = f.drop_next_newline_after_bytes or f.drop_newline_after_bytes + if newline_after and not f.drop_newline_done then + if not f.drop_newline_armed and ((type(newline_after) ~= 'number') or (dir_bytes + #piece) >= newline_after) then + f.drop_newline_armed = true + log(('UART middleware armed next-newline drop on %s after %s direction-bytes'):format( + direction, tostring(newline_after) + )) + end + if f.drop_newline_armed then + local nl = piece:find('\n', 1, true) + if nl ~= nil then + f.drop_newline_done = true + stats.dropped_newlines = (stats.dropped_newlines or 0) + 1 + log(('UART middleware dropped newline #%d on %s after threshold %s rel=%d'):format( + stats.dropped_newlines, direction, tostring(newline_after), nl + )) + piece = piece:sub(1, nl - 1) .. piece:sub(nl + 1) + end + end + end + + return piece +end + +local function write_uart_fragment_piece(direction, output, stats, piece) + if piece == '' then return true, nil end + piece = apply_uart_faults(direction, stats, piece) + if piece == '' then return true, nil end + local ok, werr = write_all(output.master, piece) + if ok ~= true then + return nil, werr or 'write_failed' + end + stats.bytes = (stats.bytes or 0) + #piece + stats.bytes_by_direction = stats.bytes_by_direction or {} + stats.bytes_by_direction[direction] = (stats.bytes_by_direction[direction] or 0) + #piece + stats.fragments = (stats.fragments or 0) + 1 + + if piece:sub(-1) == '\n' then + local mok, merr = maybe_inject_malformed_jsonl(direction, output, stats) + if mok ~= true then return nil, merr end + end + + return true, nil +end + +local function relay_fragmented_uart(direction, input, output, stats) + while true do + local want = math.random(1, UART_MAX_READ_BYTES) + local chunk, err = fibers.perform(input.master:read_some_op(want)) + if chunk == nil then + return { direction = direction, reason = err or 'closed' } + end + + local off = 1 + while off <= #chunk do + local frag_len = math.random(1, UART_MAX_FRAGMENT_BYTES) + local frag = chunk:sub(off, off + frag_len - 1) + local frag_off = 1 + while frag_off <= #frag do + local nl = frag:find('\n', frag_off, true) + local piece + if nl ~= nil then + piece = frag:sub(frag_off, nl) + frag_off = nl + 1 + else + piece = frag:sub(frag_off) + frag_off = #frag + 1 + end + local ok, werr = write_uart_fragment_piece(direction, output, stats, piece) + if ok ~= true then + return { direction = direction, reason = werr or 'write_failed' } + end + end + off = off + #frag + + if stats.bytes >= (stats.next_long_pause_at or UART_LONG_PAUSE_EVERY_BYTES) then + stats.pauses = (stats.pauses or 0) + 1 + stats.next_long_pause_at = (stats.next_long_pause_at or UART_LONG_PAUSE_EVERY_BYTES) + UART_LONG_PAUSE_EVERY_BYTES + local pause_s = UART_LONG_PAUSE_BASE_S + (math.random() * UART_LONG_PAUSE_JITTER_S) + log(('UART middleware pause #%d on %s for %.3fs at %d bytes'):format( + stats.pauses, direction, pause_s, stats.bytes + )) + fibers.perform(sleep.sleep_op(pause_s)) + end + + -- Approximate 115200 8N1 UART payload rate, with modest scheduling + -- jitter and short fragments. This tests the release protocol under + -- realistic pacing without injecting corrupt or reordered bytes. + fibers.perform(sleep.sleep_op((#frag / UART_BYTES_PER_SEC) + (math.random(0, 4) / 1000))) + end + end +end + +local function start_noisy_uart_middleware(scope, cm5_port, mcu_port, opts) + opts = opts or {} + local stats = { + bytes = 0, + fragments = 0, + pauses = 0, + malformed_lines = 0, + malformed_bytes = 0, + dropped_bytes = 0, + dropped_newlines = 0, + fault_pauses = 0, + next_long_pause_at = opts.long_pause_every_bytes or UART_LONG_PAUSE_EVERY_BYTES, + malformed_line_count = opts.malformed_line_count, + malformed_line_first_at = opts.malformed_line_first_at, + malformed_line_every_bytes = opts.malformed_line_every_bytes, + default_malformed_direction = opts.default_malformed_direction, + faults = opts.faults or {}, + bytes_by_direction = {}, + } + local ok_a, err_a = scope:spawn(function () + return relay_fragmented_uart('cm5_to_mcu', cm5_port, mcu_port, stats) + end) + assert_true(ok_a, tostring(err_a)) + local ok_b, err_b = scope:spawn(function () + return relay_fragmented_uart('mcu_to_cm5', mcu_port, cm5_port, stats) + end) + assert_true(ok_b, tostring(err_b)) + return stats +end + +local function open_pty_transport_op(slave_name) + return op.guard(function () + local stream, err = pty.open_slave_stream(slave_name) + if not stream then return fibers.always(nil, err or 'pty_slave_open_failed') end + local transport, terr = hal_transport.wrap_transport(stream, { terminator = '\n' }) + if not transport then + if type(stream.terminate) == 'function' then stream:terminate(terr or 'transport_wrap_failed') end + return fibers.always(nil, terr or 'transport_wrap_failed') + end + return fibers.always(transport, nil) + end) +end + +local function cm5_uart_fabric_config() + local cfg = cm5_fabric_config() + cfg.links[1].transport = { + source = 'uart_manager', + class = 'uart', + id = 'uart0', + terminator = '\n', + } + cfg.links[1].session.hello_interval_s = 0.20 + return cfg +end + +local function mcu_pty_fabric_config() + local cfg = mcu_fabric_config() + cfg.links[1].session.hello_interval_s = 0.20 + return cfg +end + +local function start_fabric_pair(parent_scope, cm5_bus, fake, uart, opts) + opts = opts or {} + local pair_scope = assert(parent_scope:child()) + local stats = start_noisy_uart_middleware(pair_scope, assert(uart and uart.cm5, 'cm5 PTY required'), assert(uart and uart.mcu, 'mcu PTY required'), opts.uart) + + local cm5_conn = cm5_bus:connect({ origin_base = { service = 'fabric-cm5' } }) + + -- CM5 uses the real HAL raw-host UART capability path. The MCU side is a + -- separate Lua process with its own bus, scheduler and HAL UART manager. + start_public_fabric(pair_scope, cm5_conn, cm5_uart_fabric_config(), nil, { name = 'fabric-cm5' }) + + wait_retained_payload_where(cm5_conn, fabric_topics.transfer_manager_status(), 'cm5 fabric transfer manager available', function (p) + return p and p.available == true and p + end, { timeout = 6.0 }) + + local cm5_session = wait_fabric_session_established(cm5_conn, 'CM5 fabric hello/hello_ack session established', { timeout = 6.0 }) + local cm5_snapshot = fabric_payload_snapshot(cm5_session) or {} + assert_eq(cm5_snapshot.peer_node, 'mcu', 'CM5 fabric session should identify MCU peer') + if not (fake and fake.expect_mcu_fabric_event == false) then + wait_until_test('MCU child fabric hello/hello_ack session established', function () + return fake and fake.mcu_fabric_session_seen == true + end, 6.0, 0.05) + assert_eq(fake.mcu_fabric_session and fake.mcu_fabric_session.peer_node, 'cm5', 'MCU fabric session should identify CM5 peer') + end + log_fabric_status(cm5_conn, 'CM5 fabric established status') + + return { + scope = pair_scope, + cm5_conn = cm5_conn, + uart_stats = stats, + } +end + +local function stop_fabric_pair(pair, reason) + if not pair then return end + if pair.scope then + pair.scope:cancel(reason or 'fabric pair stop') + fibers.perform(pair.scope:join_op()) + end + if pair.cm5_conn then bus_cleanup.disconnect(pair.cm5_conn) end + if pair.mcu_conn then bus_cleanup.disconnect(pair.mcu_conn) end +end + +local function start_cm5_instance(parent_scope, roots, http_port, uart_port, opts) + opts = opts or {} + local scope = assert(parent_scope:child()) + local bus = busmod.new() + local conn = bus:connect({ origin_base = { service = 'test-cm5' } }) + + -- The test UART adapter registers finalisers on the instance scope. + -- Do this before starting services that spawn onto the same scope; after a + -- scope has started, lua-fibers requires finalisers to be added from within + -- that target scope. + if uart_port ~= nil then + start_cm5_uart_manager(scope, bus, uart_port) + end + start_hal(scope, bus, roots) + start_http(scope, bus) + + return { + scope = scope, + bus = bus, + conn = conn, + start_services_after_fabric = function (fabric_client) + start_device(scope, bus, fabric_client, opts.device or opts) + start_update(scope, bus, opts.update or opts) + if http_port then start_ui(scope, bus, http_port, roots) end + end, + } +end + +local function update_fake_from_mcu_child(fake, ev) + if type(ev) ~= 'table' then return end + local event = ev.event + if event == 'ready' then + fake.boot_seq = ev.boot_seq or fake.boot_seq + fake.child_ready = true + fake.child_pid = ev.pid + log(('MCU child ready: pid=%s boot_seq=%s image=%s'):format( + tostring(ev.pid), tostring(ev.boot_seq), tostring(ev.image_id) + )) + elseif event == 'receive_opened' then + fake.transfer_begin = ev + fake.receive_started_at = fibers.now() + fake.receive_bytes = 0 + fake.receive_chunks = 0 + log(('MCU receive target opened: target=%s size=%s job_id=%s image_id=%s'):format( + tostring(ev.target), tostring(ev.size), tostring(ev.job_id), tostring(ev.image_id) + )) + elseif event == 'receive_tick' then + fake.receive_bytes = ev.bytes or fake.receive_bytes + fake.receive_chunks = ev.chunks or fake.receive_chunks + elseif event == 'receive_progress' then + fake.receive_bytes = ev.bytes or fake.receive_bytes + fake.receive_chunks = ev.chunks or fake.receive_chunks + log(('MCU received %d bytes in %d chunks after %.1fs'):format( + fake.receive_bytes or 0, fake.receive_chunks or 0, tonumber(ev.elapsed_s) or 0 + )) + elseif event == 'transfer_commit' then + fake.receive_bytes = ev.size or fake.receive_bytes + fake.receive_chunks = ev.chunks or fake.receive_chunks + fake.staged = { + size = ev.size, + digest = ev.digest, + payload_digest = ev.payload_digest, + image_id = ev.image_id, + job_id = ev.job_id, + } + fake.staged_signal = (fake.staged_signal or 0) + 1 + log(('MCU transfer commit: %d bytes in %d chunks; digest=%s payload_digest=%s'):format( + ev.size or 0, ev.chunks or 0, tostring(ev.digest), tostring(ev.payload_digest) + )) + elseif event == 'transfer_abort' then + fake.abort_reason = ev.reason + fake.abort_count = ev.abort_count or ((fake.abort_count or 0) + 1) + fake.receive_bytes = ev.bytes or fake.receive_bytes + fake.receive_chunks = ev.chunks or fake.receive_chunks + log(('MCU transfer abort: reason=%s after %d bytes in %d chunks'):format( + tostring(ev.reason), fake.receive_bytes or 0, fake.receive_chunks or 0 + )) + elseif event == 'prepare' then + fake.prepare_payload = ev.payload + fake.job_id = ev.job_id + elseif event == 'commit' then + fake.commit_payload = ev.payload + fake.commit_seen = true + fake.committed_image_id = ev.committed_image_id + elseif event == 'rebooting' then + fake.commit_seen = true + fake.rebooting_seen = true + fake.committed_image_id = ev.image_id or fake.committed_image_id + log(('MCU child rebooting: image=%s version=%s length=%s'):format( + tostring(ev.image_id), tostring(ev.version), tostring(ev.length) + )) + elseif event == 'fabric_session' then + fake.mcu_fabric_session = ev + fake.mcu_fabric_session_seen = true + log(('MCU child fabric session: phase=%s established=%s peer_node=%s peer_sid=%s gen=%s wire_errors=%s bad_frames=%s'):format( + tostring(ev.phase), tostring(ev.established), tostring(ev.peer_node), tostring(ev.peer_sid), + tostring(ev.session_generation), tostring(ev.wire_errors or 0), tostring(ev.bad_frame_count or 0) + )) + elseif event == 'fabric_status' then + log(('MCU child fabric %s: %s'):format(tostring(ev.component or ev.label or 'status'), tostring(ev.summary))) + elseif event == 'log' then + log('MCU child: ' .. tostring(ev.message)) + end +end + +local function start_mcu_instance(parent_scope, fake, uart_port) + assert(uart_port and uart_port.slave_name, 'MCU PTY slave required') + fake.mcu_fabric_session_seen = false + fake.mcu_fabric_session = nil + local scope = assert(parent_scope:child()) + local ipc_path = ('/tmp/devicecode-mcu-child-%d-%d.sock'):format(os.time(), math.random(100000, 999999)) + os.remove(ipc_path) + + local server, serr = socket.listen_unix(ipc_path, { ephemeral = true }) + assert_not_nil(server, serr or 'listen_unix failed') + local ready_ch = channel.new(1) + + scope:finally(function () + safe.pcall(function () server:close() end) + os.remove(ipc_path) + end) + + assert_true(scope:spawn(function () + local stream, aerr = server:accept() + if not stream then + ready_ch:put({ ok = false, err = aerr or 'mcu child ipc accept failed' }) + return + end + while true do + local line, rerr = fibers.perform(stream:read_line_op()) + if line == nil then + if fake.child_ready ~= true then + ready_ch:put({ ok = false, err = rerr or 'mcu child ipc closed before ready' }) + end + return + end + local ev, derr = cjson.decode(line) + if not ev then + log('MCU child sent invalid IPC JSON: ' .. tostring(derr)) + else + update_fake_from_mcu_child(fake, ev) + if ev.event == 'ready' then + ready_ch:put({ ok = true }) + end + end + end + end)) + + local child_script = './integration/devhost/support/mcu_http_uart_child.lua' + local cmd = exec.command({ + 'lua', child_script, + '--ipc', ipc_path, + '--uart', uart_port.slave_name, + '--old-image', fake.old_image_id or 'mcu-image-old', + '--committed-image', fake.committed_image_id or '', + '--boot-seq', tostring((fake.boot_seq or 0) + 1), + cwd = '.', + stdin = 'null', + stdout = 'inherit', + stderr = 'inherit', + shutdown_grace = 1.0, + env = { + MCU_HTTP_UART_TRANSFER_TIMEOUT_S = tostring(REALISTIC_TRANSFER_TIMEOUT_S), + MCU_HTTP_UART_FLASH_DELAY_S = tostring(MCU_FLASH_WRITE_DELAY_S), + }, + }) + + assert_true(scope:spawn(function () + local status, code, signal, err = fibers.perform(cmd:run_op()) + fake.child_exit = { status = status, code = code, signal = signal, err = err } + if status ~= 'signalled' and not (status == 'exited' and code == 0) then + log(('MCU child exited: status=%s code=%s signal=%s err=%s'):format( + tostring(status), tostring(code), tostring(signal), tostring(err) + )) + end + end)) + + local ready = wait_channel_get(ready_ch, 6.0, 'MCU child ready') + if not ready.ok then error(ready.err or 'MCU child failed to start', 0) end + + return { scope = scope, ipc_path = ipc_path, command = cmd } +end + + +local GO_DEVHOST_DEFAULT_REPO = 'https://github.com/jangala-dev/devicecode-go.git' +local GO_DEVHOST_DEFAULT_REF = 'rt-fixes' +local GO_DEVHOST_DEFAULT_CACHE = '/tmp/devicecode-go-devhost-cache' +local PICO2_AB_DEFAULT_REPO = 'https://github.com/jangala-dev/pico2-a-b.git' +local PICO2_AB_DEFAULT_REF = 'fabric' + +local function go_devhost_download_enabled() + local v = env_non_empty('DEVICECODE_GO_DOWNLOAD') + return v == nil or v == '1' or v == 'true' or v == 'yes' +end + +local function go_devhost_configured() + return env_non_empty('MCU_DEVHOST_PTY_BIN') ~= nil + or env_non_empty('DEVICECODE_GO_MCU_DEVHOST_PTY_BIN') ~= nil + or env_non_empty('DEVICECODE_GO_ROOT') ~= nil + or go_devhost_download_enabled() +end + +local function safe_path_token(s) + s = tostring(s or '') + s = s:gsub('[^%w_.-]', '_') + if s == '' then s = 'default' end + return s +end + +local function go_devhost_checkout_dir() + local cache = env_non_empty('DEVICECODE_GO_CACHE') or GO_DEVHOST_DEFAULT_CACHE + local ref = env_non_empty('DEVICECODE_GO_REF') or GO_DEVHOST_DEFAULT_REF + return cache .. '/devicecode-go-' .. safe_path_token(ref) +end + +local function pico2_ab_checkout_dir() + local cache = env_non_empty('DEVICECODE_GO_CACHE') or GO_DEVHOST_DEFAULT_CACHE + return cache .. '/pico2-a-b' +end + +local function go_devhost_binary_path() + local cache = env_non_empty('DEVICECODE_GO_CACHE') or GO_DEVHOST_DEFAULT_CACHE + local ref = env_non_empty('DEVICECODE_GO_REF') or GO_DEVHOST_DEFAULT_REF + return cache .. '/bin/mcu-devhost-pty-' .. safe_path_token(ref) +end + +local function ensure_git_checkout(label, repo, ref, dir) + local script = table.concat({ + 'set -eu', + 'mkdir -p ' .. shquote(dir), + 'if [ ! -d ' .. shquote(dir .. '/.git') .. ' ]; then', + ' rm -rf ' .. shquote(dir), + ' mkdir -p ' .. shquote(dir), + ' git -C ' .. shquote(dir) .. ' init -q', + ' git -C ' .. shquote(dir) .. ' remote add origin ' .. shquote(repo), + 'fi', + 'git -C ' .. shquote(dir) .. ' fetch --depth 1 origin ' .. shquote(ref), + 'git -C ' .. shquote(dir) .. ' checkout -q --detach FETCH_HEAD', + 'git -C ' .. shquote(dir) .. ' reset -q --hard FETCH_HEAD', + 'git -C ' .. shquote(dir) .. ' clean -q -fdx', + }, '\n') + + log(('go-pty: ensuring %s checkout repo=%s ref=%s dir=%s'):format(label, repo, ref, dir)) + local cmd = exec.command('sh', '-c', script) + local out, st, code, sig, err = fibers.perform(cmd:combined_output_op()) + if not (st == 'exited' and code == 0) then + return nil, ('%s checkout failed: status=%s code=%s signal=%s err=%s output=%s'):format( + label, tostring(st), tostring(code), tostring(sig), tostring(err), tostring(out) + ) + end + return dir, nil +end + +local function build_go_devhost_binary(go_dir) + local bin = go_devhost_binary_path() + local script = table.concat({ + 'set -eu', + 'mkdir -p ' .. shquote((bin:gsub('/[^/]+$', ''))), + 'cd ' .. shquote(go_dir), + 'GOMAXPROCS=1 go build -o ' .. shquote(bin) .. ' ./cmd/mcu-devhost-pty', + }, '\n') + log(('go-pty: building Go MCU devhost binary path=%s'):format(bin)) + local cmd = exec.command('sh', '-c', script) + local out, st, code, sig, err = fibers.perform(cmd:combined_output_op()) + if not (st == 'exited' and code == 0) then + return nil, ('Go devhost build failed: status=%s code=%s signal=%s err=%s output=%s'):format( + tostring(st), tostring(code), tostring(sig), tostring(err), tostring(out) + ) + end + return bin, nil +end + +local function ensure_go_devhost_checkout() + local root = env_non_empty('DEVICECODE_GO_ROOT') + if root ~= nil then + local bin, berr = build_go_devhost_binary(root) + if not bin then return nil, berr end + return root, bin, nil + end + if not go_devhost_download_enabled() then + return nil, nil, 'set MCU_DEVHOST_PTY_BIN, DEVICECODE_GO_ROOT, or DEVICECODE_GO_DOWNLOAD=1 to run the Go devhost MCU PTY rig' + end + + local go_repo = env_non_empty('DEVICECODE_GO_REPO') or GO_DEVHOST_DEFAULT_REPO + local go_ref = env_non_empty('DEVICECODE_GO_REF') or GO_DEVHOST_DEFAULT_REF + local go_dir = go_devhost_checkout_dir() + + local pico_repo = env_non_empty('DEVICECODE_PICO2_AB_REPO') or PICO2_AB_DEFAULT_REPO + local pico_ref = env_non_empty('DEVICECODE_PICO2_AB_REF') or PICO2_AB_DEFAULT_REF + local pico_dir = pico2_ab_checkout_dir() + + local dir, err = ensure_git_checkout('Go devhost', go_repo, go_ref, go_dir) + if not dir then return nil, nil, err end + + -- devicecode-go's go.mod uses a local replacement for pico2-a-b at + -- ../pico2-a-b. Keep that sibling checkout in the same cache root so + -- the Lua repo does not need direct access to a monorepo workspace. + local pdir, perr = ensure_git_checkout('pico2-a-b', pico_repo, pico_ref, pico_dir) + if not pdir then return nil, nil, perr end + + local bin, berr = build_go_devhost_binary(go_dir) + if not bin then return nil, nil, berr end + + return go_dir, bin, nil +end + + +local function resolve_go_devhost_provider() + local bin = env_non_empty('MCU_DEVHOST_PTY_BIN') or env_non_empty('DEVICECODE_GO_MCU_DEVHOST_PTY_BIN') + if bin ~= nil then + return { bin = bin }, nil + end + if not command_available('go') then + return nil, 'go is not installed' + end + local root = env_non_empty('DEVICECODE_GO_ROOT') + if root ~= nil then + local built_bin, berr = build_go_devhost_binary(root) + if not built_bin then return nil, berr end + return { bin = built_bin, cwd = root }, nil + end + if not go_devhost_download_enabled() then + return nil, 'set MCU_DEVHOST_PTY_BIN, DEVICECODE_GO_ROOT, or DEVICECODE_GO_DOWNLOAD=1 to run the Go devhost MCU PTY rig' + end + if not command_available('git') then + return nil, 'git is not installed' + end + local go_dir, built_bin, err = ensure_go_devhost_checkout() + if not go_dir or not built_bin then + return nil, err or 'Go devhost checkout/build unavailable' + end + return { bin = built_bin, cwd = go_dir }, nil +end + +local function go_devhost_command_spec(uart_slave, state_dir, fake, opts) + opts = opts or {} + local provider = opts.provider + local argv + local cwd + if provider ~= nil then + argv = { provider.bin } + cwd = provider.cwd + else + local bin = env_non_empty('MCU_DEVHOST_PTY_BIN') or env_non_empty('DEVICECODE_GO_MCU_DEVHOST_PTY_BIN') + local root + local root_err + if bin ~= nil then + argv = { bin } + else + local built_bin + root, built_bin, root_err = ensure_go_devhost_checkout() + if root == nil or built_bin == nil then return nil, root_err end + argv = { built_bin } + cwd = root + end + end + argv[#argv + 1] = '--uart'; argv[#argv + 1] = uart_slave + argv[#argv + 1] = '--state-dir'; argv[#argv + 1] = state_dir + argv[#argv + 1] = '--node'; argv[#argv + 1] = opts.node or 'mcu' + -- The devhost CM5 Fabric config in this test uses local_node='cm5'. + -- Keep the Go peer expectation aligned with that test-local node name. + argv[#argv + 1] = '--peer'; argv[#argv + 1] = opts.peer or 'cm5' + argv[#argv + 1] = '--initial-image-id'; argv[#argv + 1] = fake.old_image_id or 'mcu-image-old' + argv[#argv + 1] = '--initial-version'; argv[#argv + 1] = fake.old_version or '10.0' + argv[#argv + 1] = '--initial-build-id'; argv[#argv + 1] = fake.old_build_id or 'devhost-initial' + argv[#argv + 1] = '--reboot-exit-code'; argv[#argv + 1] = tostring(opts.reboot_exit_code or 42) + argv.cwd = cwd + argv.stdin = 'null' + argv.stdout = 'pipe' + argv.stderr = 'inherit' + argv.shutdown_grace = 1.0 + argv.env = { GOMAXPROCS = tostring(opts.gomaxprocs or 1) } + return argv, nil +end + +local function start_go_mcu_instance(parent_scope, fake, uart_port, state_dir, opts) + assert(uart_port and uart_port.slave_name, 'MCU PTY slave required') + assert(state_dir and state_dir ~= '', 'Go MCU state dir required') + fake.expect_mcu_fabric_event = false + fake.mcu_fabric_session_seen = false + fake.mcu_fabric_session = nil + local scope = assert(parent_scope:child()) + mkdir_p(state_dir) + + local spec, spec_err = go_devhost_command_spec(uart_port.slave_name, state_dir, fake, opts) + assert_not_nil(spec, spec_err) + local cmd = exec.command(spec) + local stdout, sout_err = cmd:stdout_stream() + assert_not_nil(stdout, sout_err or 'Go MCU child stdout pipe unavailable') + + local ready_ch = channel.new(1) + + assert_true(scope:spawn(function () + while true do + local line, rerr = fibers.perform(stdout:read_line_op()) + if line == nil then + if fake.child_ready ~= true then + ready_ch:put({ ok = false, err = rerr or 'go mcu child stdout closed before ready' }) + end + return + end + local ev, derr = cjson.decode(line) + if not ev then + log('Go MCU child sent invalid JSON: ' .. tostring(derr) .. ' line=' .. tostring(line)) + else + update_fake_from_mcu_child(fake, ev) + if ev.event == 'ready' then + ready_ch:put({ ok = true }) + end + end + end + end)) + + assert_true(scope:spawn(function () + local status, code, signal, err = fibers.perform(cmd:run_op()) + fake.child_exit = { status = status, code = code, signal = signal, err = err } + if status == 'exited' and code == (opts and opts.reboot_exit_code or 42) then + fake.reboot_exit_seen = true + log(('Go MCU child exited for simulated reboot: code=%s'):format(tostring(code))) + elseif status ~= 'signalled' and not (status == 'exited' and code == 0) then + log(('Go MCU child exited: status=%s code=%s signal=%s err=%s'):format( + tostring(status), tostring(code), tostring(signal), tostring(err) + )) + end + end)) + + local ready = wait_channel_get(ready_ch, 20.0, 'Go MCU child ready') + if not ready.ok then error(ready.err or 'Go MCU child failed to start', 0) end + + return { scope = scope, command = cmd, state_dir = state_dir } +end + +local function stop_instance(inst, reason) + if inst and inst.scope then + inst.scope:cancel(reason or 'test reboot') + fibers.perform(inst.scope:join_op()) + end +end + +local function parse_curl_output(out) + out = tostring(out or '') + local status = out:match('\n__HTTP_STATUS__:(%d+)%s*$') + local body = out:gsub('\n__HTTP_STATUS__:%d+%s*$', '') + return status, body +end + +local function run_curl(args) + local cmd = exec.command('curl', unpack(args)) + local out, st, code, sig, err = fibers.perform(cmd:combined_output_op()) + if not (st == 'exited' and code == 0) then + error(('curl failed: status=%s code=%s signal=%s err=%s output=%s'):format( + tostring(st), tostring(code), tostring(sig), tostring(err), tostring(out) + ), 0) + end + return parse_curl_output(out) +end + +local function append_headers(args, headers) + for k, v in pairs(headers or {}) do + args[#args + 1] = '--header' + args[#args + 1] = tostring(k) .. ': ' .. tostring(v) + end +end + +local function run_http_upload(_scope, port, body, headers) + local path = ('/tmp/devicecode-mcu-http-uart-upload-%d-%d.bin'):format(os.time(), math.random(100000, 999999)) + write_file(path, body) + local args = { + '--silent', '--show-error', + '--request', 'POST', + '--header', 'content-type: application/octet-stream', + '--data-binary', '@' .. path, + '--write-out', '\n__HTTP_STATUS__:%{http_code}\n', + } + append_headers(args, headers) + args[#args + 1] = ('http://127.0.0.1:%d/api/update/upload'):format(port) + local status, resp_body = run_curl(args) + os.remove(path) + return status, resp_body +end + +local function run_http_json(_scope, port, path, payload, headers) + local args = { + '--silent', '--show-error', + '--request', 'POST', + '--header', 'content-type: application/json', + '--data', assert(cjson.encode(payload or {})), + '--write-out', '\n__HTTP_STATUS__:%{http_code}\n', + } + append_headers(args, headers) + args[#args + 1] = ('http://127.0.0.1:%d%s'):format(port, path) + local status, resp_body = run_curl(args) + return status, resp_body, resp_body and cjson.decode(resp_body) or nil +end + + +function T.ui_http_mcu_update_short_stage_timeout_fails_via_outer_choice() + runfibers.run(function (root_scope) + local roots = temp_roots() + local blob = make_realistic_mcu_blob('mcu-image-new') + local port = 30000 + math.random(0, 20000) + local fake = { old_image_id = 'mcu-image-old' } + local uart = { cm5 = pty.open(root_scope), mcu = pty.open(root_scope) } + + log('short-timeout: booting CM5 instance') + local cm5 = start_cm5_instance(root_scope, roots, port, uart.cm5, { + stage_timeout_s = SHORT_STAGE_TIMEOUT_S, + stage_action_timeout_s = REALISTIC_TRANSFER_TIMEOUT_S, + }) + log('short-timeout: booting fake MCU instance') + local mcu = start_mcu_instance(root_scope, fake, uart.mcu) + log('short-timeout: starting Fabric pair over PTY UART middleware') + local pair = start_fabric_pair(root_scope, cm5.bus, fake, uart) + log('short-timeout: starting CM5 Device/Update/UI services') + cm5.start_services_after_fabric() + + wait_component_software(cm5.conn, 'mcu-image-old', 'mcu-boot-1') + + log(('short-timeout: sending unauthenticated upload through curl with %.1fs update stage deadline'):format(SHORT_STAGE_TIMEOUT_S)) + local status, body = run_http_upload(root_scope, port, blob, nil) + assert_eq(status, '200', 'upload HTTP status ' .. tostring(status) .. ': ' .. tostring(body)) + + log('short-timeout: waiting for job to fail due to the outer stage deadline') + local failed = wait_job_chatty(cm5.conn, 'job-mcu-http-uart', 'failed', 20.0, function () + return (('mcu_received=%d chunks=%d uart_bytes=%d uart_fragments=%d uart_pauses=%d uart_bad_json=%d; %s'):format( + fake.receive_bytes or 0, + fake.receive_chunks or 0, + pair.uart_stats and pair.uart_stats.bytes or 0, + pair.uart_stats and pair.uart_stats.fragments or 0, + pair.uart_stats and pair.uart_stats.pauses or 0, + pair.uart_stats and pair.uart_stats.malformed_lines or 0, + fabric_progress_fragment(pair.cm5_conn) + )) + end) + assert_contains(failed.error, 'stage_op_timeout', 'job should fail from the active stage deadline') + assert_true((fake.receive_bytes or 0) < #blob, 'short timeout should not stage the full artifact') + assert_true(fake.staged == nil, 'short timeout must not commit a staged artifact') + assert_true(fake.commit_seen ~= true, 'short timeout must not reach the MCU commit RPC') + fibers.perform(sleep.sleep_op(0.25)) + log(('short-timeout: cancellation observed abort_reason=%s abort_count=%d received=%d chunks=%d'):format( + tostring(fake.abort_reason), fake.abort_count or 0, fake.receive_bytes or 0, fake.receive_chunks or 0 + )) + + log('short-timeout: cleanup') + stop_fabric_pair(pair, 'short timeout complete') + stop_instance(cm5, 'short timeout complete') + stop_instance(mcu, 'short timeout complete') + rm_rf(roots.base) + end, { timeout = 60.0 }) +end + +function T.ui_http_mcu_update_survives_fake_reboot_and_reconciles() + runfibers.run(function (root_scope) + local roots = temp_roots() + local blob = make_realistic_mcu_blob('mcu-image-new') + local port = 30000 + math.random(0, 20000) + local fake = { old_image_id = 'mcu-image-old' } + + local uart = { cm5 = pty.open(root_scope), mcu = pty.open(root_scope) } + + log('booting initial CM5 instance') + local cm5 = start_cm5_instance(root_scope, roots, port, uart.cm5) + log('booting initial fake MCU instance') + local mcu = start_mcu_instance(root_scope, fake, uart.mcu) + log('starting initial Fabric pair over PTY UART middleware') + local pair = start_fabric_pair(root_scope, cm5.bus, fake, uart) + log('starting CM5 Device/Update/UI services') + cm5.start_services_after_fabric() + + log('waiting for initial canonical MCU software state') + wait_component_software(cm5.conn, 'mcu-image-old', 'mcu-boot-1') + + log(('sending real HTTP upload through curl (%d byte artifact)'):format(#blob)) + local stage_started = fibers.now() + local status, body = run_http_upload(root_scope, port, blob, nil) + assert_eq(status, '200', 'upload HTTP status ' .. tostring(status) .. ': ' .. tostring(body)) + local decoded = assert(cjson.decode(body), body) + assert_eq(decoded.status, 'ok') + assert_eq(decoded.job_id, 'job-mcu-http-uart') + assert_eq(decoded.job, nil) + + log('waiting for job awaiting_commit') + wait_job_chatty(cm5.conn, 'job-mcu-http-uart', 'awaiting_commit', REALISTIC_TRANSFER_TIMEOUT_S, function () + return (('mcu_received=%d chunks=%d uart_bytes=%d uart_fragments=%d uart_pauses=%d uart_bad_json=%d; %s'):format( + fake.receive_bytes or 0, + fake.receive_chunks or 0, + pair.uart_stats and pair.uart_stats.bytes or 0, + pair.uart_stats and pair.uart_stats.fragments or 0, + pair.uart_stats and pair.uart_stats.pauses or 0, + pair.uart_stats and pair.uart_stats.malformed_lines or 0, + fabric_progress_fragment(pair.cm5_conn) + )) + end) + local expected_payload_digest = xxhash32.digest_hex(blob) + assert_true(probe.wait_until(function () + return fake.staged + and fake.staged.size == #blob + and fake.staged.payload_digest == expected_payload_digest + end, { timeout = 10.0 }), 'fake MCU should stage transferred artifact with matching digest') + local stage_elapsed = fibers.now() - stage_started + local uart_bytes = pair.uart_stats and pair.uart_stats.bytes or 0 + local uart_fragments = pair.uart_stats and pair.uart_stats.fragments or 0 + local uart_pauses = pair.uart_stats and pair.uart_stats.pauses or 0 + local uart_malformed_lines = pair.uart_stats and pair.uart_stats.malformed_lines or 0 + log(('artifact staged in %.1fs; middleware relayed %d bytes in %d fragments with %d long pauses and %d malformed JSONL lines'):format( + stage_elapsed, uart_bytes, uart_fragments, uart_pauses, uart_malformed_lines + )) + assert_true(uart_bytes >= #blob, 'UART middleware should relay at least the artifact payload size') + assert_true(uart_malformed_lines >= 1, 'UART middleware should inject at least one standalone malformed JSONL line') + assert_true( + uart_fragments >= math.floor(#blob / UART_MAX_FRAGMENT_BYTES), + 'UART middleware should fragment the byte stream heavily' + ) + if #blob >= (2 * UART_LONG_PAUSE_EVERY_BYTES) then + assert_true(uart_pauses >= 2, 'large UART test should inject at least two long scheduling pauses') + end + assert_true( + stage_elapsed >= ((#blob / UART_BYTES_PER_SEC) * 0.75), + 'artifact should take UART-paced time to stage' + ) + log_fabric_status(cm5.conn, 'CM5 fabric post-stage status') + + log('committing job through curl HTTP update commit route') + local commit_status, commit_body, commit_decoded = run_http_json( + root_scope, + port, + '/api/update/commit', + { job_id = 'job-mcu-http-uart' }, + nil + ) + assert_eq(commit_status, '200', commit_body) + assert_not_nil(commit_decoded and commit_decoded.value, commit_body) + assert_eq(commit_decoded.value.ok, true) + log('waiting for job awaiting_return') + wait_job(cm5.conn, 'job-mcu-http-uart', 'awaiting_return', 6.0) + assert_true(probe.wait_until(function () return fake.commit_seen == true end, { timeout = 2.0 }), 'fake MCU should see commit') + assert_eq(fake.committed_image_id, 'mcu-image-new') + + log('fake reboot: stopping Fabric pair') + stop_fabric_pair(pair, 'fake fabric link reboot') + log('fake reboot: stopping CM5 instance') + stop_instance(cm5, 'fake cm5 reboot') + log('fake reboot: stopping MCU instance') + stop_instance(mcu, 'fake mcu reboot') + + local uart_b = { cm5 = pty.open(root_scope), mcu = pty.open(root_scope) } + log('reboot: starting fresh CM5 instance') + local cm5b = start_cm5_instance(root_scope, roots, nil, uart_b.cm5) + log('reboot: starting fresh MCU instance') + local mcub = start_mcu_instance(root_scope, fake, uart_b.mcu) + log('reboot: starting fresh Fabric pair over PTY UART middleware') + local pair_b = start_fabric_pair(root_scope, cm5b.bus, fake, uart_b) + log_fabric_status(cm5b.conn, 'CM5 fabric reboot-established status') + log('reboot: starting fresh CM5 Device/Update services') + cm5b.start_services_after_fabric() + + log('reboot: waiting for post-boot canonical MCU software state') + wait_component_software(cm5b.conn, 'mcu-image-new', 'mcu-boot-2') + log('reboot: waiting for job succeeded') + local final_job = wait_job(cm5b.conn, 'job-mcu-http-uart', 'succeeded', 8.0) + assert_eq(final_job.component, 'mcu') + assert_eq(final_job.job_id, 'job-mcu-http-uart') + assert_not_nil(final_job.commit_attempt, 'job should carry commit attempt details') + if final_job.commit_attempt and final_job.commit_attempt.pre_commit then + assert_eq(final_job.commit_attempt.pre_commit.pre_commit_boot_id, 'mcu-boot-1') + end + + log('cleanup: stopping second Fabric pair') + stop_fabric_pair(pair_b, 'test complete') + log('cleanup: stopping second CM5 instance') + stop_instance(cm5b, 'test complete') + log('cleanup: stopping second MCU instance') + stop_instance(mcub, 'test complete') + log('cleanup: removing temporary roots') + rm_rf(roots.base) + end, { timeout = REALISTIC_TEST_TIMEOUT_S }) +end + + +local function run_go_devhost_pty_cycle(opts) + opts = opts or {} + if not go_devhost_configured() then + log('skipping Go devhost PTY rig: set MCU_DEVHOST_PTY_BIN, DEVICECODE_GO_ROOT, or leave DEVICECODE_GO_DOWNLOAD enabled') + return { skipped = true, reason = 'go_devhost_not_configured' } + end + local result + runfibers.run(function (root_scope) + local provider, skip_reason = resolve_go_devhost_provider() + if provider == nil then + log('skipping Go devhost PTY rig: ' .. tostring(skip_reason)) + result = { skipped = true, reason = skip_reason } + return + end + local roots = temp_roots() + local state_dir = roots.base .. '/go-mcu-state' + local blob = make_realistic_mcu_blob(opts.image_id or 'mcu-image-go-new', opts.blob_bytes) + local port = 30000 + math.random(0, 20000) + local fake = { old_image_id = opts.old_image_id or 'mcu-image-go-old', old_version = opts.old_version or '10.0' } + local uart = { cm5 = pty.open(root_scope), mcu = pty.open(root_scope) } + local pair, pair_b + + log(('go-pty[%s]: booting initial CM5 instance'):format(opts.label or 'case')) + local cm5 = start_cm5_instance(root_scope, roots, port, uart.cm5) + log(('go-pty[%s]: booting Go MCU devhost instance'):format(opts.label or 'case')) + local mcu = start_go_mcu_instance(root_scope, fake, uart.mcu, state_dir, { provider = provider }) + log(('go-pty[%s]: starting Fabric link over PTY UART middleware'):format(opts.label or 'case')) + pair = start_fabric_pair(root_scope, cm5.bus, fake, uart, { uart = opts.uart }) + log(('go-pty[%s]: starting CM5 Device/Update/UI services'):format(opts.label or 'case')) + cm5.start_services_after_fabric() + + wait_component_software(cm5.conn, fake.old_image_id, nil) + + log(('go-pty[%s]: sending HTTP upload through CM5 UI (%d byte artifact)'):format(opts.label or 'case', #blob)) + local status, body = run_http_upload(root_scope, port, blob, nil) + assert_eq(status, '200', 'upload HTTP status ' .. tostring(status) .. ': ' .. tostring(body)) + local decoded = assert(cjson.decode(body), body) + assert_eq(decoded.status, 'ok') + assert_eq(decoded.job_id, 'job-mcu-http-uart') + assert_eq(decoded.job, nil) + + log(('go-pty[%s]: waiting for Go MCU-backed job awaiting_commit'):format(opts.label or 'case')) + wait_job_chatty(cm5.conn, 'job-mcu-http-uart', 'awaiting_commit', opts.stage_timeout_s or REALISTIC_TRANSFER_TIMEOUT_S, function () + return (('uart_bytes=%d uart_fragments=%d uart_pauses=%d uart_bad_json=%d uart_drop_bytes=%d uart_drop_nl=%d fault_pauses=%d; %s'):format( + pair.uart_stats and pair.uart_stats.bytes or 0, + pair.uart_stats and pair.uart_stats.fragments or 0, + pair.uart_stats and pair.uart_stats.pauses or 0, + pair.uart_stats and pair.uart_stats.malformed_lines or 0, + pair.uart_stats and pair.uart_stats.dropped_bytes or 0, + pair.uart_stats and pair.uart_stats.dropped_newlines or 0, + pair.uart_stats and pair.uart_stats.fault_pauses or 0, + fabric_progress_fragment(pair.cm5_conn) + )) + end) + + log(('go-pty[%s]: committing job through public HTTP update commit route'):format(opts.label or 'case')) + local commit_status, commit_body, commit_decoded = run_http_json( + root_scope, + port, + '/api/update/commit', + { job_id = 'job-mcu-http-uart' }, + nil + ) + assert_eq(commit_status, '200', commit_body) + assert_not_nil(commit_decoded and commit_decoded.value, commit_body) + assert_eq(commit_decoded.value.ok, true) + wait_job(cm5.conn, 'job-mcu-http-uart', 'awaiting_return', 8.0) + assert_true(probe.wait_until(function () return fake.rebooting_seen == true end, { timeout = 4.0 }), 'Go MCU child should emit rebooting event') + assert_true(probe.wait_until(function () return fake.reboot_exit_seen == true end, { timeout = 4.0 }), 'Go MCU child should exit with simulated reboot code') + + stop_fabric_pair(pair, 'go devhost fabric link reboot') + stop_instance(cm5, 'go devhost cm5 reboot') + stop_instance(mcu, 'go devhost mcu reboot') + + local uart_b = { cm5 = pty.open(root_scope), mcu = pty.open(root_scope) } + local cm5b = start_cm5_instance(root_scope, roots, nil, uart_b.cm5) + local mcub = start_go_mcu_instance(root_scope, fake, uart_b.mcu, state_dir, { provider = provider }) + pair_b = start_fabric_pair(root_scope, cm5b.bus, fake, uart_b, { uart = opts.reboot_uart }) + log_fabric_status(cm5b.conn, 'go-pty CM5 fabric reboot-established status') + cm5b.start_services_after_fabric() + + wait_component_software(cm5b.conn, opts.image_id or 'mcu-image-go-new', nil) + local final_job = wait_job(cm5b.conn, 'job-mcu-http-uart', 'succeeded', 10.0) + assert_eq(final_job.component, 'mcu') + assert_eq(final_job.job_id, 'job-mcu-http-uart') + assert_eq(final_job.expected_image_id, opts.image_id or 'mcu-image-go-new') + + result = { + ok = true, + final_job = final_job, + first_uart_stats = pair and pair.uart_stats or nil, + second_uart_stats = pair_b and pair_b.uart_stats or nil, + } + + stop_fabric_pair(pair_b, 'go devhost test complete') + stop_instance(cm5b, 'go devhost test complete') + stop_instance(mcub, 'go devhost test complete') + rm_rf(roots.base) + end, { timeout = opts.timeout_s or REALISTIC_TEST_TIMEOUT_S }) + return result +end + +function T.ui_http_mcu_update_go_devhost_pty_reboots_and_reconciles() + run_go_devhost_pty_cycle({ label = 'baseline' }) +end + + +T.__helpers = { + log = log, + run_go_devhost_pty_cycle = run_go_devhost_pty_cycle, + go_devhost_configured = go_devhost_configured, +} + +return T diff --git a/tests/integration/devhost/metrics_spec.lua b/tests/integration/devhost/metrics_spec.lua new file mode 100644 index 00000000..669b97ec --- /dev/null +++ b/tests/integration/devhost/metrics_spec.lua @@ -0,0 +1,528 @@ +-- tests/integration/metrics/service_spec.lua +-- +-- Service-level integration tests for the metrics service. +-- Each test spins up the full metrics service in a child fiber scope, +-- interacts with it via bus messages, and asserts on the output. +-- +-- Virtual time is installed inside each runfibers.run callback so that the +-- real-clock timeout guard in run_fibers.lua still functions correctly. + +local fibers = require 'fibers' +local perform = fibers.perform +local op = require 'fibers.op' +local busmod = require 'bus' +local json = require 'cjson.safe' +local virtual_time = require 'tests.support.virtual_time' +local time_harness = require 'tests.support.time_harness' +local runfibers = require 'tests.support.run_fibers' + +------------------------------------------------------------------------------- +-- Module-level constants +------------------------------------------------------------------------------- + +-- Sample mainflux.cfg the mock HAL returns for every read request. +local MAINFLUX_CFG = json.encode({ + thing_key = 'test-thing-key', + channels = { + { id = 'ch-data', name = 'test_data', metadata = { channel_type = 'data' } }, + { id = 'ch-control', name = 'test_control', metadata = { channel_type = 'control' } }, + }, +}) + +------------------------------------------------------------------------------- +-- Helpers (shared across all tests in this file) +------------------------------------------------------------------------------- + +local function make_bus() + return busmod.new({ q_length = 100, s_wild = '+', m_wild = '#' }) +end + +local function new_test_clock() + return virtual_time.install({ monotonic = 0, realtime = 1700000000 }) +end + +local function flush_ticks(max_ticks) + time_harness.flush_ticks(max_ticks or 20) +end + +-- Subscribe to obs/v1/metrics/output/# and wait for the next message, +-- advancing the virtual clock in steps until one arrives or timeout lapses. +local function recv_metric(clock, sub, timeout_s) + local step = 0.05 + local max_ticks = 20 + local elapsed = 0 + while true do + local ok, msg = time_harness.try_op_now(function() return sub:recv_op() end) + if ok then return msg end + if elapsed >= timeout_s then return nil end + local advance = math.min(step, timeout_s - elapsed) + clock:advance(advance) + time_harness.flush_ticks(max_ticks) + elapsed = elapsed + advance + end +end + +-- Drain all currently-available messages without advancing time. +local function drain_non_status(sub, max_ticks) + max_ticks = max_ticks or 20 + local messages = {} + for tick = 1, max_ticks do + local saw_any = false + while true do + local ok, msg = time_harness.try_op_now(function() return sub:recv_op() end) + if not ok then break end + saw_any = true + messages[#messages + 1] = msg + end + if not saw_any or tick == max_ticks then break end + time_harness.flush_ticks(1) + end + return messages +end + +-- Bind cap/fs/configs/rpc/read and respond with MAINFLUX_CFG forever. +-- The metrics service uses the plural 'configs' path which fake_hal does not cover. +local function start_mock_hal(conn, root_scope) + local ep = conn:bind( + { 'cap', 'fs', 'credentials', 'rpc', 'read' }, + { queue_len = 5 }) + + root_scope:spawn(function() + while true do + local req, _ = perform(ep:recv_op()) + if not req then break end + req:reply({ ok = true, reason = MAINFLUX_CFG }) + end + end) +end + +-- Start the metrics service in a fresh child scope. +-- Clears package.loaded so each test gets a clean module-level State. +local function start_metrics(bus, root_scope, opts) + local svc_scope = root_scope:child() + svc_scope:spawn(function() + package.loaded['services.metrics'] = nil + local metrics = require 'services.metrics' + local svc_conn = bus:connect() + metrics.start(svc_conn, opts or { name = 'metrics' }) + end) + return svc_scope +end + +local function stop_scope(svc_scope) + svc_scope:cancel('test done') + perform(svc_scope:join_op()) +end + +local METRICS_SCHEMA = 'devicecode.config/metrics/1' + +local function bus_pipeline_config(metric_name, publish_period, process) + return { + data = { + schema = METRICS_SCHEMA, + publish_period = publish_period or 0.1, + pipelines = { + [metric_name] = { + protocol = 'bus', + process = process or {}, + }, + }, + }, + } +end + +------------------------------------------------------------------------------- +-- Tests +------------------------------------------------------------------------------- + +local T = {} + +-- Publish a metric and verify the processed value is re-published on the bus. +function T.metric_published_via_bus() + runfibers.run(function(scope) + local clock = new_test_clock() + scope:finally(function() clock:restore() end) + local bus = make_bus() + local test_conn = bus:connect() + + test_conn:retain({ 'cap', 'fs', 'credentials', 'state' }, 'added') + start_mock_hal(test_conn, scope) + test_conn:retain({ 'state', 'time', 'synced' }, true) + + local result_sub = test_conn:subscribe( + { 'obs', 'v1', 'metrics', 'output', '#' }, + { queue_len = 10, full = 'drop_oldest' }) + + test_conn:retain({ 'cfg', 'metrics' }, bus_pipeline_config('sim', 0.1)) + + local svc_scope = start_metrics(bus, scope) + flush_ticks() + + test_conn:publish( + { 'obs', 'v1', 'modem', 'metric', 'sim' }, + { value = 'present', namespace = { 'modem', 1, 'sim' } }) + + local msg = recv_metric(clock, result_sub, 0.5) + + assert(msg ~= nil, 'expected bus publish of sim metric') + assert(msg.payload.value == 'present', + 'expected value=present, got ' .. tostring(msg.payload.value)) + + stop_scope(svc_scope) + clock:restore() + end, { timeout = 3.0 }) +end + +-- A namespace field in the payload overrides the output bus topic segments. +function T.namespace_overrides_topic_key() + runfibers.run(function(scope) + local clock = new_test_clock() + scope:finally(function() clock:restore() end) + local bus = make_bus() + local test_conn = bus:connect() + + test_conn:retain({ 'cap', 'fs', 'credentials', 'state' }, 'added') + start_mock_hal(test_conn, scope) + test_conn:retain({ 'state', 'time', 'synced' }, true) + + local result_sub = test_conn:subscribe( + { 'obs', 'v1', 'metrics', 'output', '#' }, + { queue_len = 10, full = 'drop_oldest' }) + + test_conn:retain({ 'cfg', 'metrics' }, bus_pipeline_config('rx_bytes', 0.1)) + local svc_scope = start_metrics(bus, scope) + flush_ticks() + + test_conn:publish( + { 'obs', 'v1', 'network', 'metric', 'rx_bytes' }, + { value = 1024, namespace = { 'wan', 'rx_bytes' } }) + + local msg = recv_metric(clock, result_sub, 0.5) + + assert(msg ~= nil, 'expected bus publish with namespace key') + assert(msg.topic[5] == 'wan', + 'expected topic[5]=wan, got ' .. tostring(msg.topic[5])) + assert(msg.topic[6] == 'rx_bytes', + 'expected topic[6]=rx_bytes, got ' .. tostring(msg.topic[6])) + assert(msg.payload.value == 1024, + 'expected value=1024, got ' .. tostring(msg.payload.value)) + + stop_scope(svc_scope) + clock:restore() + end, { timeout = 3.0 }) +end + +-- A metric with no matching pipeline is silently dropped. +function T.unknown_metric_dropped() + runfibers.run(function(scope) + local clock = new_test_clock() + scope:finally(function() clock:restore() end) + local bus = make_bus() + local test_conn = bus:connect() + + test_conn:retain({ 'cap', 'fs', 'credentials', 'state' }, 'added') + start_mock_hal(test_conn, scope) + test_conn:retain({ 'state', 'time', 'synced' }, true) + + local result_sub = test_conn:subscribe( + { 'obs', 'v1', 'metrics', 'output', '#' }, + { queue_len = 10, full = 'drop_oldest' }) + + -- Config only knows about 'sim'; we will publish 'rx_bytes'. + test_conn:retain({ 'cfg', 'metrics' }, bus_pipeline_config('sim', 0.1)) + local svc_scope = start_metrics(bus, scope) + flush_ticks() + + test_conn:publish( + { 'obs', 'v1', 'network', 'metric', 'rx_bytes' }, + { value = 9999 }) + + clock:advance(0.25) + time_harness.flush_ticks(20) + local messages = drain_non_status(result_sub) + + assert(#messages == 0, + 'expected 0 messages for unknown metric, got ' .. tostring(#messages)) + + stop_scope(svc_scope) + clock:restore() + end, { timeout = 3.0 }) +end + +-- DiffTrigger with any-change suppresses the second publish when value is unchanged. +function T.difftrigger_suppresses_unchanged_value() + runfibers.run(function(scope) + local clock = new_test_clock() + scope:finally(function() clock:restore() end) + local bus = make_bus() + local test_conn = bus:connect() + + test_conn:retain({ 'cap', 'fs', 'credentials', 'state' }, 'added') + start_mock_hal(test_conn, scope) + test_conn:retain({ 'state', 'time', 'synced' }, true) + + local result_sub = test_conn:subscribe( + { 'obs', 'v1', 'metrics', 'output', '#' }, + { queue_len = 10, full = 'drop_oldest' }) + + test_conn:retain({ 'cfg', 'metrics' }, bus_pipeline_config('sim', 0.1, { + { type = 'DiffTrigger', diff_method = 'any-change' }, + })) + local svc_scope = start_metrics(bus, scope) + flush_ticks() + + -- First publish: new value — should pass DiffTrigger. + test_conn:publish( + { 'obs', 'v1', 'modem', 'metric', 'sim' }, + { value = 'present' }) + + local msg1 = recv_metric(clock, result_sub, 0.4) + assert(msg1 ~= nil, 'expected first publish to pass DiffTrigger') + assert(msg1.payload.value == 'present', + 'unexpected value: ' .. tostring(msg1.payload.value)) + + -- Second publish: same value — DiffTrigger must suppress it. + test_conn:publish( + { 'obs', 'v1', 'modem', 'metric', 'sim' }, + { value = 'present' }) + + clock:advance(0.25) + time_harness.flush_ticks(20) + + local ok, m = time_harness.try_op_now(function() return result_sub:recv_op() end) + assert(not ok or m == nil, 'second publish should be suppressed by DiffTrigger') + + stop_scope(svc_scope) + clock:restore() + end, { timeout = 3.0 }) +end + +-- DeltaValue transforms a cumulative counter into a per-period delta. +function T.delta_value_pipeline() + runfibers.run(function(scope) + local clock = new_test_clock() + scope:finally(function() clock:restore() end) + local bus = make_bus() + local test_conn = bus:connect() + + test_conn:retain({ 'cap', 'fs', 'credentials', 'state' }, 'added') + start_mock_hal(test_conn, scope) + test_conn:retain({ 'state', 'time', 'synced' }, true) + + local result_sub = test_conn:subscribe( + { 'obs', 'v1', 'metrics', 'output', '#' }, + { queue_len = 10, full = 'drop_oldest' }) + + test_conn:retain({ 'cfg', 'metrics' }, bus_pipeline_config('rx_bytes', 0.1, { + { type = 'DeltaValue', initial_val = 0 }, + })) + local svc_scope = start_metrics(bus, scope) + flush_ticks() + + -- First reading: 1000 bytes; delta from initial 0 = 1000. + test_conn:publish( + { 'obs', 'v1', 'network', 'metric', 'rx_bytes' }, + { value = 1000 }) + + local msg1 = recv_metric(clock, result_sub, 0.4) + assert(msg1 ~= nil, 'expected first delta publish') + assert(msg1.payload.value == 1000, + 'expected delta=1000, got ' .. tostring(msg1.payload.value)) + + -- Second reading: 1500 bytes; delta from 1000 = 500. + test_conn:publish( + { 'obs', 'v1', 'network', 'metric', 'rx_bytes' }, + { value = 1500 }) + + local msg2 = recv_metric(clock, result_sub, 0.4) + assert(msg2 ~= nil, 'expected second delta publish') + assert(msg2.payload.value == 500, + 'expected delta=500, got ' .. tostring(msg2.payload.value)) + + stop_scope(svc_scope) + clock:restore() + end, { timeout = 3.0 }) +end + +-- HTTP pipelines should enqueue a well-formed Mainflux request payload. +function T.http_pipeline_enqueues_request_payload() + -- Save originals so we can restore after the test. + local original_http_mod = package.loaded['services.metrics.http'] + + local captured = nil + + -- Stub the HTTP publisher so we can inspect what gets enqueued. + package.loaded['services.metrics.http'] = { + start_http_publisher = function() + return { + put_op = function(_, data) + captured = data + return op.always(true) + end, + } + end, + } + + local ok_run, run_err = pcall(function() + runfibers.run(function(scope) + local clock = new_test_clock() + scope:finally(function() clock:restore() end) + local bus = make_bus() + local test_conn = bus:connect() + + test_conn:retain({ 'cap', 'fs', 'credentials', 'state' }, 'added') + start_mock_hal(test_conn, scope) + test_conn:retain({ 'state', 'time', 'synced' }, true) + + test_conn:retain({ 'cfg', 'metrics' }, { + data = { + schema = METRICS_SCHEMA, + publish_period = 0.1, + cloud_url = 'http://localhost:18080', + pipelines = { + sim = { + protocol = 'http', + process = {}, + }, + }, + }, + }) + + local svc_scope = start_metrics(bus, scope) + flush_ticks() + + test_conn:publish( + { 'obs', 'v1', 'modem', 'metric', 'sim' }, + { value = 'present', namespace = { 'modem', 1, 'sim' } }) + + clock:advance(0.3) + time_harness.flush_ticks(20) + + assert(captured ~= nil, 'expected HTTP payload to be enqueued') + assert(captured.uri == 'http://localhost:18080/http/channels/ch-data/messages', + 'unexpected uri: ' .. tostring(captured.uri)) + assert(captured.auth == 'Thing test-thing-key', + 'unexpected auth: ' .. tostring(captured.auth)) + assert(captured.body ~= nil, 'expected non-nil body') + + local recs, decode_err = json.decode(captured.body) + assert(decode_err == nil, 'JSON decode error: ' .. tostring(decode_err)) + assert(type(recs) == 'table', 'expected table of records') + assert(#recs == 1, 'expected 1 record, got ' .. tostring(#recs)) + assert(recs[1].n == 'modem.1.sim', + 'expected n=modem.1.sim, got ' .. tostring(recs[1].n)) + assert(recs[1].vs == 'present', + 'expected vs=present, got ' .. tostring(recs[1].vs)) + + stop_scope(svc_scope) + clock:restore() + end, { timeout = 3.0 }) + end) + + -- Always restore the stub. + package.loaded['services.metrics.http'] = original_http_mod + + assert(ok_run, tostring(run_err)) +end + +-- Receiving a new config replaces existing pipelines; old metric names are dropped. +function T.config_update_replaces_pipelines() + runfibers.run(function(scope) + local clock = new_test_clock() + scope:finally(function() clock:restore() end) + local bus = make_bus() + local test_conn = bus:connect() + + test_conn:retain({ 'cap', 'fs', 'credentials', 'state' }, 'added') + start_mock_hal(test_conn, scope) + test_conn:retain({ 'state', 'time', 'synced' }, true) + + local result_sub = test_conn:subscribe( + { 'obs', 'v1', 'metrics', 'output', '#' }, + { queue_len = 20, full = 'drop_oldest' }) + + -- Initial config: pipeline for 'sim'. + test_conn:retain({ 'cfg', 'metrics' }, bus_pipeline_config('sim', 0.1)) + local svc_scope = start_metrics(bus, scope) + flush_ticks() + + -- Confirm 'sim' publishes under the initial config. + test_conn:publish( + { 'obs', 'v1', 'modem', 'metric', 'sim' }, + { value = 'present' }) + + local msg1 = recv_metric(clock, result_sub, 0.4) + assert(msg1 ~= nil, 'expected sim metric before config update') + + -- Update config: replace 'sim' pipeline with 'rx_bytes'. + test_conn:retain({ 'cfg', 'metrics' }, bus_pipeline_config('rx_bytes', 0.1)) + flush_ticks() + + -- 'rx_bytes' must publish after the config update. + test_conn:publish( + { 'obs', 'v1', 'network', 'metric', 'rx_bytes' }, + { value = 42 }) + + local msg2 = recv_metric(clock, result_sub, 0.4) + assert(msg2 ~= nil, 'expected rx_bytes metric after config update') + assert(msg2.payload.value == 42, + 'expected value=42, got ' .. tostring(msg2.payload.value)) + + stop_scope(svc_scope) + clock:restore() + end, { timeout = 3.0 }) +end + +-- Two endpoints sharing the same pipeline name maintain isolated processing state. +function T.per_endpoint_state_isolation() + runfibers.run(function(scope) + local clock = new_test_clock() + scope:finally(function() clock:restore() end) + local bus = make_bus() + local test_conn = bus:connect() + + test_conn:retain({ 'cap', 'fs', 'credentials', 'state' }, 'added') + start_mock_hal(test_conn, scope) + test_conn:retain({ 'state', 'time', 'synced' }, true) + + local result_sub = test_conn:subscribe( + { 'obs', 'v1', 'metrics', 'output', '#' }, + { queue_len = 20, full = 'drop_oldest' }) + + test_conn:retain({ 'cfg', 'metrics' }, bus_pipeline_config('rx_bytes', 0.1, { + { type = 'DeltaValue', initial_val = 0 }, + })) + local svc_scope = start_metrics(bus, scope) + flush_ticks() + + -- WAN endpoint: 500 bytes → delta = 500 (from initial 0). + test_conn:publish( + { 'obs', 'v1', 'network', 'metric', 'rx_bytes' }, + { value = 500, namespace = { 'wan', 'rx_bytes' } }) + + -- LAN endpoint: 200 bytes → delta = 200 (independent state from WAN). + test_conn:publish( + { 'obs', 'v1', 'network', 'metric', 'rx_bytes' }, + { value = 200, namespace = { 'lan', 'rx_bytes' } }) + + -- Collect both publishes within one tick window. + local received = {} + for _ = 1, 2 do + local msg = recv_metric(clock, result_sub, 0.4) + if msg then + local key = table.concat(msg.topic, '.') + received[key] = msg.payload.value + end + end + + assert(received['obs.v1.metrics.output.wan.rx_bytes'] == 500, + 'expected wan delta=500, got ' .. tostring(received['obs.v1.metrics.output.wan.rx_bytes'])) + assert(received['obs.v1.metrics.output.lan.rx_bytes'] == 200, + 'expected lan delta=200, got ' .. tostring(received['obs.v1.metrics.output.lan.rx_bytes'])) + + stop_scope(svc_scope) + clock:restore() + end, { timeout = 3.0 }) +end + +return T diff --git a/tests/integration/devhost/monitor_logging_spec.lua b/tests/integration/devhost/monitor_logging_spec.lua new file mode 100644 index 00000000..f56e8ba9 --- /dev/null +++ b/tests/integration/devhost/monitor_logging_spec.lua @@ -0,0 +1,189 @@ +local busmod = require 'bus' +local fibers = require 'fibers' +local cjson_ok, cjson = pcall(require, 'cjson.safe') +if not cjson_ok then cjson = require 'cjson' end + +local run_fibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' +local monitor = require 'services.monitor' +local request = require 'services.ui.http.request' + +local T = {} + +local function assert_eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end + +local function assert_true(v, msg) + if not v then error(msg or 'assertion failed', 2) end + return v +end + +local function monitor_rpc(method) return { 'cap', 'monitor', 'main', 'rpc', method } end +local function log_topic(service) return { 'obs', 'v1', service, 'event', 'log' } end + +local function fake_ctx(method, path) + return { + method = method, + path = path, + replies = {}, + _current = nil, + write_headers_op = function(self, status, headers, opts) + return fibers.always(function () + self._current = { status = status, body = '', headers = headers, end_stream = opts and opts.end_stream } + if opts and opts.end_stream then + self.replies[#self.replies + 1] = self._current + self._current = nil + end + return true, nil + end):wrap(function (th) return th() end) + end, + write_chunk_op = function(self, chunk, opts) + return fibers.always(function () + assert(self._current, 'no response headers') + self._current.body = self._current.body .. tostring(chunk or '') + if opts and opts.end_stream then + self.replies[#self.replies + 1] = self._current + self._current = nil + end + return true, nil + end):wrap(function (th) return th() end) + end, + terminate = function(self, reason) self.abandoned = reason; return true end, + } +end + +local function publish_log(conn, service, level, what, summary) + conn:publish(log_topic(service), { + service = service, + level = level or 'info', + what = what, + summary = summary or what, + }) +end + +function T.monitor_public_capability_feeds_ui_initial_logs() + run_fibers.run(function (scope) + local b = busmod.new() + local monitor_conn = b:connect({ origin_base = { service = 'monitor' } }) + local app_conn = b:connect({ origin_base = { service = 'app' } }) + local ui_conn = b:connect({ origin_base = { service = 'ui' } }) + local ok, err = scope:spawn(function () monitor.start(monitor_conn, { env = 'test' }) end) + assert_true(ok, tostring(err)) + probe.wait_retained_payload(ui_conn, { 'cap', 'monitor', 'main' }, { timeout = 0.5 }) + + publish_log(app_conn, 'net', 'info', 'speedtest_completed', 'speedtest wan on wan/vl-wan: 100 Mbps') + probe.wait_until(function () + local rep = ui_conn:call(monitor_rpc('query-logs'), { service = 'net', limit = 1 }, { timeout = 0.05 }) + return rep and rep.count == 1 + end, { timeout = 0.8, interval = 0.01 }) + + local ctx = fake_ctx('GET', '/api/logs') + local result = request.run(scope, ctx, { + conn = ui_conn, + encode_json = function (v) return assert(cjson.encode(v)) end, + }) + assert_eq(result.status, 'ok') + assert_eq(#ctx.replies, 1) + assert_eq(ctx.replies[1].status, 200) + local decoded = assert_true(cjson.decode(ctx.replies[1].body), ctx.replies[1].body) + assert_true(decoded.records and #decoded.records >= 1, 'expected log records in UI response') + local found = false + for _, rec in ipairs(decoded.records) do + if rec.service == 'net' and rec.what == 'speedtest_completed' then found = true end + end + assert_true(found, 'UI logs response should include monitor query result') + end) +end + +function T.ui_logs_endpoint_can_request_boot_buffer() + run_fibers.run(function (scope) + local b = busmod.new() + local admin = b:connect({ origin_base = { service = 'monitor-test-admin' } }) + admin:retain({ 'cfg', 'monitor' }, { + data = { storage = { boot_records = 10, ring_records = 2, boot_seconds = 0 } }, + rev = 1, + }) + local monitor_conn = b:connect({ origin_base = { service = 'monitor' } }) + local app_conn = b:connect({ origin_base = { service = 'app' } }) + local ui_conn = b:connect({ origin_base = { service = 'ui' } }) + local ok, err = scope:spawn(function () monitor.start(monitor_conn, { env = 'test' }) end) + assert_true(ok, tostring(err)) + probe.wait_retained_payload(ui_conn, { 'cap', 'monitor', 'main' }, { timeout = 0.5 }) + + for i = 1, 5 do publish_log(app_conn, 'alpha', 'info', 'record_' .. i, 'record ' .. i) end + probe.wait_until(function () + local rep = ui_conn:call(monitor_rpc('query-logs'), { service = 'alpha', limit = 10 }, { timeout = 0.05 }) + return rep and rep.count == 2 and rep.records[1].what == 'record_4' + end, { timeout = 0.8, interval = 0.01 }) + + local ring_ctx = fake_ctx('GET', '/api/logs') + local ring_result = request.run(scope, ring_ctx, { + conn = ui_conn, + encode_json = function (v) return assert(cjson.encode(v)) end, + }) + assert_eq(ring_result.status, 'ok') + local ring = assert_true(cjson.decode(ring_ctx.replies[1].body), ring_ctx.replies[1].body) + + local boot_ctx = fake_ctx('GET', '/api/logs?boot=true') + local boot_result = request.run(scope, boot_ctx, { + conn = ui_conn, + encode_json = function (v) return assert(cjson.encode(v)) end, + }) + assert_eq(boot_result.status, 'ok') + local boot = assert_true(cjson.decode(boot_ctx.replies[1].body), boot_ctx.replies[1].body) + + local ring_found_first = false + for _, rec in ipairs(ring.records or {}) do + if rec.service == 'alpha' and rec.what == 'record_1' then ring_found_first = true end + end + assert_true(not ring_found_first, 'plain UI logs response should use the ring buffer') + + local boot_found_first = false + for _, rec in ipairs(boot.records or {}) do + if rec.service == 'alpha' and rec.what == 'record_1' then boot_found_first = true end + end + assert_true(boot_found_first, 'boot UI logs response should include startup records') + end) +end + +function T.monitor_profile_endpoint_changes_profile_through_ui_route() + run_fibers.run(function (scope) + local b = busmod.new() + local monitor_conn = b:connect({ origin_base = { service = 'monitor' } }) + local ui_conn = b:connect({ origin_base = { service = 'ui' } }) + local ok, err = scope:spawn(function () monitor.start(monitor_conn, { env = 'test' }) end) + assert_true(ok, tostring(err)) + probe.wait_retained_payload(ui_conn, { 'cap', 'monitor', 'main' }, { timeout = 0.5 }) + + local ctx = fake_ctx('POST', '/api/monitor/profile') + ctx.headers = { ['content-type'] = 'application/json', ['x-session-id'] = 'sid-1' } + ctx.read_body_as_string_op = function () return fibers.always('{"profile":"debug"}', nil) end + local sessions = { + get = function (_, sid) + if sid == 'sid-1' then + return { id = sid, principal = { kind = 'user', id = 'tester' } } + end + end, + } + local result = request.run(scope, ctx, { + conn = ui_conn, + bus = b, + sessions = sessions, + -- Return the borrowed UI connection here to keep the test focused on + -- route behaviour. request.run passes deps.conn as the borrowed + -- connection owner, so user_operation must not disconnect it. + connect = function () return ui_conn end, + encode_json = function (v) return assert(cjson.encode(v)) end, + }) + assert_eq(result.status, 'ok') + assert_eq(#ctx.replies, 1) + assert_eq(ctx.replies[1].status, 200) + probe.wait_until(function () + local rep = ui_conn:call(monitor_rpc('query-logs'), { limit = 0 }, { timeout = 0.05 }) + return rep and rep.summary and rep.summary.profile == 'debug' + end, { timeout = 0.8, interval = 0.01 }) + end) +end + +return T diff --git a/tests/integration/devhost/rtl8380m_switch_spec.lua b/tests/integration/devhost/rtl8380m_switch_spec.lua new file mode 100644 index 00000000..c9ab906b --- /dev/null +++ b/tests/integration/devhost/rtl8380m_switch_spec.lua @@ -0,0 +1,533 @@ +-- tests/integration/devhost/rtl8380m_switch_spec.lua +-- +-- Opt-in devhost tests for the real RTL8380M PoE/VLAN switch provider. +-- +-- These tests talk to a real switch and are skipped unless the operator +-- supplies explicit test credentials: +-- +-- SWITCH_TEST_BASE_URL=http://192.168.1.1/ \ +-- SWITCH_TEST_USERNAME=admin \ +-- SWITCH_TEST_PASSWORD=admin \ +-- TEST_FILTER=rtl8380m_real_switch \ +-- lua tests/run.lua +-- +-- The tests exercise the production provider through cap/http/main. They do +-- not submit VLAN, PoE, save, reboot or other configuration writes. + +local busmod = require 'bus' +local fibers = require 'fibers' +local channel = require 'fibers.channel' + +local runfibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' + +local http_service = require 'services.http.service' +local wired_manager = require 'services.hal.managers.wired' +local wired_service = require 'services.wired.service' +local wired_config = require 'services.wired.config' +local wired_topics = require 'services.wired.topics' +local provider_mod = require 'services.hal.backends.wired.providers.rtl8380m_http' +local hal_deps = require 'services.hal.dependencies' + +local T = {} + +local function skip(reason) + return { skip = true, reason = reason } +end + +local function assert_true(v, msg) + if v ~= true then error(msg or ('expected true, got ' .. tostring(v)), 2) end +end + +local function assert_eq(a, b, msg) + if a ~= b then error(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a)), 2) end +end + +local function assert_not_nil(v, msg) + if v == nil then error(msg or 'expected non-nil', 2) end + return v +end + +local function required_env() + local base_url = os.getenv('SWITCH_TEST_BASE_URL') + local username = os.getenv('SWITCH_TEST_USERNAME') + local password = os.getenv('SWITCH_TEST_PASSWORD') + local missing = {} + if not base_url or base_url == '' then missing[#missing + 1] = 'SWITCH_TEST_BASE_URL' end + if not username or username == '' then missing[#missing + 1] = 'SWITCH_TEST_USERNAME' end + if not password or password == '' then missing[#missing + 1] = 'SWITCH_TEST_PASSWORD' end + if #missing > 0 then + return nil, 'set ' .. table.concat(missing, ', ') .. ' to run the real switch integration test' + end + return { + base_url = base_url, + username = username, + password = password, + timeout_s = tonumber(os.getenv('SWITCH_TEST_HTTP_TIMEOUT_S') or '8') or 8, + run_timeout_s = tonumber(os.getenv('SWITCH_TEST_RUN_TIMEOUT_S') or '30') or 30, + openssl_bin = os.getenv('SWITCH_TEST_OPENSSL') or os.getenv('SWITCH_OPENSSL') or 'openssl', + } +end + +local function wait_http_available(bus) + local reader = bus:connect({ origin_base = { kind = 'local', component = 'test-http-reader' } }) + probe.wait_retained_payload(reader, { 'cap', 'http', 'main', 'status' }, { + timeout = 2.0, + view_topic = { 'cap', 'http', 'main', 'status' }, + }) + assert_true(probe.wait_until(function () + local view = reader:retained_view({ 'cap', 'http', 'main', 'status' }) + local msg = view:get({ 'cap', 'http', 'main', 'status' }) + view:close() + local payload = msg and msg.payload or nil + return payload and payload.available == true + end, { timeout = 2.0, interval = 0.01 }), 'HTTP capability should become available') +end + +local function start_http_capability(bus, opts) + local conn = bus:connect({ origin_base = { kind = 'local', component = 'test-http-service' } }) + return assert(http_service.open_handle(conn, { + id = 'main', + backend_timeout = opts.timeout_s, + connection_setup_timeout = opts.timeout_s, + intra_stream_timeout = opts.timeout_s, + max_accept_queue = 8, + policy = { + allowed_response_parsers = { strict = true, ['legacy-http1-close'] = true }, + legacy_http1_close_max_response_bytes = 1024 * 1024, + }, + })) +end + +local function count_surfaces_with_prefix(surfaces, prefix) + local n = 0 + for name in pairs(surfaces or {}) do + if tostring(name):sub(1, #prefix) == prefix then n = n + 1 end + end + return n +end + +local function count_poe_surfaces(surfaces) + local n = 0 + for _, surface in pairs(surfaces or {}) do + if surface.capabilities and surface.capabilities.poe == true then n = n + 1 end + end + return n +end + +local function has_known_vlan_mode(surface) + local mode = surface and surface.attachment and surface.attachment.mode + return mode == nil or mode == 'hybrid' or mode == 'access' or mode == 'trunk' or mode == 'tunnel' +end + +local function sorted_surface_ids(surfaces) + local ids = {} + for id in pairs(surfaces or {}) do ids[#ids + 1] = id end + table.sort(ids, function (a, b) return tostring(a) < tostring(b) end) + return ids +end + +local function choose_probe_surface(surfaces) + local ids = sorted_surface_ids(surfaces) + for i = 1, #ids do + local id = ids[i] + local s = surfaces[id] + if tostring(id):match('^GE') and id ~= 'GE8' and s and s.capabilities and s.capabilities.poe == true then + return id, s + end + end + for i = 1, #ids do + local id = ids[i] + if tostring(id):match('^GE') and id ~= 'GE8' then return id, surfaces[id] end + end + for i = 1, #ids do + local id = ids[i] + if tostring(id):match('^GE') then return id, surfaces[id] end + end + local id = ids[1] + return id, id and surfaces[id] or nil +end + +local function new_real_switch_provider(bus, env) + local provider_conn = bus:connect({ origin_base = { kind = 'local', component = 'rtl8380m-switch-test' } }) + local resolver = assert(hal_deps.resolver(provider_conn)) + return assert(provider_mod.new({ + base_url = env.base_url, + username = env.username, + password = env.password, + timeout_s = env.timeout_s, + openssl_bin = env.openssl_bin, + include_raw = true, + http = { capability = 'main', response_parser = 'legacy-http1-close' }, + }, { provider_id = 'switch-main', http_client_for = resolver:factory('http_client') })) +end + +local function require_successful_snapshot(provider) + local snap = fibers.perform(provider:snapshot_op({})) + assert_not_nil(snap, 'snapshot should return a table') + if snap.ok ~= true then + local status = snap.status or {} + error('switch snapshot failed: ' .. tostring(status.err or snap.err or 'unknown error'), 2) + end + return snap +end + +local function require_successful_observe_groups(provider, groups) + local result = fibers.perform(provider:observe_groups_op({ groups = groups })) + assert_not_nil(result, 'observe_groups should return a table') + if result.ok ~= true then + local status = result.status or {} + error('switch observe_groups failed: ' .. tostring(status.err or result.err or 'unknown error'), 2) + end + return result +end + +local function assert_command_once(commands, command) + local n = 0 + for _, cmd in ipairs(commands or {}) do if cmd == command then n = n + 1 end end + assert_eq(n, 1, 'expected command ' .. tostring(command) .. ' exactly once') +end + +local function retain_switch_raw(conn, snap) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'status' }, snap.status or {}) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'identity' }, snap.identity or {}) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'runtime' }, snap.runtime or {}) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'power' }, snap.power or {}) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'surfaces' }, { + surfaces = snap.surfaces or {}, + }) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'topology' }, snap.topology or {}) +end + +local function retain_device_assembly(conn, observed_surface) + conn:retain({ 'state', 'device', 'assembly' }, { + kind = 'device.assembly', + product = 'big-box', + components = { + cm5 = { kind = 'compute', role = 'controller' }, + mcu = { kind = 'microcontroller', role = 'power-sensor-controller' }, + ['switch-main'] = { kind = 'switch', role = 'wired-fabric' }, + }, + links = { + ['cm5-switch'] = { + kind = 'wired', + role = 'controller-switch-uplink', + internal = true, + a = { component = 'cm5', observed_surface = 'eth0' }, + b = { component = 'switch-main', observed_surface = 'GE8' }, + }, + }, + surfaces = { + ['lan-probe'] = { + kind = 'ethernet', + exposure = 'external', + component = 'switch-main', + observed_surface = observed_surface, + }, + }, + }) +end + +local function retain_wired_config(conn) + conn:retain({ 'cfg', 'wired' }, { + rev = 1, + data = { + schema = wired_config.SCHEMA, + version = 1, + surfaces = { + ['lan-probe'] = { + kind = 'ethernet-port', + role = 'external-observed-port', + attachment = { mode = 'none' }, + }, + }, + }, + }) +end + +local function raw_provider_topic(id, suffix) + local topic = { 'raw', 'host', 'wired', 'provider', id } + for i = 1, #(suffix or {}) do topic[#topic + 1] = suffix[i] end + return topic +end + +local function start_wired_manager_hal_harness(scope, bus, dev_ev_ch, cap_emit_ch) + local child = assert(scope:child()) + local writer = bus:connect({ origin_base = { kind = 'local', component = 'wired-manager-hal-harness' } }) + + assert(child:spawn(function () + while true do + local ev = fibers.perform(dev_ev_ch:get_op()) + if ev == nil then return end + if ev.class == 'wired' and ev.id == 'main' then + if ev.event_type == 'added' then + for _, cap in ipairs(ev.capabilities or {}) do + if cap.class == 'wired-provider' then + writer:retain(raw_provider_topic(cap.id, { 'status' }), { + state = 'available', + available = true, + source_kind = 'host', + source = 'wired', + }) + writer:retain(raw_provider_topic(cap.id, { 'meta' }), { + offerings = cap.offerings or {}, + source_kind = 'host', + source = 'wired', + }) + end + end + if ev.ready_cond then ev.ready_cond:signal() end + elseif ev.event_type == 'removed' then + for _, cap in ipairs(ev.capabilities or {}) do + if cap.class == 'wired-provider' then writer:unretain(raw_provider_topic(cap.id, { 'status' })) end + end + if ev.ready_cond then ev.ready_cond:signal() end + end + end + end + end)) + + assert(child:spawn(function () + while true do + local emit = fibers.perform(cap_emit_ch:get_op()) + if emit == nil then return end + if emit.class == 'wired-provider' and emit.mode == 'state' then + if emit.key == 'status' then + writer:retain(raw_provider_topic(emit.id, { 'status' }), emit.data or {}) + else + writer:retain(raw_provider_topic(emit.id, { 'state', emit.key }), emit.data or {}) + end + end + end + end)) + + return child +end + +local function switch_manager_config(env) + return { + providers = { + ['switch-main'] = { + provider = 'rtl8380m_http', + base_url = env.base_url, + username = env.username, + password = env.password, + timeout_s = env.timeout_s, + openssl_bin = env.openssl_bin, + include_raw = true, + http = { capability = 'main', response_parser = 'legacy-http1-close' }, + poll = { + fast = { interval_s = 1.0, groups = { 'panel', 'poe', 'counters' } }, + }, + }, + }, + } +end + +function T.rtl8380m_real_switch_observe_groups_via_http_capability() + local env, err = required_env() + if not env then return skip(err) end + + runfibers.run(function () + local b = busmod.new() + local http = start_http_capability(b, env) + wait_http_available(b) + local provider = new_real_switch_provider(b, env) + local obs = require_successful_observe_groups(provider, { 'panel', 'poe', 'counters' }) + + assert_eq(obs.provider_id, 'switch-main') + assert_eq(obs.status.driver, 'rtl8380m_http') + assert_true(obs.status.available, 'observe_groups status should be available') + assert_not_nil(obs.surfaces, 'observe_groups should include surfaces') + assert_not_nil(obs.surfaces.GE8, 'observe_groups should include GE8') + assert_not_nil(obs.surfaces.GE9, 'observe_groups should include GE9') + assert_eq(obs.surfaces.GE9.link.media, 'fiber') + assert_not_nil(obs.power, 'poe group should include power') + assert_not_nil(obs.power.poe, 'poe group should include power.poe') + assert_not_nil(obs.raw, 'include_raw=true should preserve grouped source payloads') + assert_not_nil(obs.raw.home_main, 'grouped observation should capture home_main') + assert_not_nil(obs.raw.panel_info, 'grouped observation should capture panel_info') + assert_not_nil(obs.raw.poe_poe, 'grouped observation should capture poe_poe') + assert_not_nil(obs.raw.rmon_statistics, 'grouped observation should capture rmon_statistics') + assert_command_once(obs.commands, 'home_main') + assert_command_once(obs.commands, 'panel_info') + assert_command_once(obs.commands, 'poe_poe') + assert_command_once(obs.commands, 'rmon_statistics') + + provider:terminate('test complete') + http:terminate('test complete') + end, { timeout = env.run_timeout_s }) +end + +function T.rtl8380m_real_switch_runner_publishes_raw_observations() + local env, err = required_env() + if not env then return skip(err) end + + runfibers.run(function (scope) + local b = busmod.new() + local http = start_http_capability(b, env) + wait_http_available(b) + + local manager_conn = b:connect({ origin_base = { kind = 'local', component = 'wired-manager-real-switch-test' } }) + local resolver = assert(hal_deps.resolver(manager_conn)) + local dev_ev_ch = channel.new(16) + local cap_emit_ch = channel.new(32) + local harness = start_wired_manager_hal_harness(scope, b, dev_ev_ch, cap_emit_ch) + local reader = b:connect({ origin_base = { kind = 'local', component = 'wired-runner-test-reader' } }) + + wired_manager.terminate('test reset') + local ok_start, start_err = fibers.perform(wired_manager.start_op(nil, dev_ev_ch, cap_emit_ch, { + http_client_for = resolver:factory('http_client'), + })) + assert_true(ok_start, tostring(start_err)) + + local ok_apply, apply_err = fibers.perform(wired_manager.apply_config_op(switch_manager_config(env))) + assert_true(ok_apply, tostring(apply_err)) + + local status = probe.wait_retained_payload(reader, raw_provider_topic('switch-main', { 'status' }), { timeout = 3.0 }) + assert_not_nil(status, 'runner should publish raw provider status') + assert_true(status.available == true or status.state == 'observing', 'raw provider status should be observing or available') + + local surfaces_payload = probe.wait_retained_payload(reader, raw_provider_topic('switch-main', { 'state', 'surfaces' }), { timeout = env.run_timeout_s }) + local surfaces = assert_not_nil(surfaces_payload.surfaces, 'runner should publish raw surfaces') + assert_not_nil(surfaces.GE8, 'runner surfaces should include GE8') + assert_not_nil(surfaces.GE9, 'runner surfaces should include GE9') + assert_eq(surfaces.GE9.link.media, 'fiber') + + local power = probe.wait_retained_payload(reader, raw_provider_topic('switch-main', { 'state', 'power' }), { timeout = env.run_timeout_s }) + assert_not_nil(power.poe, 'runner should publish PoE power') + + wired_manager.terminate('test complete') + harness:cancel('test complete') + fibers.perform(harness:join_op()) + http:terminate('test complete') + end, { timeout = env.run_timeout_s + 5 }) +end + +function T.rtl8380m_real_switch_snapshot_via_http_capability() + local env, err = required_env() + if not env then return skip(err) end + + runfibers.run(function () + local b = busmod.new() + local http = start_http_capability(b, env) + wait_http_available(b) + local provider = new_real_switch_provider(b, env) + local snap = require_successful_snapshot(provider) + + assert_eq(snap.provider_id, 'switch-main') + assert_eq(snap.mode, 'read_only') + assert_eq(snap.writable, false) + assert_not_nil(snap.status, 'snapshot should include provider status') + assert_eq(snap.status.driver, 'rtl8380m_http') + assert_eq(snap.status.login, 'confirmed') + assert_true(snap.status.available, 'provider status should be available') + + assert_not_nil(snap.identity, 'snapshot should include identity') + assert_not_nil(snap.identity.model, 'identity.model should be populated') + assert_not_nil(snap.identity.mac, 'identity.mac should be populated') + assert_not_nil(snap.identity.firmware, 'identity.firmware should be populated') + + assert_not_nil(snap.runtime, 'snapshot should include runtime') + assert_not_nil(snap.runtime.cpu, 'runtime.cpu should be present') + assert_not_nil(snap.runtime.memory, 'runtime.memory should be present') + assert_not_nil(snap.power, 'snapshot should include power') + assert_not_nil(snap.power.poe, 'power.poe should be present') + + local surfaces = assert_not_nil(snap.surfaces, 'snapshot should include surfaces') + assert_true(count_surfaces_with_prefix(surfaces, 'GE') >= 10, 'expected ten GE-labelled switch ports') + assert_not_nil(surfaces.GE8, 'GE8 should be present as the Big Box CM5 switch-uplink port') + assert_eq(surfaces.GE9.link.media, 'fiber', 'GE9 should be the first SFP/fibre surface') + assert_eq(surfaces.GE10.link.media, 'fiber', 'GE10 should be the second SFP/fibre surface') + assert_true(count_poe_surfaces(surfaces) >= 1, 'expected at least one PoE-capable surface') + + for name, surface in pairs(surfaces) do + assert_eq(surface.provider_surface_id, name, 'surface id should match table key') + assert_not_nil(surface.kind, 'surface should have kind') + assert_not_nil(surface.link, 'surface should have link state') + assert_not_nil(surface.attachment, 'surface should have attachment state') + assert_true(has_known_vlan_mode(surface), 'surface ' .. tostring(name) .. ' should have a known VLAN mode') + if surface.capabilities and surface.capabilities.poe == true then + assert_not_nil(surface.poe, 'PoE-capable surface should include poe state') + assert_not_nil(surface.poe.state, 'PoE state should be normalised') + end + end + + local raw = assert_not_nil(snap.raw, 'include_raw=true should preserve source payloads') + assert_not_nil(raw.home_main, 'raw home_main should be captured') + assert_not_nil(raw.panel_info, 'raw panel_info should be captured') + assert_not_nil(raw.port_port, 'raw port_port should be captured') + assert_not_nil(raw.vlan_port, 'raw vlan_port should be captured') + assert_not_nil(raw.vlan_membership, 'raw vlan_membership should be captured') + assert_not_nil(raw.poe_poe, 'raw poe_poe should be captured') + assert_not_nil(raw.sys_cpumem, 'raw sys_cpumem should be captured') + assert_not_nil(raw.rmon_statistics, 'raw rmon_statistics should be captured') + + provider:terminate('test complete') + http:terminate('test complete') + end, { timeout = env.run_timeout_s }) +end + +function T.rtl8380m_real_switch_raw_observations_project_to_state_wired() + local env, err = required_env() + if not env then return skip(err) end + + runfibers.run(function (scope) + local b = busmod.new() + local http = start_http_capability(b, env) + wait_http_available(b) + local provider = new_real_switch_provider(b, env) + local snap = require_successful_snapshot(provider) + local surfaces = assert_not_nil(snap.surfaces, 'snapshot should include surfaces') + local probe_id, raw_surface = choose_probe_surface(surfaces) + assert_not_nil(probe_id, 'snapshot should include a switch surface suitable for projection') + assert_not_nil(raw_surface, 'chosen switch surface should be present') + + local service_conn = b:connect({ origin_base = { kind = 'local', component = 'wired-service-test' } }) + local writer = b:connect({ origin_base = { kind = 'local', component = 'wired-test-writer' } }) + local reader = b:connect({ origin_base = { kind = 'local', component = 'wired-test-reader' } }) + + writer:retain({ 'state', 'net', 'segments' }, { rev = 1, segments = {} }) + retain_device_assembly(writer, probe_id) + retain_switch_raw(writer, snap) + retain_wired_config(writer) + + local child = assert(scope:child()) + assert(child:spawn(function (svc_scope) wired_service.run(svc_scope, { conn = service_conn, service_id = 'test-wired' }) end)) + + local projected = probe.wait_retained_payload(reader, wired_topics.surface('lan-probe'), { timeout = 2.0 }) + assert_eq(projected.surface_id, 'lan-probe') + assert_eq(projected.availability.state, 'available') + assert_eq(projected.source.component, 'switch-main') + assert_eq(projected.source.observed_surface, probe_id) + assert_eq(projected.observed.source.component, 'switch-main') + assert_eq(projected.observed.source.observed_surface, probe_id) + assert_eq(projected.observed.source.exposure, 'external') + assert_eq(projected.link.state, raw_surface.link and raw_surface.link.state) + if raw_surface.link and raw_surface.link.speed_mbps ~= nil then + assert_eq(projected.link.speed_mbps, raw_surface.link.speed_mbps) + end + assert_eq(projected.observed.attachment.mode, raw_surface.attachment and raw_surface.attachment.mode) + assert_eq(projected.counters, nil, 'stable public surface should not carry volatile counters') + local raw_counters = snap.counters and snap.counters[probe_id] + if raw_counters and raw_counters.rx and raw_counters.rx.bytes ~= nil then + local projected_counters = probe.wait_retained_payload(reader, wired_topics.surface_counters('lan-probe'), { timeout = 1.0 }) + assert_eq(projected_counters.counters.rx.bytes, raw_counters.rx.bytes) + end + if raw_surface.capabilities and raw_surface.capabilities.poe == true then + assert_true(projected.source.observed_surface == probe_id, 'PoE-capable source should still project via assembly') + end + + local violations = probe.wait_retained_payload(reader, wired_topics.violations(), { timeout = 1.0 }) + assert_eq(#(violations.violations or {}), 0) + + child:cancel('test complete') + fibers.perform(child:join_op()) + provider:terminate('test complete') + http:terminate('test complete') + end, { timeout = env.run_timeout_s }) +end + + + +return T diff --git a/tests/integration/devhost/startup_dependency_order_spec.lua b/tests/integration/devhost/startup_dependency_order_spec.lua new file mode 100644 index 00000000..ab769adf --- /dev/null +++ b/tests/integration/devhost/startup_dependency_order_spec.lua @@ -0,0 +1,216 @@ +local fibers = require 'fibers' +local busmod = require 'bus' + +local net_service = require 'services.net.service' +local net_config = require 'services.net.config' +local net_topics = require 'services.net.topics' +local update_service = require 'services.update.service' +local update_topics = require 'services.update.topics' +local probe = require 'tests.support.bus_probe' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end return v end + +local function net_cfg() + return { + schema = net_config.SCHEMA, + version = 1, + segments = { + lan = { kind = 'lan', vlan = 10, addressing = { ipv4 = { mode = 'static', cidr = '172.28.10.1/24' } } }, + }, + interfaces = { + lan_bridge = { kind = 'bridge', role = 'lan', segment = 'lan', members = { 'eth0' } }, + }, + wan = { members = {} }, + } +end + +local function bind_network_config_apply(scope, bus, calls) + local conn = bus:connect() + local ep = assert(conn:bind({ 'cap', 'network-config', 'main', 'rpc', 'apply' }, { queue_len = 8 })) + local ok, err = scope:spawn(function () + while true do + local req = fibers.perform(ep:recv_op()) + if req == nil then return end + local payload = req.payload or {} + calls[#calls + 1] = payload + req:reply({ + ok = true, + reason = { + ok = true, + applied = true, + changed = true, + backend = 'test-network-config', + intent_rev = payload.intent and payload.intent.rev, + }, + }) + end + end) + assert_true(ok, err) + conn:retain({ 'cap', 'network-config', 'main', 'status' }, { + schema = 'devicecode.cap.status/1', + state = 'available', + available = true, + }) + return conn +end + +local function bind_fake_control_store(scope, bus, backing) + backing = backing or {} + local conn = bus:connect() + for _, method in ipairs({ 'list', 'get', 'put', 'delete' }) do + local loop_method = method + local ep = assert(conn:bind({ 'cap', 'control-store', 'update', 'rpc', loop_method }, { queue_len = 8 })) + local ok, err = scope:spawn(function () + while true do + local req = fibers.perform(ep:recv_op()) + if req == nil then return end + local p = req.payload or {} + if loop_method == 'list' then + local keys = {} + local prefix = p.prefix or '' + for k in pairs(backing) do + if k:sub(1, #prefix) == prefix then keys[#keys + 1] = k end + end + table.sort(keys) + req:reply({ ok = true, reason = keys }) + elseif loop_method == 'get' then + if backing[p.key] == nil then + req:reply({ ok = false, reason = 'not found' }) + else + req:reply({ ok = true, reason = backing[p.key] }) + end + elseif loop_method == 'put' then + backing[p.key] = p.data + req:reply({ ok = true, reason = nil }) + elseif loop_method == 'delete' then + backing[p.key] = nil + req:reply({ ok = true, reason = nil }) + end + end + end) + assert_true(ok, err) + end + conn:retain({ 'cap', 'control-store', 'update', 'status' }, { + schema = 'devicecode.cap.status/1', + state = 'available', + available = true, + }) + return conn +end + +local function retain_artifact_store_available(bus) + local conn = bus:connect() + conn:retain({ 'cap', 'artifact-store', 'main', 'status' }, { + schema = 'devicecode.cap.status/1', + state = 'available', + available = true, + }) + return conn +end + +function tests.test_net_and_update_wait_for_delayed_hal_capabilities_then_recover() + fibers.run(function (root_scope) + local bus = busmod.new() + retain_artifact_store_available(bus) + + local net_conn = bus:connect() + local update_conn = bus:connect() + local reader = bus:connect() + local caller = bus:connect() + + local net_child = assert(root_scope:child()) + local ok_net, net_err = net_child:spawn(function (scope) + net_service.run(scope, { + conn = net_conn, + config = net_cfg(), + rev = 100, + observe = false, + }) + end) + assert_true(ok_net, net_err) + + local update_child = assert(root_scope:child()) + local ok_update, update_err = update_child:spawn(function (scope) + update_service.run(scope, { + conn = update_conn, + service_id = 'update', + watch_config = false, + job_store_kind = 'control-store', + config = { schema = 'devicecode.update/1', components = { { component = 'cm5' } } }, + }) + end) + assert_true(ok_update, update_err) + + local net_view = reader:retained_view(net_topics.summary()) + local net_waiting = probe.wait_versioned_until('net waits for delayed network-config', + function () return net_view:version() end, + function (seen) return net_view:changed_op(seen) end, + function () + local msg = net_view:get(net_topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'waiting_for_hal' + and payload.dependencies and payload.dependencies.network_config + and payload.dependencies.network_config.available == false + and payload.pending and payload.pending.network_apply + and payload or nil + end, + { timeout = 0.7 }) + assert_eq(net_waiting.reason, 'network_config_unavailable') + assert_eq(net_waiting.pending.network_apply.rev, 100) + + assert_true(probe.wait_until(function () + local status = caller:call(update_topics.update_manager_rpc('status'), {}, { timeout = 0.05 }) + local snap = status and status.snapshot + return snap and snap.state == 'waiting_for_job_store' + and snap.dependencies and snap.dependencies.job_store + and snap.dependencies.job_store.available == false + and snap.pending and snap.pending.runtime + and snap.pending.runtime.dependency == 'job_store' + end, { timeout = 0.7, interval = 0.01 }), 'update should wait for delayed control-store') + + local cap_scope = assert(root_scope:child()) + local net_calls = {} + bind_network_config_apply(cap_scope, bus, net_calls) + bind_fake_control_store(cap_scope, bus, {}) + + local net_running = probe.wait_versioned_until('net applies after delayed network-config appears', + function () return net_view:version() end, + function (seen) return net_view:changed_op(seen) end, + function () + local msg = net_view:get(net_topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'running' + and payload.apply and payload.apply.state == 'applied' + and payload.dependencies.network_config.available == true + and payload or nil + end, + { timeout = 0.8 }) + assert_eq(#net_calls, 1) + assert_eq(net_calls[1].intent.rev, 100) + assert_eq(net_running.pending and net_running.pending.network_apply, nil) + + assert_true(probe.wait_until(function () + local status = caller:call(update_topics.update_manager_rpc('status'), {}, { timeout = 0.05 }) + local snap = status and status.snapshot + return snap and snap.state == 'running' + and snap.dependencies and snap.dependencies.job_store + and snap.dependencies.job_store.available == true + and (not snap.pending or snap.pending.runtime == nil) + end, { timeout = 0.8, interval = 0.01 }), 'update should start after delayed control-store appears') + + net_view:close() + net_child:cancel('test complete') + update_child:cancel('test complete') + cap_scope:cancel('test complete') + fibers.perform(net_child:join_op()) + fibers.perform(update_child:join_op()) + fibers.perform(cap_scope:join_op()) + end) +end + +return tests diff --git a/tests/integration/devhost/support/http_hostile_tcp_child.lua b/tests/integration/devhost/support/http_hostile_tcp_child.lua new file mode 100644 index 00000000..432ab0b2 --- /dev/null +++ b/tests/integration/devhost/support/http_hostile_tcp_child.lua @@ -0,0 +1,329 @@ +-- tests/integration/devhost/support/http_hostile_tcp_child.lua +-- Child-process payload for the hostile HTTP regression. It deliberately uses a +-- raw TCP upstream so cap/http/main's lua-http client sees incomplete and hostile +-- responses while a service-owned listener remains active in the same backend. + +local function add_path(prefix) + package.path = prefix .. '?.lua;' .. prefix .. '?/init.lua;' .. package.path +end + +package.path = '../src/?.lua;' .. package.path +package.path = '../?.lua;../?/init.lua;./?.lua;./?/init.lua;' .. package.path +add_path('../vendor/lua-fibers/src/') +add_path('../vendor/lua-bus/src/') +add_path('../vendor/lua-trie/src/') +add_path('./') + +local stdlib_ok, stdlib = pcall(require, 'posix.stdlib') +if stdlib_ok and stdlib and stdlib.setenv then + stdlib.setenv('CONFIG_TARGET', 'services', true) +end + +local fibers = require 'fibers' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local socket = require 'fibers.io.socket' +local bus = require 'bus' + +local http_headers = require 'http.headers' +local http_service = require 'services.http.service' +local sdk_mod = require 'services.http.sdk' +local blob_source = require 'devicecode.blob_source' + +local LOG = os.getenv('HTTP_HOSTILE_CHILD_LOG') or '/tmp/devicecode-http-hostile-child.log' +local T0 = os.clock() + +local function log(msg) + local f = io.open(LOG, 'a') + if f then + f:write(('[%.6f] %s\n'):format(os.clock() - T0, tostring(msg))) + f:close() + end +end + +local function fail(msg) + log('child:fail ' .. tostring(msg)) + error(msg, 0) +end +local function ok(v, msg) if not v then fail(msg or 'assertion failed') end return v end +local function eq(a, b, msg) if a ~= b then fail((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end + +local function env_num(name, default) + local v = tonumber(os.getenv(name) or '') + if v == nil then return default end + return v +end + +local function parse_modes() + local s = os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_MODES') or 'partial_body,stall_no_response' + local modes = {} + for part in s:gmatch('[^,]+') do + part = part:gsub('^%s+', ''):gsub('%s+$', '') + if part ~= '' then modes[#modes + 1] = part end + end + if #modes == 0 then modes[1] = 'partial_body' end + return modes +end + +local function response_headers(status, content_type) + local h = http_headers.new() + h:append(':status', tostring(status or 200)) + if content_type then h:append('content-type', content_type) end + return h +end + +local function wait_until(predicate, timeout_s, label) + local deadline = os.clock() + (timeout_s or 2.0) + while true do + local v = predicate() + if v then return v end + if os.clock() >= deadline then fail('timed out waiting for ' .. (label or 'condition')) end + fibers.perform(sleep.sleep_op(0.005)) + end +end + +local function write_all_op(stream, data) + return fibers.run_scope_op(function () + local off = 1 + while off <= #data do + local n, err = fibers.perform(stream:write_op(data:sub(off))) + if n == nil then return nil, err or 'write_failed' end + if n <= 0 then return nil, 'zero_length_write' end + off = off + n + end + return true, nil + end):wrap(function (st, _rep, okv, err) + if st == 'ok' then return okv, err end + return nil, okv or st + end) +end + +local function perform_with_timeout(label, operation, timeout_s) + local which, a, b = fibers.perform(op.named_choice({ + value = operation, + timeout = sleep.sleep_op(timeout_s or 1.0), + })) + if which == 'timeout' then return nil, label .. '_timeout' end + return a, b +end + +local function bind_inet_with_retry() + local base = env_num('HTTP_METRICS_TIMEOUT_HOSTILE_PORT_BASE', 39000 + (os.time() % 20000)) + for i = 0, 200 do + local port = base + i + if port > 65000 then port = 39000 + (i % 20000) end + local s = socket.listen_inet('127.0.0.1', port) + if s then return s, port end + end + return nil, 'no_free_port' +end + +local function read_request_head(stream) + local buf = '' + while not buf:find('\r\n\r\n', 1, true) and #buf < 32768 do + local chunk, err = perform_with_timeout('read_request_head', stream:read_some_op(512), 1.0) + if chunk == nil then return nil, err or 'closed' end + buf = buf .. chunk + end + return buf, nil +end + +local function parse_path(req) + return req and req:match('^[A-Z]+%s+([^%s]+)') or '/' +end + +local function parse_index(path) + return tonumber(tostring(path or ''):match('(%d+)$')) +end + +local function start_hostile_server(scope, records, auto_release_s) + local server, port = bind_inet_with_retry() + ok(server, port or 'bind_failed') + ok(scope:spawn(function () + while true do + local stream = fibers.perform(server:accept_op()) + if not stream then return end + ok(scope:spawn(function () + local req = read_request_head(stream) + local path = parse_path(req) + local idx = parse_index(path) + local rec = idx and records[idx] or nil + if rec then + rec.path = path + rec.seen = true + end + local mode = rec and rec.mode or 'success' + if mode == 'partial_status' then + fibers.perform(write_all_op(stream, 'HTTP/1.1 202')) + elseif mode == 'partial_headers' then + fibers.perform(write_all_op(stream, 'HTTP/1.1 202 Accepted\r\nContent-Length: 100')) + elseif mode == 'partial_body' then + fibers.perform(write_all_op(stream, 'HTTP/1.1 202 Accepted\r\nContent-Length: 100\r\nConnection: close\r\n\r\nshort')) + elseif mode == 'success' then + fibers.perform(write_all_op(stream, 'HTTP/1.1 202 Accepted\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok')) + else + -- stall_no_response: the server has seen the request and intentionally + -- gives the client no response bytes. + end + fibers.perform(sleep.sleep_op(auto_release_s or 0.1)) + if rec then rec.auto_released = true end + if stream and type(stream.terminate) == 'function' then stream:terminate('hostile_done') end + end)) + end + end)) + return server, port +end + +local function start_cap_service(opts) + opts = opts or {} + local b = bus.new() + local svc_conn = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(svc_conn, { + id = 'main', + backend_timeout = opts.backend_timeout or 2, + connection_setup_timeout = opts.connection_setup_timeout or 2, + intra_stream_timeout = opts.intra_stream_timeout or 2, + max_accept_queue = 32, + })) + local user_conn = b:connect({ origin_base = { kind = 'local' } }) + return b, svc, sdk_mod.new_ref(user_conn, 'main') +end + +local function http_uri(port, path) + return ('http://127.0.0.1:%d%s'):format(port, path or '/') +end + +local function assert_http_backend_ready(svc, label) + local st = svc:stats() + if st.backend ~= 'ready' then + fail((label or 'http backend check') .. ': backend=' .. tostring(st.backend) .. ' last_error=' .. tostring(st.last_error or st.reason)) + end + if tostring(st.last_error or ''):find('Bad file descriptor', 1, true) then + fail((label or 'http backend check') .. ': EBADF reported: ' .. tostring(st.last_error)) + end + return st +end + +local function run_ui_raw_probe(scope, listener, port, i) + log(('iter:%03d ui_probe_spawn_handler'):format(i)) + local handled = { accepted = false, wrote_body = false } + ok(scope:spawn(function () + log(('iter:%03d ui_handler_accept_wait'):format(i)) + local ctx, aerr = fibers.perform(listener:accept_op()) + if not ctx then fail('ui accept failed: ' .. tostring(aerr)) end + handled.accepted = true + log(('iter:%03d ui_handler_accept_done'):format(i)) + local req_headers = ok(fibers.perform(ctx:get_headers_op())) + log(('iter:%03d ui_handler_headers_done path=%s'):format(i, tostring(req_headers:get(':path')))) + ok(fibers.perform(ctx:write_headers_op(response_headers(200, 'text/plain')))) + log(('iter:%03d ui_handler_write_headers_done'):format(i)) + ok(fibers.perform(ctx:write_body_from_string_op(('listener-still-alive-hostile-%d'):format(i)))) + handled.wrote_body = true + log(('iter:%03d ui_handler_write_body_done'):format(i)) + end)) + + log(('iter:%03d ui_raw_client_begin'):format(i)) + local stream = ok(socket.connect_inet('127.0.0.1', port), 'raw UI client connect failed') + local request = ('GET /ui-hostile/%03d HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n'):format(i) + ok(fibers.perform(write_all_op(stream, request)), 'raw UI client write failed') + local buf = '' + while not buf:find('\r\n\r\n', 1, true) and #buf < 8192 do + local chunk, err = perform_with_timeout('ui_raw_read', stream:read_some_op(512), 1.0) + if chunk == nil then fail('raw UI client read failed: ' .. tostring(err)) end + buf = buf .. chunk + end + if stream and type(stream.terminate) == 'function' then stream:terminate('ui_probe_done') end + local status = buf:match('^HTTP/%d%.%d%s+(%d+)') + log(('iter:%03d ui_raw_client_done status=%s body_len=%d'):format(i, tostring(status), math.max(0, #buf - (buf:find('\r\n\r\n', 1, true) or #buf)))) + eq(status, '200', 'UI raw probe status') + wait_until(function () return handled.accepted and handled.wrote_body end, 1.0, 'UI handler completion') +end + +local M = {} + +local function run_child() + log('child:start') + local iterations = math.floor(env_num('HTTP_METRICS_TIMEOUT_HOSTILE_ITERATIONS', 4)) + local attempts_timeout = env_num('HTTP_METRICS_TIMEOUT_HOSTILE_TIMEOUT_S', 0.05) + local backend_timeout = env_num('HTTP_METRICS_TIMEOUT_HOSTILE_BACKEND_TIMEOUT_S', 0.10) + local intra_stream_timeout = env_num('HTTP_METRICS_TIMEOUT_HOSTILE_INTRA_STREAM_TIMEOUT_S', 0.10) + local auto_release_s = env_num('HTTP_METRICS_TIMEOUT_HOSTILE_AUTO_RELEASE_S', 0.10) + local ui_every = math.floor(env_num('HTTP_METRICS_TIMEOUT_HOSTILE_UI_EVERY', 3)) + local modes = parse_modes() + + fibers.run(function (scope) + local records = {} + local hostile_server, hostile_port = start_hostile_server(scope, records, auto_release_s) + local _, svc, ref = start_cap_service({ backend_timeout = backend_timeout, intra_stream_timeout = intra_stream_timeout }) + local rep = ok(fibers.perform(ref:listen_op({ host = '127.0.0.1', port = 0, tls = false }, { timeout = 3 }))) + local ui_listener = ok(rep.listener) + local _, _, ui_port = ui_listener:localname() + log(('child:configured iterations=%d caller_timeout_s=%s backend_timeout_s=%s intra_stream_timeout_s=%s auto_release_s=%s ui_every=%d modes=%s hostile_port=%d ui_port=%d'):format( + iterations, tostring(attempts_timeout), tostring(backend_timeout), tostring(intra_stream_timeout), tostring(auto_release_s), ui_every, table.concat(modes, ','), hostile_port, ui_port)) + + for i = 1, iterations do + local mode = modes[((i - 1) % #modes) + 1] + records[i] = { mode = mode } + log(('iter:%03d begin mode=%s'):format(i, mode)) + log(('iter:%03d before_exchange_op_build'):format(i)) + local response_op = ref:exchange_op({ + uri = http_uri(hostile_port, ('/mainflux-hostile/%03d'):format(i)), + method = 'POST', + headers = { + ['content-type'] = 'application/senml+json', + ['authorization'] = 'Bearer hostile-regression', + }, + body_source = blob_source.from_string('[{"n":"x","v":1}]'), + }) + log(('iter:%03d response_op_built'):format(i)) + log(('iter:%03d before_exchange_choice_perform'):format(i)) + local which, result, err = fibers.perform(op.named_choice({ + response = response_op, + timeout = sleep.sleep_op(attempts_timeout), + })) + log(('iter:%03d exchange_choice_returned which=%s result=%s err=%s'):format(i, tostring(which), tostring(result), tostring(err))) + eq(which, 'timeout', 'caller timeout should win for hostile exchange') + log(('iter:%03d wait_rec_seen'):format(i)) + wait_until(function () return records[i].seen end, 1.0, 'hostile peer to see request ' .. tostring(i)) + log(('iter:%03d rec_seen path=%s'):format(i, tostring(records[i].path))) + records[i].released = true + log(('iter:%03d rec_released done=%s auto_released=%s'):format(i, tostring(records[i].done), tostring(records[i].auto_released))) + assert_http_backend_ready(svc, ('iter:%03d post_exchange_backend_ready'):format(i)) + + if ui_every > 0 and (i % ui_every) == 0 then + log(('iter:%03d ui_probe_begin client=raw'):format(i)) + run_ui_raw_probe(scope, ui_listener, ui_port, i) + assert_http_backend_ready(svc, ('iter:%03d post_ui_probe_backend_ready'):format(i)) + end + end + + log('child:post_loop_checks') + assert_http_backend_ready(svc, 'post_loop_backend_ready') + local final_i = iterations + 1 + records[final_i] = { mode = 'success' } + log('child:final_retry_begin') + local final = ok(fibers.perform(ref:exchange_op({ + uri = http_uri(hostile_port, ('/mainflux-hostile/%03d'):format(final_i)), + method = 'POST', + headers = { ['content-type'] = 'application/senml+json' }, + body_source = blob_source.from_string('[{"n":"final","v":1}]'), + }))) + eq(final.result.status, '202', 'final retry status') + assert_http_backend_ready(svc, 'after_final_retry_backend_ready') + wait_until(function () return (svc:stats().active_exchanges or 0) == 0 end, 1.0, 'active exchanges to drain') + log('child:success') + io.stdout:flush() + io.stderr:flush() + os.exit(0) + end) +end + +function M.run() + return run_child() +end + +if ... == nil then + return M.run() +end + +return M diff --git a/tests/integration/devhost/support/mcu_http_uart_child.lua b/tests/integration/devhost/support/mcu_http_uart_child.lua new file mode 100644 index 00000000..5f650ad7 --- /dev/null +++ b/tests/integration/devhost/support/mcu_http_uart_child.lua @@ -0,0 +1,532 @@ +-- tests/integration/devhost/support/mcu_http_uart_child.lua +-- +-- Separate-process fake MCU for the HTTP-over-UART devhost test. This keeps +-- the MCU bus, scheduler and HAL UART manager out of the parent test process +-- while still using the real Fabric and HAL UART open path on the MCU side. + +local function add_path(prefix) + package.path = prefix .. '?.lua;' .. prefix .. '?/init.lua;' .. package.path +end + +package.path = '../src/?.lua;' .. package.path +package.path = '../?.lua;../?/init.lua;./?.lua;./?/init.lua;' .. package.path +add_path('../vendor/lua-fibers/src/') +add_path('../vendor/lua-bus/src/') +add_path('../vendor/lua-trie/src/') +add_path('./') + +local stdlib_ok, stdlib = pcall(require, 'posix.stdlib') +if stdlib_ok and stdlib and stdlib.setenv then + stdlib.setenv('CONFIG_TARGET', 'services', true) +end + +local busmod = require 'bus' +local fibers = require 'fibers' +local channel = require 'fibers.channel' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local socket = require 'fibers.io.socket' +local safe = require 'coxpcall' + +local cjson_ok, cjson = pcall(require, 'cjson.safe') +if not cjson_ok then cjson = require 'cjson' end + +local hal_types = require 'services.hal.types.core' +local cap_args = require 'services.hal.types.capability_args' +local fabric = require 'services.fabric' +local fabric_topics = require 'services.fabric.topics' +local bus_cleanup = require 'devicecode.support.bus_cleanup' +local xxhash32 = require 'shared.hash.xxhash32' + +local UART_BYTES_PER_SEC = 11520 +local REALISTIC_TRANSFER_TIMEOUT_S = tonumber(os.getenv('MCU_HTTP_UART_TRANSFER_TIMEOUT_S')) or 300.0 +local MCU_PROGRESS_LOG_BYTES = 16 * 1024 +local MCU_FLASH_WRITE_DELAY_S = tonumber(os.getenv('MCU_HTTP_UART_FLASH_DELAY_S')) or 0.010 + +local function parse_args(argv) + local out = {} + local i = 1 + while i <= #argv do + local k = argv[i] + if k == '--ipc' then i = i + 1; out.ipc = argv[i] + elseif k == '--uart' then i = i + 1; out.uart = argv[i] + elseif k == '--old-image' then i = i + 1; out.old_image_id = argv[i] + elseif k == '--committed-image' then i = i + 1; out.committed_image_id = argv[i] ~= '' and argv[i] or nil + elseif k == '--boot-seq' then i = i + 1; out.boot_seq = tonumber(argv[i]) or 1 + else error('unknown argument: ' .. tostring(k), 0) end + i = i + 1 + end + assert(out.ipc and out.ipc ~= '', '--ipc required') + assert(out.uart and out.uart ~= '', '--uart required') + out.old_image_id = out.old_image_id or 'mcu-image-old' + return out +end + +local function assert_true(v, msg) if v ~= true then error(msg or ('expected true, got ' .. tostring(v)), 0) end end +local function assert_not_nil(v, msg) if v == nil then error(msg or 'expected non-nil', 0) end end +local function assert_eq(a, b, msg) if a ~= b then error(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a)), 0) end end + +local function dummy_logger() + local logger = {} + for _, k in ipairs({ 'debug', 'info', 'warn', 'error' }) do logger[k] = function () end end + function logger:child() return self end + return logger +end + +local function wait_channel_get(ch, timeout_s, what) + local which, a, b = fibers.perform(op.named_choice({ + item = ch:get_op(), + timeout = sleep.sleep_op(timeout_s or 1.0), + })) + if which == 'timeout' then error(('timed out waiting for %s'):format(what or 'channel item'), 0) end + if a == nil then error(('channel closed while waiting for %s: %s'):format(what or 'channel item', tostring(b)), 0) end + return a +end + +local function wait_device_event(dev_ev_ch, event_type, class, id, timeout_s) + local deadline = fibers.now() + (timeout_s or 1.5) + while fibers.now() < deadline do + local ev = wait_channel_get(dev_ev_ch, deadline - fibers.now(), 'UART device event') + if ev.event_type == event_type and ev.class == class and ev.id == id then return ev end + end + error(('timed out waiting for UART device event %s %s/%s'):format(tostring(event_type), tostring(class), tostring(id)), 0) +end + +local function wait_uart_cap(dev_ev_ch) + local added = wait_device_event(dev_ev_ch, 'added', 'uart', 'uart0', 1.5) + assert_true(type(added.capabilities) == 'table' and #added.capabilities == 1, 'UART added event missing capability') + local cap = added.capabilities[1] + assert_eq(cap.class, 'uart') + assert_eq(cap.id, 'uart0') + assert_true(type(cap.control_ch) == 'table', 'UART capability should expose control_ch') + return cap +end + +local function normalise_uart_open_opts(opts) + if opts == nil or getmetatable(opts) ~= cap_args.UARTOpenOpts then + local open_opts, err = cap_args.new.UARTOpenOpts(opts) + assert_not_nil(open_opts, tostring(err)) + return open_opts + end + return opts +end + +local function call_hal_control(cap, verb, opts) + local reply_ch = channel.new(1) + local req, err = hal_types.new.ControlRequest(verb, opts or {}, reply_ch) + assert_not_nil(req, tostring(err)) + fibers.perform(cap.control_ch:put_op(req)) + local reply = wait_channel_get(reply_ch, 1.0, 'HAL UART control reply') + assert_true(type(reply) == 'table', 'HAL control reply must be a table') + return reply +end + +local function expose_raw_host_uart_open(scope, bus, cap, source) + source = source or 'uart_manager' + local conn = bus:connect({ origin_base = { service = 'mcu-child-hal-uart-adapter' } }) + local cap_id = cap.id + local ep = conn:bind({ 'raw', 'host', source, 'cap', 'uart', cap_id, 'rpc', 'open' }, { queue_len = 8 }) + + conn:retain({ 'raw', 'host', source, 'status' }, { state = 'available', available = true, source = source, class = 'uart', id = cap_id }) + conn:retain({ 'raw', 'host', source, 'meta' }, { source = source, class = 'uart', id = cap_id }) + conn:retain({ 'raw', 'host', source, 'cap', 'uart', cap_id, 'status' }, { state = 'available', available = true, source_kind = 'host', source = source }) + conn:retain({ 'raw', 'host', source, 'cap', 'uart', cap_id, 'meta' }, { source_kind = 'host', source = source, offerings = { open = true } }) + + scope:finally(function () + safe.pcall(function () ep:unbind() end) + bus_cleanup.disconnect(conn) + end) + + assert_true(scope:spawn(function () + while true do + local req = ep:recv() + if req == nil then return end + local open_opts = normalise_uart_open_opts(req.payload) + local reply = call_hal_control(cap, 'open', open_opts) + local replied = req:reply(reply) + if not replied and reply.ok == true and type(reply.reason) == 'table' + and type(reply.reason.session) == 'table' + and type(reply.reason.session.terminate) == 'function' + then + reply.reason.session:terminate('fabric request abandoned') + end + end + end)) + + return { source = source, class = 'uart', id = cap_id } +end + +local function fresh_uart_manager() + package.loaded['services.hal.managers.uart'] = nil + package.loaded['services.hal.drivers.uart'] = nil + return require 'services.hal.managers.uart' +end + +local function start_uart_manager(scope, bus, uart_slave) + local uart_mgr = fresh_uart_manager() + local dev_ev_ch = channel.new(16) + local cap_emit_ch = channel.new(32) + local ok_start, start_err = fibers.perform(uart_mgr.start_op(dummy_logger(), dev_ev_ch, cap_emit_ch)) + assert_true(ok_start, tostring(start_err)) + scope:finally(function () safe.pcall(function () fibers.perform(uart_mgr.shutdown_op()) end) end) + + local ok_cfg, cfg_err = fibers.perform(uart_mgr.apply_config_op({ + serial_ports = { + { id = 'uart0', path = uart_slave, baud = 115200, mode = '8N1' }, + }, + })) + assert_true(ok_cfg, tostring(cfg_err)) + local cap = wait_uart_cap(dev_ev_ch) + return expose_raw_host_uart_open(scope, bus, cap, 'uart_manager') +end + + +local function fabric_payload_snapshot(payload) + if type(payload) ~= 'table' then return nil end + return type(payload.snapshot) == 'table' and payload.snapshot or payload +end + +local function compact_fabric_session(s) + s = type(s) == 'table' and s or {} + return ('phase=%s established=%s local=%s peer_node=%s peer_sid=%s gen=%s wire_errors=%s bad_frames=%s last_wire_error=%s why=%s'):format( + tostring(s.phase), tostring(s.established), tostring(s.local_node), tostring(s.peer_node), + tostring(s.peer_sid), tostring(s.session_generation), tostring(s.wire_errors or 0), + tostring(s.bad_frame_count or 0), tostring(s.last_wire_error), tostring(s.why) + ) +end + +local function compact_fabric_bridge(s) + s = type(s) == 'table' and s or {} + return ('state=%s imported=%s pending=%s inbound=%s frames_sent=%s frames_recv=%s session_peer=%s drop=%s err=%s'):format( + tostring(s.state), tostring(s.imported_topics), tostring(s.pending_calls), tostring(s.inbound_calls), + tostring(s.frames_sent), tostring(s.frames_received), + tostring(type(s.session) == 'table' and s.session.peer_sid or nil), + tostring(s.session_drop_reason), tostring(s.last_err) + ) +end + +local function compact_fabric_transfer(s) + s = type(s) == 'table' and s or {} + local stats = type(s.stats) == 'table' and s.stats or {} + local active = type(s.active) == 'table' and s.active or nil + local last = type(s.last) == 'table' and s.last or nil + return ('active=%s active_status=%s last_status=%s completed=%s failed=%s cancelled=%s stale=%s'):format( + tostring(active ~= nil), tostring(active and active.status), tostring(last and last.status), + tostring(stats.completed), tostring(stats.failed), tostring(stats.cancelled), tostring(stats.stale) + ) +end + +local function compact_fabric_link(s) + s = type(s) == 'table' and s or {} + local comps = {} + if type(s.components) == 'table' then + for name, rec in pairs(s.components) do + comps[#comps + 1] = tostring(name) .. '=' .. tostring(type(rec) == 'table' and rec.status or rec) + end + table.sort(comps) + end + return ('state=%s completed=%s/%s reason=%s components=[%s]'):format( + tostring(s.state), tostring(s.completed), tostring(s.total), tostring(s.reason), table.concat(comps, ',') + ) +end + +local function describe_fabric_status_payload(payload, component) + local s = fabric_payload_snapshot(payload) + if component == 'session' then return compact_fabric_session(s) end + if component == 'rpc_bridge' then return compact_fabric_bridge(s) end + if component == 'transfer_manager' or component == 'transfer' then return compact_fabric_transfer(s) end + return compact_fabric_link(s) +end + +local function retained_payload_now(conn, topic) + local view = conn:retained_view(topic) + local msg = view:get(topic) + view:close() + return msg and msg.payload or nil +end + +local function wait_retained_payload_where(conn, topic, label, pred, timeout_s) + local view = conn:retained_view(topic) + local deadline = fibers.now() + (timeout_s or 6.0) + local seen = view:version() + while true do + local msg = view:get(topic) + local payload = msg and msg.payload or nil + local out = pred(payload) + if out then view:close(); return out end + local remaining = deadline - fibers.now() + if remaining <= 0 then view:close(); error('timed out waiting for ' .. tostring(label), 0) end + local which, version, reason = fibers.perform(op.named_choice({ + changed = view:changed_op(seen), + timeout = sleep.sleep_op(math.min(0.50, remaining)), + })) + if which == 'changed' then + if version == nil then view:close(); error(tostring(label) .. ' closed: ' .. tostring(reason or 'closed'), 0) end + seen = version + end + end +end + +local function start_fabric_status_reporter(scope, conn, emit) + assert_true(scope:spawn(function () + local payload = wait_retained_payload_where( + conn, + fabric_topics.state_link_component('link-a', 'session'), + 'MCU fabric session established', + function (p) + local s = fabric_payload_snapshot(p) + if type(s) == 'table' and s.established == true and type(s.peer_sid) == 'string' and s.peer_sid ~= '' then + return p + end + return nil + end, + 6.0 + ) + local s = fabric_payload_snapshot(payload) or {} + emit({ + event = 'fabric_session', + phase = s.phase, + established = s.established, + local_node = s.local_node, + peer_node = s.peer_node, + peer_sid = s.peer_sid, + session_generation = s.session_generation, + wire_errors = s.wire_errors or 0, + bad_frame_count = s.bad_frame_count or 0, + last_wire_error = s.last_wire_error, + summary = describe_fabric_status_payload(payload, 'session'), + }) + local items = { + { label = 'link', topic = fabric_topics.state_link('link-a') }, + { label = 'session', component = 'session', topic = fabric_topics.state_link_component('link-a', 'session') }, + { label = 'rpc_bridge', component = 'rpc_bridge', topic = fabric_topics.state_link_component('link-a', 'rpc_bridge') }, + { label = 'transfer_manager', component = 'transfer_manager', topic = fabric_topics.state_link_component('link-a', 'transfer_manager') }, + } + for _, item in ipairs(items) do + local p = retained_payload_now(conn, item.topic) + emit({ event = 'fabric_status', component = item.label, summary = describe_fabric_status_payload(p, item.component) }) + end + end)) +end + +local function fabric_config(local_node, peer_node, bridge, transfer) + return { + schema = fabric.config.SCHEMA, + local_node = local_node, + links = { + { + id = 'link-a', + peer_id = peer_node, + transport = { source = 'uart_manager', class = 'uart', id = 'uart0', terminator = '\n' }, + session = { hello_interval_s = 0.20, ping_interval_s = 5.0, liveness_timeout_s = 5.0 }, + bridge = bridge or {}, + transfer = transfer or { chunk_size = 2048, timeout_s = REALISTIC_TRANSFER_TIMEOUT_S }, + }, + }, + } +end + +local function mcu_fabric_config() + return fabric_config('mcu', 'cm5', { + exports = { + { id = 'mcu-state-export', ['local'] = { 'state', 'self' }, remote = { 'state', 'self' }, publish = true, retain = true }, + { id = 'mcu-cap-export', ['local'] = { 'cap', 'self' }, remote = { 'cap', 'self' }, publish = true, retain = true }, + }, + rpc = { + inbound = { + { id = 'mcu-prepare-in', ['local'] = { 'cap', 'self', 'updater', 'main', 'rpc', 'prepare-update' }, remote = { 'cap', 'self', 'updater', 'main', 'rpc', 'prepare-update' }, timeout_s = 2.0 }, + { id = 'mcu-commit-in', ['local'] = { 'cap', 'self', 'updater', 'main', 'rpc', 'commit-update' }, remote = { 'cap', 'self', 'updater', 'main', 'rpc', 'commit-update' }, timeout_s = 2.0 }, + }, + }, + }, { chunk_size = 2048, timeout_s = REALISTIC_TRANSFER_TIMEOUT_S }) +end + +local function start_public_fabric(scope, conn, cfg, opts) + opts = opts or {} + assert_true(scope:spawn(function () + fabric.start(conn, { + name = opts.name or 'fabric-mcu-child', + env = 'test', + config = cfg, + link_overrides = opts.link_overrides, + }) + end)) +end + +local function publish_mcu_facts(conn, fake) + local image = fake.committed_image_id or fake.old_image_id + local boot = 'mcu-boot-' .. tostring(fake.boot_seq) + conn:retain({ 'state', 'self', 'software' }, { image_id = image, boot_id = boot, version = image }) + conn:retain({ 'state', 'self', 'updater' }, { + state = 'ready', + last_error = nil, + staged_image_id = fake.staged and fake.staged.image_id or nil, + pending_image_id = fake.committed_image_id, + job_id = fake.job_id, + }) + conn:retain({ 'cap', 'self', 'updater', 'main', 'meta' }, { class = 'updater', id = 'main', methods = { 'prepare-update', 'commit-update' } }) + conn:retain({ 'cap', 'self', 'updater', 'main', 'status' }, { available = true, state = 'available' }) +end + +local function new_mcu_receive_target(fake, emit) + local target = {} + function target:open_sink_op(req) + fake.transfer_begin = req + fake.receive_started_at = fibers.now() + fake.receive_bytes = 0 + fake.receive_chunks = 0 + fake.receive_next_log = MCU_PROGRESS_LOG_BYTES + assert_eq(req.target, 'updater/main') + emit({ event = 'receive_opened', target = req.target, size = req.size, job_id = req.meta and req.meta.job_id, image_id = req.meta and (req.meta.image_id or req.meta.expected_image_id) }) + local chunks = {} + local sink = {} + function sink:append_op(chunk) + return fibers.run_scope_op(function () + fibers.perform(sleep.sleep_op(MCU_FLASH_WRITE_DELAY_S)) + chunks[#chunks + 1] = chunk + fake.receive_bytes = (fake.receive_bytes or 0) + #(chunk or '') + fake.receive_chunks = (fake.receive_chunks or 0) + 1 + if fake.receive_bytes >= (fake.receive_next_log or MCU_PROGRESS_LOG_BYTES) then + local elapsed = fibers.now() - (fake.receive_started_at or fibers.now()) + emit({ event = 'receive_progress', bytes = fake.receive_bytes, chunks = fake.receive_chunks or 0, elapsed_s = elapsed }) + fake.receive_next_log = fake.receive_bytes + MCU_PROGRESS_LOG_BYTES + else + emit({ event = 'receive_tick', bytes = fake.receive_bytes, chunks = fake.receive_chunks or 0 }) + end + return true, nil + end):wrap(function (status, _report, ok, err) + if status ~= 'ok' then return nil, err or status end + return ok, err + end) + end + function sink:commit_op(req2) + local bytes = table.concat(chunks) + local payload_digest = xxhash32.digest_hex(bytes) + fake.staged = { size = #bytes, digest = req2.digest, payload_digest = payload_digest, image_id = req.meta and (req.meta.image_id or req.meta.expected_image_id), job_id = req.meta and req.meta.job_id } + fake.staged_signal = (fake.staged_signal or 0) + 1 + emit({ event = 'transfer_commit', size = #bytes, chunks = fake.receive_chunks or 0, digest = req2.digest, payload_digest = payload_digest, image_id = fake.staged.image_id, job_id = fake.staged.job_id }) + return op.always({ staged = true, digest = req2.digest }, nil) + end + function sink:abort(reason) + fake.abort_reason = reason + fake.abort_count = (fake.abort_count or 0) + 1 + emit({ event = 'transfer_abort', reason = tostring(reason), bytes = fake.receive_bytes or 0, chunks = fake.receive_chunks or 0, abort_count = fake.abort_count }) + return true, nil + end + return op.always(sink, nil) + end + return target +end + +local function start_fake_mcu(scope, bus, fake, emit) + local conn = bus:connect({ origin_base = { service = 'fake-mcu-child' } }) + publish_mcu_facts(conn, fake) + + local eps = {} + local function bind(topic) + local ep, err = bus_cleanup.bind(conn, topic, { queue_len = 8 }) + assert_not_nil(ep, err) + eps[#eps + 1] = ep + return ep + end + + local prepare_ep = bind({ 'cap', 'self', 'updater', 'main', 'rpc', 'prepare-update' }) + local commit_ep = bind({ 'cap', 'self', 'updater', 'main', 'rpc', 'commit-update' }) + + scope:finally(function () + for _, ep in ipairs(eps) do bus_cleanup.unbind(conn, ep) end + bus_cleanup.disconnect(conn) + end) + + assert_true(scope:spawn(function () + while true do + local req = fibers.perform(prepare_ep:recv_op()) + if req == nil then return end + fake.prepare_payload = req.payload + assert_eq(type(req.payload) == 'table' and req.payload.target or nil, 'mcu') + fake.job_id = type(req.payload) == 'table' and req.payload.job_id or nil + conn:retain({ 'state', 'self', 'updater' }, { state = 'ready', last_error = nil, job_id = fake.job_id }) + emit({ event = 'prepare', payload = req.payload, job_id = fake.job_id }) + req:reply({ ready = true, target = 'updater/main', max_chunk_size = 2048 }) + end + end)) + + assert_true(scope:spawn(function () + while true do + local req = fibers.perform(commit_ep:recv_op()) + if req == nil then return end + fake.commit_payload = req.payload + fake.commit_seen = true + fake.committed_image_id = (fake.staged and fake.staged.image_id) or (type(req.payload) == 'table' and req.payload.expected_image_id) + conn:retain({ 'state', 'self', 'updater' }, { + state = 'rebooting', + last_error = nil, + pending_image_id = fake.committed_image_id, + staged_image_id = fake.staged and fake.staged.image_id or nil, + job_id = fake.job_id, + }) + emit({ event = 'commit', payload = req.payload, committed_image_id = fake.committed_image_id }) + req:reply({ accepted = true, reboot_required = true }) + end + end)) + + return conn +end + +local unix_ok, unistd = pcall(require, 'posix.unistd') +local child_pid = (unix_ok and unistd and unistd.getpid and tostring(unistd.getpid())) or '' + +local function main(scope) + local args = parse_args(arg or {}) + local ipc, ierr = socket.connect_unix(args.ipc) + assert(ipc, 'IPC connect failed: ' .. tostring(ierr)) + + local events = channel.new(256) + local function emit(ev) + ev = ev or {} + ev.pid = child_pid + local ok, err = fibers.perform(events:put_op(ev)) + return ok, err + end + + scope:spawn(function () + while true do + local ev = fibers.perform(events:get_op()) + if ev == nil then return end + local line = assert(cjson.encode(ev)) .. '\n' + local _, werr = fibers.perform(ipc:write_op(line)) + if werr then return end + end + end) + + local bus = busmod.new() + local fake = { + old_image_id = args.old_image_id, + committed_image_id = args.committed_image_id, + boot_seq = args.boot_seq, + } + + start_uart_manager(scope, bus, args.uart) + start_fake_mcu(scope, bus, fake, emit) + + local conn = bus:connect({ origin_base = { service = 'fabric-mcu-child' } }) + scope:finally(function () bus_cleanup.disconnect(conn) end) + start_public_fabric(scope, conn, mcu_fabric_config(), { + name = 'fabric-mcu-child', + link_overrides = { + ['link-a'] = { + transfer = { + chunk_size = 2048, + timeout_s = REALISTIC_TRANSFER_TIMEOUT_S, + receive_targets = { ['updater/main'] = new_mcu_receive_target(fake, emit) }, + }, + }, + }, + }) + start_fabric_status_reporter(scope, conn, emit) + + emit({ event = 'ready', boot_seq = fake.boot_seq, image_id = fake.committed_image_id or fake.old_image_id, uart = args.uart }) + fibers.perform(op.never()) +end + +fibers.run(main) diff --git a/tests/integration/devhost/test_http_service_real.lua b/tests/integration/devhost/test_http_service_real.lua new file mode 100644 index 00000000..64470976 --- /dev/null +++ b/tests/integration/devhost/test_http_service_real.lua @@ -0,0 +1,451 @@ +-- Real devhost integration for cap/http/main over the local bus. +-- +-- These tests use the real cqueues/lua-http backend. They focus on the +-- capability-service seam: local handle return, listener/context ownership +-- transfer, service shutdown backstop termination, outgoing exchange, and +-- outgoing WebSocket transport. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local runtime = require 'fibers.runtime' +local bus = require 'bus' + +local http_request = require 'http.request' +local http_headers = require 'http.headers' +local http_server = require 'http.server' +local http_websocket = require 'http.websocket' + +local http_service = require 'services.http.service' +local sdk_mod = require 'services.http.sdk' +local driver_mod = require 'services.http.transport.cqueues_driver' +local lua_http = require 'services.http.transport.lua_http' +local ws_wrap = require 'services.http.transport.websocket' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +local function yield_once() + runtime.yield() +end + +local function yield_many(n) + for _ = 1, n do yield_once() end +end + +local function wait_until(predicate, timeout, label) + local deadline = runtime.now() + (timeout or 5) + while not predicate() do + if runtime.now() >= deadline then + error('timed out waiting for ' .. (label or 'condition'), 2) + end + fibers.perform(sleep.sleep_op(0.01)) + end +end + +local function response_headers(status, content_type) + local h = http_headers.new() + h:append(':status', tostring(status or 200)) + if content_type then h:append('content-type', content_type) end + return h +end + +local function http_uri(port, path) + return ('http://127.0.0.1:%d%s'):format(port, path or '/') +end + +local function ws_uri(port, path) + return ('ws://127.0.0.1:%d%s'):format(port, path or '/ws') +end + +local function memory_sink(sinks, key) + sinks[key] = sinks[key] or { chunks = {} } + local rec = sinks[key] + return { + write_chunk_op = function (_, chunk) rec.chunks[#rec.chunks + 1] = chunk; return fibers.always(true) end, + finish_op = function () rec.finished = true; return fibers.always(true) end, + terminate = function (_, reason) rec.terminated = reason or true; return true end, + } +end + +local function memory_source(sources, key) + local chunks = sources[key] or {} + local i = 0 + return { + read_chunk_op = function () + return fibers.guard(function () + i = i + 1 + return fibers.always(chunks[i], nil) + end) + end, + terminate = function (_, reason) sources[key .. '_terminated'] = reason or true; return true end, + } +end + +local function start_cap_service(opts) + opts = opts or {} + local memory_sinks = opts.memory_sinks or {} + local b = bus.new() + local svc_conn = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(svc_conn, { + id = opts.id or 'main', + backend_timeout = opts.backend_timeout or 2, + connection_setup_timeout = opts.connection_setup_timeout or 2, + intra_stream_timeout = opts.intra_stream_timeout or 2, + max_accept_queue = opts.max_accept_queue or 32, + max_event_history = opts.max_event_history or 32, + })) + + local user_conn = b:connect({ origin_base = { kind = 'local' } }) + local ref = sdk_mod.new_ref(user_conn, opts.id or 'main') + return b, svc, ref, memory_sinks +end + +local function new_backend_listener(opts) + opts = opts or {} + local driver = ok(driver_mod.new { label = opts.label or 'http-service-real-backend' }) + local listener = ok(lua_http.listen { + driver = driver, + http_server = http_server, + host = '127.0.0.1', + port = 0, + tls = false, + max_accept_queue = opts.max_accept_queue or 32, + connection_setup_timeout = opts.connection_setup_timeout or 2, + intra_stream_timeout = opts.intra_stream_timeout or 2, + }) + return listener, driver +end + +local function start_backend_listener(scope, opts) + local listener, driver = new_backend_listener(opts) + ok(driver:start(scope), 'backend driver should start') + ok(listener:start(scope), 'backend listener should start') + ok(fibers.perform(listener:listen_op()), 'backend listener should bind') + local _, _, port = listener:localname() + ok(type(port) == 'number' and port > 0, 'backend listener should bind an ephemeral port') + return listener, driver, port +end + +local function run_http_client(driver, uri, body, method) + return fibers.perform(driver:run_op('real-http-service-test-client', function () + local req = http_request.new_from_uri(uri) + if method then req.headers:upsert(':method', method) end + if body ~= nil then + req.headers:upsert(':method', method or 'POST') + req:set_body(body) + end + + local headers, stream = assert(req:go(2)) + local response_body = assert(stream:get_body_as_string(2)) + return headers:get(':status'), response_body + end)) +end + +function M.test_real_http_service_listen_returns_local_listener_handle() + fibers.run(function () + local _, svc, ref = start_cap_service() + + local rep = ok(fibers.perform(ref:listen_op({ + host = '127.0.0.1', + port = 0, + tls = false, + }, { timeout = 3 }))) + local listener = ok(rep.listener, 'listen should return a local listener handle') + local _, _, port = listener:localname() + ok(type(port) == 'number' and port > 0, 'listener should bind an ephemeral port') + + ok(fibers.spawn(function () + local ctx = ok(fibers.perform(listener:accept_op())) + local req_headers = ok(fibers.perform(ctx:get_headers_op())) + eq(req_headers:get(':path'), '/cap-test') + ok(fibers.perform(ctx:write_headers_op(response_headers(200, 'text/plain')))) + ok(fibers.perform(ctx:write_body_from_string_op('served-by-cap-http'))) + end)) + + local status, body = run_http_client(svc._driver, http_uri(port, '/cap-test')) + eq(status, '200') + eq(body, 'served-by-cap-http') + yield_many(8) + local saw_context = false + for _, ev in ipairs(svc:events()) do + if ev.kind == 'context_admitted' then + saw_context = true + ok(ev.listener_id ~= nil, 'context_admitted should carry listener identity') + ok(ev.context_id ~= nil, 'context_admitted should carry context identity') + end + end + ok(saw_context, 'expected context_admitted event') + + svc:terminate('done') + end) +end + +function M.test_real_service_shutdown_terminates_live_listener_and_wakes_accept_op() + fibers.run(function (scope) + local _, svc, ref = start_cap_service() + + local rep = ok(fibers.perform(ref:listen_op({ host = '127.0.0.1', port = 0, tls = false }, { timeout = 3 }))) + local listener = ok(rep.listener) + local accepted, accept_err + + ok(scope:spawn(function () + accepted, accept_err = fibers.perform(listener:accept_op()) + end)) + + yield_many(4) + svc:terminate('service_shutdown') + yield_many(8) + + eq(accepted, nil) + eq(accept_err, 'service_shutdown') + ok(listener:is_closed(), 'service shutdown should terminate the live listener handle') + end) +end + +function M.test_real_accepted_context_survives_listener_close_after_ownership_transfer() + fibers.run(function (scope) + local _, svc, ref = start_cap_service() + + local rep = ok(fibers.perform(ref:listen_op({ host = '127.0.0.1', port = 0, tls = false }, { timeout = 3 }))) + local listener = ok(rep.listener) + local _, _, port = listener:localname() + local served = false + + ok(scope:spawn(function () + local ctx = ok(fibers.perform(listener:accept_op())) + local req_headers = ok(fibers.perform(ctx:get_headers_op())) + eq(req_headers:get(':path'), '/accepted-transfer') + + listener:terminate('listener_closed') + ok(not ctx:is_closed(), 'accepted context should no longer be listener-owned') + + ok(fibers.perform(ctx:write_headers_op(response_headers(200, 'text/plain')))) + ok(fibers.perform(ctx:write_body_from_string_op('context-still-owned-by-request'))) + served = true + end)) + + local status, body = run_http_client(svc._driver, http_uri(port, '/accepted-transfer')) + eq(status, '200') + eq(body, 'context-still-owned-by-request') + ok(served, 'request scope should finish response after listener close') + ok(listener:is_closed(), 'listener should be closed') + + svc:terminate('done') + end) +end + +function M.test_real_cap_exchange_get_and_post_round_trip_to_local_server() + fibers.run(function (scope) + local listener, driver, port = start_backend_listener(scope, { label = 'http-service-real-exchange-backend' }) + local sources = { post = { 'upload-', 'payload' } } + local _, svc, ref, sinks = start_cap_service() + local handled = 0 + + for _ = 1, 2 do + ok(scope:spawn(function () + local ctx = ok(fibers.perform(listener:accept_op())) + local req_headers = ok(fibers.perform(ctx:get_headers_op())) + local method = req_headers:get(':method') or 'GET' + local path = req_headers:get(':path') or '/' + local payload = '' + if method == 'POST' then + payload = ':' .. ok(fibers.perform(ctx:read_body_as_string_op())) + end + handled = handled + 1 + ok(fibers.perform(ctx:write_headers_op(response_headers(200, 'text/plain')))) + ok(fibers.perform(ctx:write_body_from_string_op(method .. ':' .. path .. payload))) + end)) + end + + local get_rep = ok(fibers.perform(ref:exchange_op({ + uri = http_uri(port, '/out-get'), + method = 'GET', + headers = { ['x-test'] = 'get' }, + response_sink = memory_sink(sinks, 'get'), + }, { timeout = 3 }))) + eq(get_rep.result.status, '200') + eq(table.concat(sinks.get.chunks), 'GET:/out-get') + + local post_rep = ok(fibers.perform(ref:exchange_op({ + uri = http_uri(port, '/out-post'), + method = 'POST', + headers = { ['x-test'] = 'post' }, + body_source = memory_source(sources, 'post'), + response_sink = memory_sink(sinks, 'post'), + }, { timeout = 3 }))) + eq(post_rep.result.status, '200') + eq(table.concat(sinks.post.chunks), 'POST:/out-post:upload-payload') + eq(handled, 2) + + svc:terminate('done') + listener:terminate('done') + driver:terminate('done') + end) +end + +function M.test_real_open_exchange_handle_is_terminated_by_service_shutdown_backstop() + fibers.run(function (scope) + local listener, driver, port = start_backend_listener(scope, { label = 'http-service-real-open-exchange-backend' }) + local _, svc, ref = start_cap_service() + local server_ready = false + local release_server = false + local server_done = false + + ok(scope:spawn(function () + local ctx = ok(fibers.perform(listener:accept_op())) + ok(fibers.perform(ctx:get_headers_op())) + ok(fibers.perform(ctx:write_headers_op(response_headers(200, 'text/plain')))) + server_ready = true + + -- Keep the response stream open while the returned exchange handle is + -- handed off to the caller and then terminated by the HTTP service + -- shutdown backstop. Do not leave an independent sleeper behind after + -- the assertion: fibers.run would otherwise wait for it at scope exit. + while not release_server do + fibers.perform(sleep.sleep_op(0.01)) + end + + ctx:terminate('server_done') + server_done = true + end)) + + local rep = ok(fibers.perform(ref:open_exchange_op({ uri = http_uri(port, '/held-open'), method = 'GET' }, { timeout = 3 }))) + local exchange = ok(rep.exchange, 'open_exchange should return a local exchange handle') + eq(exchange:status(), '200') + ok(server_ready, 'server should have written response headers') + ok(not exchange:is_closed(), 'returned exchange should survive setup operation handoff') + + svc:terminate('service_shutdown') + ok(exchange:is_closed(), 'service shutdown should terminate handed-off exchange as backend backstop') + eq(exchange:why(), 'service_shutdown') + + release_server = true + wait_until(function () return server_done end, 1, 'held-open server handler to finish') + + listener:terminate('done') + driver:terminate('done') + end) +end + +function M.test_real_cap_connect_ws_send_receive_and_close_round_trip() + fibers.run(function (scope) + local listener, driver, port = start_backend_listener(scope, { label = 'http-service-real-ws-backend' }) + local _, svc, ref = start_cap_service() + local server_done = false + + ok(scope:spawn(function () + local ctx = ok(fibers.perform(listener:accept_op())) + local req_headers = ok(fibers.perform(ctx:get_headers_op())) + local ws = ok(fibers.perform(ws_wrap.from_context_op(ctx, req_headers, { + websocket_module = http_websocket, + }))) + ok(fibers.perform(ws:accept_op({}))) + + local data, opcode = fibers.perform(ws:receive_op()) + ok(data, 'server websocket should receive data') + eq(opcode, 'text') + ok(fibers.perform(ws:send_op(data .. ':via-cap-http', 'text'))) + fibers.perform(ws:close_op(1000, 'server_done')) + server_done = true + end)) + + local rep = ok(fibers.perform(ref:connect_ws_op({ uri = ws_uri(port, '/ws') }, { timeout = 3 }))) + local ws = ok(rep.websocket, 'connect_ws should return a local WebSocket handle') + ok(fibers.perform(ws:send_op('ping', 'text'))) + local data, opcode = fibers.perform(ws:receive_op()) + eq(data, 'ping:via-cap-http') + eq(opcode, 'text') + fibers.perform(ws:close_op(1000, 'client_done')) + + wait_until(function () return server_done end, 5, 'websocket server close') + + svc:terminate('done') + listener:terminate('done') + driver:terminate('done') + end) +end + + +local function hostile_shquote(s) + s = tostring(s or '') + return "'" .. s:gsub("'", "'\\''") .. "'" +end + +local function hostile_append_file(path, text) + local f = assert(io.open(path, 'a')) + f:write(text) + f:close() +end + +local function hostile_read_file(path) + local f = io.open(path, 'r') + if not f then return '' end + local data = f:read('*a') or '' + f:close() + return data +end + +local function hostile_tail_lines(text, n) + n = tonumber(n) or 120 + local lines = {} + for line in tostring(text or ''):gmatch('[^\n]+') do lines[#lines + 1] = line end + local first = math.max(1, #lines - n + 1) + local out = {} + for i = first, #lines do out[#out + 1] = lines[i] end + return table.concat(out, '\n') +end + +local function hostile_shell_exit_status(a, b, c) + if type(a) == 'number' then + if a >= 256 then return math.floor(a / 256) end + return a + end + if a == true then return 0 end + if b == 'exit' and type(c) == 'number' then return c end + if type(c) == 'number' then return c end + return 1 +end + +function M.test_real_metrics_style_exchange_timeout_regression_hostile_tcp_keeps_backend_and_listener_ready() + local log_path = os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_LOG') or '/tmp/devicecode-http-hostile-child.log' + local attempts = tonumber(os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_CHILD_ATTEMPTS') or '') or 3 + local timeout_s = tonumber(os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_PARENT_TIMEOUT_S') or '') or 8 + local tail_n = tonumber(os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_TAIL_LINES') or '') or 120 + os.remove(log_path) + + local env = { + HTTP_HOSTILE_CHILD_LOG = log_path, + HTTP_METRICS_TIMEOUT_HOSTILE_MODES = os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_MODES') or 'partial_body,stall_no_response', + HTTP_METRICS_TIMEOUT_HOSTILE_ITERATIONS = os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_ITERATIONS') or '4', + HTTP_METRICS_TIMEOUT_HOSTILE_TIMEOUT_S = os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_TIMEOUT_S') or '0.05', + HTTP_METRICS_TIMEOUT_HOSTILE_BACKEND_TIMEOUT_S = os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_BACKEND_TIMEOUT_S') or '0.10', + HTTP_METRICS_TIMEOUT_HOSTILE_INTRA_STREAM_TIMEOUT_S = os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_INTRA_STREAM_TIMEOUT_S') or '0.10', + HTTP_METRICS_TIMEOUT_HOSTILE_AUTO_RELEASE_S = os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_AUTO_RELEASE_S') or '0.10', + HTTP_METRICS_TIMEOUT_HOSTILE_UI_EVERY = os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_UI_EVERY') or '3', + } + if os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_PORT_BASE') then + env.HTTP_METRICS_TIMEOUT_HOSTILE_PORT_BASE = os.getenv('HTTP_METRICS_TIMEOUT_HOSTILE_PORT_BASE') + end + + for attempt = 1, attempts do + hostile_append_file(log_path, ('[parent %.6f] attempt:%03d begin timeout_s=%s\n'):format(os.clock(), attempt, tostring(timeout_s))) + local env_parts = {} + for k, v in pairs(env) do env_parts[#env_parts + 1] = k .. '=' .. hostile_shquote(v) end + table.sort(env_parts) + local cmd = ('timeout %s env %s luajit integration/devhost/support/http_hostile_tcp_child.lua >> %s 2>&1'):format( + tostring(timeout_s), table.concat(env_parts, ' '), hostile_shquote(log_path)) + local code = hostile_shell_exit_status(os.execute(cmd)) + hostile_append_file(log_path, ('[parent %.6f] attempt:%03d exit=%s\n'):format(os.clock(), attempt, tostring(code))) + if code ~= 0 then + local tail = hostile_tail_lines(hostile_read_file(log_path), tail_n) + error(('hostile TCP child failed or timed out: attempt=%d/%d exit=%s timeout_s=%s log=%s\n%s'):format( + attempt, attempts, tostring(code), tostring(timeout_s), log_path, tail), 0) + end + end +end + +return M diff --git a/tests/integration/devhost/test_http_transport_real.lua b/tests/integration/devhost/test_http_transport_real.lua new file mode 100644 index 00000000..2f1c99c8 --- /dev/null +++ b/tests/integration/devhost/test_http_transport_real.lua @@ -0,0 +1,356 @@ +-- tests/integration/devhost/test_http_transport_real.lua +-- +-- Real devhost integration tests for the HTTP transport seam. +-- +-- These tests intentionally use real cqueues and lua-http modules. There are no +-- fakes or mocks in this file. Add this module to the top-level integration +-- runner when the devhost/CI image has cqueues and lua-http installed. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local runtime = require 'fibers.runtime' + +local cqueues_condition = require 'cqueues.condition' +local http_server = require 'http.server' +local http_request = require 'http.request' +local http_headers = require 'http.headers' +local http_websocket = require 'http.websocket' + +local driver_mod = require 'services.http.transport.cqueues_driver' +local lua_http = require 'services.http.transport.lua_http' +local ws_wrap = require 'services.http.transport.websocket' +local terminate = require 'services.http.transport.terminate' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then + error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) + end +end + +local function ok(v, msg) + if not v then error(msg or 'assertion failed', 2) end + return v +end + +local function yield_once() + runtime.yield() +end + +local function yield_many(n) + for _ = 1, n do yield_once() end +end + +local function wait_until(predicate, timeout, label) + local deadline = runtime.now() + (timeout or 5) + while not predicate() do + if runtime.now() >= deadline then + error('timed out waiting for ' .. (label or 'condition'), 2) + end + fibers.perform(sleep.sleep_op(0.01)) + end +end + +local function response_headers(status, content_type) + local h = http_headers.new() + h:append(':status', tostring(status or 200)) + if content_type then h:append('content-type', content_type) end + return h +end + +local function http_uri(port, path) + return ('http://127.0.0.1:%d%s'):format(port, path or '/') +end + +local function ws_uri(port, path) + return ('ws://127.0.0.1:%d%s'):format(port, path or '/ws') +end + +local function new_listener(opts) + opts = opts or {} + + local driver = ok(driver_mod.new { + label = opts.label or 'http-transport-real-integration', + }) + + local listener = ok(lua_http.listen { + driver = driver, + http_server = http_server, + host = '127.0.0.1', + port = 0, + tls = false, + max_accept_queue = opts.max_accept_queue or 32, + connection_setup_timeout = opts.connection_setup_timeout or 2, + intra_stream_timeout = opts.intra_stream_timeout or 2, + }) + + return listener, driver +end + +local function start_listener(scope, listener, driver) + ok(driver:start(scope), 'driver should start') + ok(listener:start(scope), 'listener should start') + ok(fibers.perform(listener:listen_op()), 'listener:listen should succeed') + + local family, addr, port = listener:_raw_server_for_test():localname() + ok(family, 'server:localname should return a family') + ok(addr, 'server:localname should return an address') + ok(type(port) == 'number' and port > 0, 'server should bind an ephemeral TCP port') + + return port +end + +local function run_http_client(driver, uri, body, method) + return fibers.perform(driver:run_op('real-http-client', function () + local req = http_request.new_from_uri(uri) + if method then req.headers:upsert(':method', method) end + if body ~= nil then + req.headers:upsert(':method', method or 'POST') + req:set_body(body) + end + + local headers, stream = assert(req:go(5)) + local response_body = assert(stream:get_body_as_string(5)) + return headers:get(':status'), response_body + end)) +end + + +function M.test_real_transport_terminate_server_helper_closes_lua_http_server() + fibers.run(function (scope) + local listener, driver = new_listener({ label = 'http-transport-real-terminate-helper' }) + local port = start_listener(scope, listener, driver) + ok(type(port) == 'number' and port > 0) + ok(terminate.terminate_server(listener:_raw_server_for_test(), 'helper_close')) + listener:terminate('done') + driver:terminate('done') + end) +end + +function M.test_real_http_get_round_trips_through_listener_context() + fibers.run(function (scope) + local listener, driver = new_listener() + local port = start_listener(scope, listener, driver) + + local seen_path + ok(scope:spawn(function () + local ctx = ok(fibers.perform(listener:accept_op())) + local req_headers = ok(fibers.perform(ctx:get_headers_op())) + seen_path = req_headers:get(':path') + + ok(fibers.perform(ctx:write_headers_op(response_headers(200, 'text/plain')))) + ok(fibers.perform(ctx:write_body_from_string_op('hello from fibers'))) + end)) + + local status, body = run_http_client(driver, http_uri(port, '/hello')) + eq(status, '200') + eq(body, 'hello from fibers') + eq(seen_path, '/hello') + + listener:terminate('done') + driver:terminate('done') + end) +end + +function M.test_real_http_post_body_is_read_and_echoed() + fibers.run(function (scope) + local listener, driver = new_listener() + local port = start_listener(scope, listener, driver) + + ok(scope:spawn(function () + local ctx = ok(fibers.perform(listener:accept_op())) + local req_headers = ok(fibers.perform(ctx:get_headers_op())) + eq(req_headers:get(':method'), 'POST') + + local payload = ok(fibers.perform(ctx:read_body_as_string_op())) + ok(fibers.perform(ctx:write_headers_op(response_headers(201, 'text/plain')))) + ok(fibers.perform(ctx:write_body_from_string_op('echo:' .. payload))) + end)) + + local status, body = run_http_client(driver, http_uri(port, '/upload'), 'abc123', 'POST') + eq(status, '201') + eq(body, 'echo:abc123') + + listener:terminate('done') + driver:terminate('done') + end) +end + +function M.test_real_multiple_concurrent_http_requests_complete() + fibers.run(function (scope) + local listener, driver = new_listener() + local port = start_listener(scope, listener, driver) + local n = 5 + local handled = 0 + local completed = 0 + local results = {} + + for _ = 1, n do + ok(scope:spawn(function () + local ctx = ok(fibers.perform(listener:accept_op())) + local req_headers = ok(fibers.perform(ctx:get_headers_op())) + local path = req_headers:get(':path') or '/none' + handled = handled + 1 + ok(fibers.perform(ctx:write_headers_op(response_headers(200, 'text/plain')))) + ok(fibers.perform(ctx:write_body_from_string_op('seen:' .. path))) + end)) + end + + for i = 1, n do + ok(scope:spawn(function () + local status, body = run_http_client(driver, http_uri(port, '/r' .. i)) + results[i] = { status = status, body = body } + completed = completed + 1 + end)) + end + + wait_until(function () return completed == n end, 5, 'concurrent HTTP clients') + + for i = 1, n do + eq(results[i].status, '200') + eq(results[i].body, 'seen:/r' .. i) + end + eq(handled, n) + + listener:terminate('done') + driver:terminate('done') + end) +end + +function M.test_real_websocket_echo_round_trip_uses_context_wrapper() + fibers.run(function (scope) + local listener, driver = new_listener() + local port = start_listener(scope, listener, driver) + + ok(scope:spawn(function () + local ctx = ok(fibers.perform(listener:accept_op())) + local req_headers = ok(fibers.perform(ctx:get_headers_op())) + local ws = ok(fibers.perform(ws_wrap.from_context_op(ctx, req_headers, { + websocket_module = http_websocket, + }))) + ok(fibers.perform(ws:accept_op({}))) + + local data, opcode = fibers.perform(ws:receive_op()) + ok(data, 'server WebSocket receive should return data') + eq(opcode, 'text') + + ok(fibers.perform(ws:send_op(data .. ':echo', 'text'))) + fibers.perform(ws:close_op(1000, 'done')) + end)) + + local client_data, client_opcode = fibers.perform(driver:run_op('real-websocket-client', function () + local ws = assert(http_websocket.new_from_uri(ws_uri(port, '/ws'))) + assert(ws:connect(5)) + assert(ws:send('ping', 'text', 5)) + local data, opcode = assert(ws:receive(5)) + ws:close(1000, 'client_done', 5) + return data, opcode + end)) + + eq(client_data, 'ping:echo') + eq(client_opcode, 'text') + + listener:terminate('done') + driver:terminate('done') + end) +end + +function M.test_real_losing_driver_run_op_aborts_before_controller_pump() + local driver = ok(driver_mod.new { label = 'http-transport-losing-run-op' }) + local ran = false + + fibers.run(function (scope) + local which = fibers.perform(fibers.named_choice { + winner = fibers.always('now'), + loser = driver:run_op('must-not-run', function () + ran = true + return 'bad' + end), + }) + eq(which, 'winner') + + ok(driver:start(scope)) + yield_many(8) + ok(not ran, 'aborted cqueues job must not run after losing choice') + + driver:terminate('done') + end) +end + +function M.test_real_multiple_concurrent_driver_run_ops_complete() + local driver = ok(driver_mod.new { label = 'http-transport-concurrent-run-ops' }) + + fibers.run(function (scope) + ok(driver:start(scope)) + + local n = 6 + local completed = 0 + local results = {} + + for i = 1, n do + ok(scope:spawn(function () + local value, err = fibers.perform(driver:run_op('real-job-' .. i, function () + return i * 10 + end)) + results[i] = { value = value, err = err } + completed = completed + 1 + end)) + end + + wait_until(function () return completed == n end, 5, 'concurrent cqueues jobs') + + for i = 1, n do + eq(results[i].value, i * 10) + eq(results[i].err, nil) + end + + driver:terminate('done') + end) +end + +function M.test_real_listener_termination_wakes_pending_accept() + fibers.run(function (scope) + local listener, driver = new_listener() + start_listener(scope, listener, driver) + + local got, err + ok(scope:spawn(function () + got, err = fibers.perform(listener:accept_op()) + end)) + + yield_many(4) + listener:terminate('listener_closed') + yield_many(4) + + eq(got, nil) + eq(err, 'listener_closed') + + driver:terminate('done') + end) +end + +function M.test_real_driver_termination_wakes_pending_cqueues_job() + fibers.run(function (scope) + local driver = ok(driver_mod.new { label = 'http-transport-driver-termination' }) + ok(driver:start(scope)) + + local result, err + ok(scope:spawn(function () + result, err = fibers.perform(driver:run_op('blocked-real-cqueues-job', function () + local cv = cqueues_condition.new and cqueues_condition.new() or cqueues_condition() + cv:wait() + return 'unexpected' + end)) + end)) + + yield_many(8) + driver:terminate('driver_stopped') + yield_many(8) + + eq(result, nil) + eq(err, 'driver_stopped') + end) +end + +return M diff --git a/tests/integration/devhost/update_public_seams_spec.lua b/tests/integration/devhost/update_public_seams_spec.lua new file mode 100644 index 00000000..5abba5cb --- /dev/null +++ b/tests/integration/devhost/update_public_seams_spec.lua @@ -0,0 +1,238 @@ +-- tests/integration/devhost/update_public_seams_spec.lua +-- +-- Public Update control-plane seam tests. These exercise the canonical +-- cap/... manager interfaces and retained state/workflow publication, rather +-- than service-private command trees. + +local busmod = require 'bus' +local fibers = require 'fibers' + +local runfibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' + +local service = require 'services.update.service' +local topics = require 'services.update.topics' +local upload = require 'services.ui.update.upload' + +local T = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end + + +local function wait_retained_payload_where(conn, topic, label, pred, opts) + opts = opts or {} + local view = conn:retained_view(topic) + local value = probe.wait_versioned_until(label, function () + return view:version() + end, function (seen) + return view:changed_op(seen) + end, function () + local msg = view:get(topic) + local payload = msg and msg.payload or nil + if pred(payload) then return payload end + return nil + end, opts) + view:close() + return value +end + +local function has_value(list, value) + for _, item in ipairs(list or {}) do + if item == value then return true end + end + return false +end + +local function update_config() + return { + schema = 'devicecode.update/1', + components = { + { component = 'cm5' }, + { component = 'mcu' }, + }, + } +end + +local function start_update(scope, params) + params = params or {} + local bus = params.bus or busmod.new() + local svc_conn = bus:connect() + local caller = bus:connect() + local child = assert(scope:child()) + + local ok, err = child:spawn(function (service_scope) + service.run(service_scope, { + conn = svc_conn, + service_id = 'update', + watch_config = false, + config = params.config or update_config(), + backend = params.backend, + job_store = params.job_store, + initial_jobs = params.initial_jobs, + job_store_kind = params.job_store_kind or 'memory', + }) + end) + assert_true(ok, err) + + wait_retained_payload_where(caller, topics.update_manager_status(), 'update manager available', function (p) + return p and p.available == true + end, { timeout = 1.0 }) + return { + bus = bus, + child = child, + caller = caller, + svc_conn = svc_conn, + } +end + +local function new_sink(artifact) + artifact = artifact or { artifact_id = 'artifact-1', ref = 'artifact-1' } + return { + chunks = {}, + terminated = 0, + append_op = function (self, chunk) + self.chunks[#self.chunks + 1] = chunk + return fibers.always(true, nil) + end, + commit_op = function () + return fibers.always(artifact, nil) + end, + terminate = function (self, reason) + self.terminated = self.terminated + 1 + self.termination_reason = reason + return true, nil + end, + } +end + +local function body_from_chunks(chunks) + return { + _chunks = chunks, + read_chunk_op = function (self) + local chunk = table.remove(self._chunks, 1) + return fibers.always(chunk, nil) + end, + } +end + +function T.update_public_manager_retains_capabilities_and_compact_component_state() + runfibers.run(function(scope) + local h = start_update(scope) + local conn = h.caller + + local meta = probe.wait_retained_payload(conn, topics.update_manager_meta(), { timeout = 1.0 }) + assert_eq(meta.owner, 'update') + assert_eq(meta.class, 'update-manager') + assert_true(has_value(meta.methods, 'create-job'), 'manager meta should list create-job') + assert_true(has_value(meta.methods, 'start-job'), 'manager meta should list start-job') + assert_eq(meta.workflow_family, nil) + + local status = wait_retained_payload_where(conn, topics.update_manager_status(), 'update manager status', function (p) + return p and p.available == true + end, { timeout = 1.0 }) + assert_eq(status.available, true) + + local summary = wait_retained_payload_where(conn, topics.update_summary(), 'update summary ready', function (p) + return p and p.ready == true + end, { timeout = 1.0 }) + assert_eq(summary.service, 'update') + assert_eq(summary.ready, true) + + local reply, err = conn:call(topics.update_manager_rpc('create-job'), { + job_id = 'j-cap-1', + component = 'cm5', + artifact_ref = 'artifact-cap-1', + }, { timeout = 1.0 }) + assert_not_nil(reply, err) + assert_eq(reply.ok, true) + assert_eq(reply.job.job_id, 'j-cap-1') + + local component = probe.wait_retained_payload(conn, topics.update_component('cm5'), { timeout = 1.0 }) + assert_eq(component.kind, 'update.component') + assert_not_nil(component.current_job) + assert_eq(component.current_job.job_id, 'j-cap-1') + assert_eq(component.jobs, nil) + + h.child:cancel('test complete') + end, { timeout = 5.0 }) +end + +function T.update_artifact_ingest_uses_public_cap_and_compact_job_state() + runfibers.run(function(scope) + local h = start_update(scope) + local conn = h.caller + local sink = new_sink({ artifact_id = 'artifact-ingest-1', ref = 'artifact-ingest-1' }) + + local meta = probe.wait_retained_payload(conn, topics.artifact_ingest_meta(), { timeout = 1.0 }) + assert_eq(meta.owner, 'update') + assert_eq(meta.class, 'artifact-ingest') + assert_true(has_value(meta.methods, 'create'), 'ingest meta should list create') + assert_true(has_value(meta.methods, 'commit'), 'ingest meta should list commit') + + local created, create_err = conn:call(topics.artifact_ingest_rpc('create'), { + ingest_id = 'ing-public-1', + component = 'cm5', + sink = sink, + }, { timeout = 1.0 }) + assert_not_nil(created, create_err) + assert_eq(created.ok, true) + + local appended, append_err = conn:call(topics.artifact_ingest_rpc('append'), { + ingest_id = 'ing-public-1', + chunk = 'abc', + }, { timeout = 1.0 }) + assert_not_nil(appended, append_err) + assert_eq(appended.ok, true) + assert_eq(sink.chunks[1], 'abc') + + local committed, commit_err = conn:call(topics.artifact_ingest_rpc('commit'), { + ingest_id = 'ing-public-1', + }, { timeout = 1.0 }) + assert_not_nil(committed, commit_err) + assert_eq(committed.ok, true) + assert_eq(committed.commit.artifact.artifact_id, 'artifact-ingest-1') + + + h.child:cancel('test complete') + end, { timeout = 5.0 }) +end + +function T.ui_upload_drives_update_public_ingest_and_manager_caps() + runfibers.run(function(scope) + local bus = busmod.new() + local h = start_update(scope, { bus = bus }) + local conn = h.caller + local sink = new_sink({ artifact_id = 'artifact-upload-1', ref = 'artifact-upload-1' }) + + local st, _, result = fibers.perform(upload.run_op({ + body_stream = body_from_chunks({ 'hello', 'world' }), + }, { + bus = bus, + sink = sink, + component = 'cm5', + ingest_id = 'ing-upload-1', + job_id = 'j-upload-1', + create_job = true, + timeout = 1.0, + })) + assert_eq(st, 'ok') + assert_not_nil(result) + assert_eq(result.job_id, 'j-upload-1') + assert_nil(result.job) + assert_eq(sink.chunks[1], 'hello') + assert_eq(sink.chunks[2], 'world') + + + local component = probe.wait_retained_payload(conn, topics.update_component('cm5'), { timeout = 1.0 }) + assert_not_nil(component.current_job) + assert_eq(component.current_job.job_id, 'j-upload-1') + + h.child:cancel('test complete') + end, { timeout = 5.0 }) +end + +return T diff --git a/tests/integration/devhost/wired_assembly_projection_spec.lua b/tests/integration/devhost/wired_assembly_projection_spec.lua new file mode 100644 index 00000000..32c1e7a1 --- /dev/null +++ b/tests/integration/devhost/wired_assembly_projection_spec.lua @@ -0,0 +1,184 @@ +local fibers = require 'fibers' +local busmod = require 'bus' + +local runfibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' + +local wired_service = require 'services.wired.service' +local wired_config = require 'services.wired.config' +local wired_topics = require 'services.wired.topics' + +local T = {} + +local function assert_eq(a, b, msg) + if a ~= b then error(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a)), 2) end +end + +local function assert_not_nil(v, msg) + if v == nil then error(msg or 'expected non-nil', 2) end + return v +end + +local function retain_net_segments(conn) + conn:retain({ 'state', 'net', 'segments' }, { + rev = 1, + segments = { + adm = { kind = 'system', protected = true, vlan = { id = 8 } }, + int = { kind = 'system', protected = true, vlan = { id = 100 } }, + jan = { kind = 'user', vlan = { id = 32 } }, + wan = { kind = 'wan', vlan = { id = 4 } }, + }, + }) +end + +local function retain_device_assembly(conn) + conn:retain({ 'state', 'device', 'assembly' }, { + kind = 'device.assembly', + product = 'big-box', + components = { + cm5 = { kind = 'compute', role = 'controller' }, + mcu = { kind = 'microcontroller', role = 'power-sensor-controller' }, + ['cm5-local-wired'] = { kind = 'direct-nic', role = 'controller-wired-port' }, + ['switch-main'] = { kind = 'switch', role = 'wired-fabric' }, + }, + links = { + ['cm5-switch'] = { + kind = 'wired', + role = 'controller-switch-uplink', + internal = true, + a = { component = 'cm5-local-wired', observed_surface = 'eth0' }, + b = { component = 'switch-main', observed_surface = 'GE8' }, + }, + }, + surfaces = { + ['cm5-eth0'] = { component = 'cm5-local-wired', observed_surface = 'eth0', exposure = 'internal' }, + ['switch-uplink-cm5'] = { component = 'switch-main', observed_surface = 'GE8', exposure = 'internal' }, + ['lan-1'] = { component = 'switch-main', observed_surface = 'GE1', exposure = 'external' }, + }, + }) +end + +local function retain_raw_wired_observations(conn) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'cm5-local-wired', 'status' }, { + state = 'available', + available = true, + }) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'cm5-local-wired', 'state', 'surfaces' }, { + surfaces = { + eth0 = { + observed_surface = 'eth0', + kind = 'direct-nic', + capabilities = { trunk = true, access = false, poe = false }, + link = { state = 'up', speed_mbps = 1000 }, + attachment = { mode = 'trunk', vlans = { 8, 100, 32, 4 } }, + }, + }, + }) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'status' }, { + state = 'available', + available = true, + mode = 'read_only', + driver = 'rtl8380m_http', + }) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'runtime' }, { + cpu = { utilisation_pct = 3 }, + memory = { utilisation_pct = 61 }, + }) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'power' }, { + poe = { total_power_mw = 0, total_power_w = 0, temperature_c = 28 }, + }) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'surfaces' }, { + surfaces = { + GE8 = { + observed_surface = 'GE8', + kind = 'switch-port', + capabilities = { trunk = true, access = false, poe = false }, + link = { state = 'up', speed_mbps = 1000 }, + attachment = { mode = 'trunk', vlans = { 8, 100, 32, 4 } }, + }, + GE1 = { + observed_surface = 'GE1', + kind = 'ethernet-port', + capabilities = { access = true, trunk = true, poe = true }, + link = { state = 'up', speed_mbps = 1000 }, + attachment = { mode = 'access', vlan = 32 }, + poe = { state = 'off' }, + }, + }, + }) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'counters' }, { + counters = { + GE1 = { rx = { bytes = 1234, packets = 12, drops = 0, errors = 0 } }, + }, + }) +end + +local function retain_wired_config(conn) + conn:retain({ 'cfg', 'wired' }, { + rev = 1, + data = { + schema = wired_config.SCHEMA, + version = 1, + surfaces = { + ['switch-uplink-cm5'] = { + kind = 'switch-port', + role = 'internal-trunk', + protected = true, + attachment = { mode = 'trunk', required_segments = { 'adm', 'int' }, user_segments = 'all-realised-user-segments' }, + }, + ['lan-1'] = { + kind = 'ethernet-port', + role = 'access', + capabilities = { poe = true }, + attachment = { mode = 'access', segment = 'jan' }, + }, + }, + }, + }) +end + +function T.state_device_assembly_and_raw_switch_observations_project_to_state_wired() + runfibers.run(function (scope) + local b = busmod.new() + local service_conn = b:connect({ origin_base = { kind = 'local', component = 'wired-service-test' } }) + local writer = b:connect({ origin_base = { kind = 'local', component = 'wired-test-writer' } }) + local reader = b:connect({ origin_base = { kind = 'local', component = 'wired-test-reader' } }) + + retain_net_segments(writer) + retain_device_assembly(writer) + retain_raw_wired_observations(writer) + retain_wired_config(writer) + + local child = assert(scope:child()) + assert(child:spawn(function (svc_scope) wired_service.run(svc_scope, { conn = service_conn, service_id = 'test-wired' }) end)) + + local lan1 = probe.wait_retained_payload(reader, wired_topics.surface('lan-1'), { timeout = 1.5 }) + assert_eq(lan1.surface_id, 'lan-1') + assert_eq(lan1.attachment.mode, 'access') + assert_eq(lan1.attachment.segment, 'jan') + assert_eq(lan1.availability.state, 'available') + assert_eq(lan1.link.state, 'up') + assert_eq(lan1.source.component, 'switch-main') + assert_eq(lan1.source.observed_surface, 'GE1') + assert_eq(lan1.observed.source.exposure, 'external') + assert_eq(lan1.counters, nil, 'stable public surface should not carry volatile counters') + local counters = probe.wait_retained_payload(reader, wired_topics.surface_counters('lan-1'), { timeout = 1.0 }) + assert_eq(counters.counters.rx.bytes, 1234) + + local uplink = probe.wait_retained_payload(reader, wired_topics.surface('switch-uplink-cm5'), { timeout = 1.5 }) + assert_eq(uplink.availability.state, 'available') + assert_eq(uplink.source.component, 'switch-main') + assert_eq(uplink.source.observed_surface, 'GE8') + assert_eq(uplink.observed.source.exposure, 'internal') + + local topology = probe.wait_retained_payload(reader, wired_topics.topology(), { timeout = 1.0 }) + assert_not_nil(topology.protected_trunks['switch-uplink-cm5']) + local violations = probe.wait_retained_payload(reader, wired_topics.violations(), { timeout = 1.0 }) + assert_eq(#(violations.violations or {}), 0) + + child:cancel('test complete') + fibers.perform(child:join_op()) + end, { timeout = 3.0 }) +end + +return T diff --git a/tests/integration/openwrt_vm/Makefile b/tests/integration/openwrt_vm/Makefile new file mode 100644 index 00000000..1a7cbe75 --- /dev/null +++ b/tests/integration/openwrt_vm/Makefile @@ -0,0 +1,283 @@ +.PHONY: \ + preflight fetch verify reset run wait provision provision-force ensure-mwan3 \ + stop ssh smoke logs render-default-configs print-default-configs baseline test-baseline test-tc-veth test-lua-uci test-devicecode-uci-manager test-devicecode-uci-manager-async-activation test-devicecode-openwrt-no-blocking-os-io test-devicecode-exec-containment test-openwrt-uart-stty-coreutils test-openwrt-network-provider-apply test-openwrt-network-provider-async-activation test-openwrt-network-provider-fw4-schema test-openwrt-network-provider-snapshot test-openwrt-network-provider-live-snapshot test-openwrt-network-observer-event-ingress test-openwrt-network-provider-vlan-mwan-shaping test-openwrt-network-provider-mwan-live-weights test-openwrt-network-provider-mwan-live-weights-fast test-openwrt-vm-generated-configs-expected test-openwrt-vm-mwan-connected test-openwrt-network-provider-segment-trunk test-openwrt-jan-client-per-host-shaping test-openwrt-shaping-idempotent-apply test-openwrt-segment-shaping-modes test-openwrt-wan-mark-shaping-contract test-openwrt-wan-mark-upload-exemption test-openwrt-wan-mark-download-capability test-openwrt-shaping-fast test-openwrt-shaping-clean test-devicecode-wired-static-provider test-devicecode-full-stack-mock-hal test-devicecode-bigbox-phase1-composition test-devicecode-bigbox-phase1-broken-trunk test-openwrt-jan-client-dhcp-dns test-openwrt-int-bridge-client-dhcp-dns test-openwrt-dnsmasq-multi-instance-resilience \ + setup-bridge-client-fabric teardown-bridge-client-fabric \ + network-lab-fetch network-lab-start network-lab-wait network-lab-provision network-lab-sync network-lab-ssh network-lab-stop network-lab-test test-network-lab \ + test openwrt-vm-test clean + +preflight: + ./scripts/preflight + +fetch: + ./scripts/fetch-image + +verify: + ./scripts/verify-image + +reset: + ./scripts/reset-disk + +run: + ./scripts/run-vm + +wait: + ./scripts/wait-ssh + +provision: + ./scripts/provision + +provision-force: + OPENWRT_PROVISION_FORCE=1 ./scripts/provision + +ensure-mwan3: + ./scripts/ensure-mwan3 + +stop: + ./scripts/stop-vm + +ssh: + ./scripts/ssh + +smoke: + ./scripts/wait-ssh + ./scripts/ssh 'uname -a; uci show system | head; ip link; for x in uci ip nft tc; do printf "%s: " "$$x"; command -v "$$x" || echo missing; done' + +logs: + tail -n 200 work/qemu.log + +render-default-configs: + ./scripts/wait-ssh + DEVICECODE_RENDER_PRINT=0 ./scripts/render-default-configs + +print-default-configs: + ./scripts/wait-ssh + DEVICECODE_RENDER_PRINT=1 ./scripts/render-default-configs + +baseline: test-baseline + +test-baseline: + ./scripts/wait-ssh + ./tests/test_openwrt_baseline.sh + +test-tc-veth: + ./scripts/wait-ssh + ./tests/test_tc_veth_baseline.sh + +test-lua-uci: + ./scripts/wait-ssh + ./tests/test_lua_uci_baseline.sh + +test-devicecode-uci-manager: + ./scripts/wait-ssh + ./tests/test_devicecode_uci_manager.sh + +test-devicecode-uci-manager-async-activation: + ./scripts/wait-ssh + ./tests/test_devicecode_uci_manager_async_activation.sh + +test-devicecode-openwrt-no-blocking-os-io: + ./tests/test_devicecode_openwrt_no_blocking_os_io.sh + +test-openwrt-uart-stty-coreutils: + ./scripts/wait-ssh + ./tests/test_openwrt_uart_stty_coreutils.sh + +test-openwrt-network-provider-async-activation: + ./scripts/wait-ssh + ./tests/test_openwrt_network_provider_async_activation.sh + +test-openwrt-network-provider-apply: + ./scripts/wait-ssh + ./tests/test_openwrt_network_provider_apply.sh + +test-openwrt-network-provider-fw4-schema: + ./scripts/wait-ssh + ./tests/test_openwrt_network_provider_fw4_schema.sh + +test-openwrt-network-provider-snapshot: + ./scripts/wait-ssh + ./tests/test_openwrt_network_provider_snapshot.sh + +test-openwrt-network-provider-live-snapshot: + ./scripts/wait-ssh + ./tests/test_openwrt_network_provider_live_snapshot.sh + +test-openwrt-network-observer-event-ingress: + ./scripts/wait-ssh + ./tests/test_openwrt_network_observer_event_ingress.sh + +test-openwrt-network-provider-vlan-mwan-shaping: + ./scripts/wait-ssh + ./tests/test_openwrt_network_provider_vlan_mwan_shaping.sh + +test-openwrt-network-provider-mwan-live-weights: + ./scripts/wait-ssh + ./scripts/ensure-mwan3 + ./tests/setup_mwan3_three_wan_fixture.sh + ./tests/test_openwrt_network_provider_mwan_live_weights.sh + +test-openwrt-network-provider-mwan-live-weights-fast: + ./scripts/wait-ssh + ./tests/test_openwrt_network_provider_mwan_live_weights.sh + +test-openwrt-vm-generated-configs-expected: + ./scripts/wait-ssh + ./scripts/ensure-mwan3 + ./tests/test_openwrt_vm_generated_configs_expected.sh + +test-openwrt-vm-mwan-connected: + ./scripts/wait-ssh + ./scripts/ensure-mwan3 + ./tests/test_openwrt_vm_mwan_connected.sh + + +test-openwrt-network-provider-segment-trunk: + ./scripts/wait-ssh + ./tests/test_openwrt_network_provider_segment_trunk.sh + +test-devicecode-wired-static-provider: + ./scripts/wait-ssh + ./tests/test_devicecode_wired_static_provider.sh + +test-devicecode-bigbox-phase1-composition: + ./scripts/wait-ssh + ./tests/test_devicecode_bigbox_phase1_composition.sh + +test-devicecode-bigbox-phase1-broken-trunk: + ./scripts/wait-ssh + ./tests/test_devicecode_bigbox_phase1_broken_trunk.sh + + + +test-devicecode-full-stack-mock-hal: + ./scripts/wait-ssh + ./tests/test_devicecode_full_stack_mock_hal.sh + +test-openwrt-jan-client-dhcp-dns: + ./scripts/ensure-jan-client-dhcp-dns-deps + ./tests/test_openwrt_jan_client_dhcp_dns.sh + +test-openwrt-jan-client-per-host-shaping: + ./scripts/ensure-jan-client-dhcp-dns-deps + ./tests/test_openwrt_jan_client_per_host_shaping.sh + + +test-openwrt-shaping-idempotent-apply: + ./scripts/ensure-jan-client-dhcp-dns-deps + ./tests/test_openwrt_shaping_idempotent_apply.sh + + +test-openwrt-segment-shaping-modes: + ./scripts/ensure-jan-client-dhcp-dns-deps + ./tests/test_openwrt_segment_shaping_modes.sh + + +test-openwrt-wan-mark-shaping-contract: + ./scripts/wait-ssh + ./tests/test_openwrt_wan_mark_shaping_contract.sh + + +test-openwrt-wan-mark-upload-exemption: + ./scripts/ensure-jan-client-dhcp-dns-deps + ./scripts/ensure-mwan3 + ./tests/test_openwrt_wan_mark_upload_exemption.sh + + +test-openwrt-wan-mark-download-capability: + ./scripts/ensure-jan-client-dhcp-dns-deps + ./tests/test_openwrt_wan_mark_download_capability.sh + + +test-openwrt-shaping-clean: + ./scripts/cleanup-shaping-state + + +test-openwrt-shaping-fast: + ./scripts/ensure-jan-client-dhcp-dns-deps + ./scripts/ensure-devicecode-staged >/dev/null + ./scripts/backup-openwrt-config-state >/dev/null + set -e; \ + cleanup() { \ + ./scripts/cleanup-shaping-state >/dev/null 2>&1 || true; \ + DEVICECODE_RESTORE_ACTIVATE=0 DEVICECODE_RESTORE_NOWAIT=1 ./scripts/restore-openwrt-config-state >/dev/null 2>&1 || true; \ + }; \ + trap cleanup EXIT INT TERM; \ + ./scripts/cleanup-shaping-state >/dev/null 2>&1 || true; \ + DC_SHAPING_CLEANUP_DISABLED=1 ./tests/test_openwrt_shaping_idempotent_apply.sh; \ + DC_SHAPING_CLEANUP_DISABLED=1 ./tests/test_openwrt_jan_client_per_host_shaping.sh; \ + DC_SHAPING_CLEANUP_DISABLED=1 ./tests/test_openwrt_segment_shaping_modes.sh; \ + DC_SHAPING_CLEANUP_DISABLED=1 ./tests/test_openwrt_wan_mark_shaping_contract.sh; \ + ./scripts/ensure-mwan3; \ + DC_SHAPING_CLEANUP_DISABLED=1 ./tests/test_openwrt_wan_mark_upload_exemption.sh; \ + DC_SHAPING_CLEANUP_DISABLED=1 ./tests/test_openwrt_wan_mark_download_capability.sh + +setup-bridge-client-fabric: + ./scripts/setup-bridge-client-fabric + +teardown-bridge-client-fabric: + ./scripts/teardown-bridge-client-fabric + +test-openwrt-int-bridge-client-dhcp-dns: + ./scripts/stop-vm || true + ./scripts/setup-bridge-client-fabric + . ./env.sh; OPENWRT_TAP_IFACES="$$OPENWRT_CLIENT_TAP" ./scripts/run-vm + ./scripts/wait-ssh + ./scripts/provision + ./tests/test_openwrt_int_bridge_client_dhcp_dns.sh + + +test-openwrt-dnsmasq-multi-instance-resilience: + ./scripts/stop-vm || true + ./scripts/setup-bridge-client-fabric + . ./env.sh; OPENWRT_TAP_IFACES="$$OPENWRT_CLIENT_TAP" ./scripts/run-vm + ./scripts/wait-ssh + ./scripts/provision + ./tests/test_openwrt_dnsmasq_multi_instance_resilience.sh + + +network-lab-fetch: + sh ./scripts/network-lab-fetch-image + +network-lab-start: + sh ./scripts/network-lab-start + +network-lab-wait: + sh ./scripts/network-lab-wait + +network-lab-provision: + sh ./scripts/network-lab-provision + +network-lab-sync: + sh ./scripts/network-lab-sync + +network-lab-ssh: + sh ./scripts/network-lab-ssh + +network-lab-stop: + sh ./scripts/network-lab-stop + +network-lab-test test-network-lab: + sh ./scripts/network-lab-test + +test-scp: + ./scripts/wait-ssh + tmp="$$(mktemp)"; \ + echo "devicecode-openwrt-vm" > "$$tmp"; \ + ./scripts/scp-to "$$tmp" /tmp/devicecode-scp-test; \ + ./scripts/ssh 'grep -q devicecode-openwrt-vm /tmp/devicecode-scp-test'; \ + ./scripts/scp-from /tmp/devicecode-scp-test "$$tmp.out"; \ + grep -q devicecode-openwrt-vm "$$tmp.out"; \ + rm -f "$$tmp" "$$tmp.out" + +test: test-baseline test-tc-veth test-lua-uci test-devicecode-uci-manager test-devicecode-uci-manager-async-activation test-devicecode-openwrt-no-blocking-os-io test-devicecode-exec-containment test-openwrt-uart-stty-coreutils test-openwrt-network-provider-apply test-openwrt-network-provider-async-activation test-openwrt-network-provider-fw4-schema test-openwrt-network-provider-snapshot test-openwrt-network-provider-live-snapshot test-openwrt-network-observer-event-ingress test-openwrt-network-provider-vlan-mwan-shaping test-openwrt-network-provider-mwan-live-weights test-openwrt-vm-generated-configs-expected test-openwrt-vm-mwan-connected test-openwrt-network-provider-segment-trunk test-openwrt-jan-client-per-host-shaping test-openwrt-shaping-idempotent-apply test-openwrt-segment-shaping-modes test-openwrt-wan-mark-shaping-contract test-openwrt-wan-mark-upload-exemption test-openwrt-wan-mark-download-capability test-devicecode-wired-static-provider test-devicecode-full-stack-mock-hal test-devicecode-bigbox-phase1-composition test-devicecode-bigbox-phase1-broken-trunk test-scp + +openwrt-vm-test: preflight fetch verify stop reset run wait provision test + +clean: + ./scripts/stop-vm + rm -rf work + +test-devicecode-exec-containment: + @echo "==> devicecode exec containment" + @$(RUN) tests/test_devicecode_exec_containment.sh + diff --git a/tests/integration/openwrt_vm/env.sh b/tests/integration/openwrt_vm/env.sh new file mode 100755 index 00000000..a31e2134 --- /dev/null +++ b/tests/integration/openwrt_vm/env.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env sh +# Shared environment for the OpenWrt VM integration test lane. +# +# Scripts normally set VM_DIR before sourcing this file. When sourced directly, +# fall back to the directory containing this file if the caller has cd'd here. + +if [ -z "${VM_DIR:-}" ]; then + VM_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +fi + +# Some minimal Debian/cloud images keep administrative networking tools such as +# `bridge` under /usr/sbin, which is not always in a non-login user's PATH. +PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}" +export PATH + +OPENWRT_VERSION="${OPENWRT_VERSION:-24.10.4}" +OPENWRT_SSH_PORT="${OPENWRT_SSH_PORT:-2222}" +OPENWRT_SSH_WAIT_S="${OPENWRT_SSH_WAIT_S:-90}" +OPENWRT_VM_MEM="${OPENWRT_VM_MEM:-512M}" +OPENWRT_VM_CPUS="${OPENWRT_VM_CPUS:-2}" +# Number of additional QEMU user-mode NICs to attach as deterministic WAN +# devices for network/MWAN integration tests. With the management NIC first, +# OpenWrt normally sees these as eth1..ethN. +OPENWRT_VM_WAN_IFACES="${OPENWRT_VM_WAN_IFACES:-3}" +OPENWRT_KVM="${OPENWRT_KVM:-auto}" + +# Optional host tap devices for dataplane tests. These are deliberately separate +# from the QEMU user-mode management NIC. Leave empty for the default self-contained +# VM tests, which create veth/ifb devices inside OpenWrt. +# Example: OPENWRT_TAP_IFACES="tap-dc0 tap-dc1" +OPENWRT_TAP_IFACES="${OPENWRT_TAP_IFACES:-}" + +# Optional host-side Linux bridge dataplane fixture. This is used only by +# explicit external-client targets; the default VM suite remains self-contained. +OPENWRT_CLIENT_BRIDGE="${OPENWRT_CLIENT_BRIDGE:-brdcint}" +OPENWRT_CLIENT_TAP="${OPENWRT_CLIENT_TAP:-tapdcint}" +OPENWRT_CLIENT_NS="${OPENWRT_CLIENT_NS:-dcintc}" +OPENWRT_CLIENT_HOST_IFACE="${OPENWRT_CLIENT_HOST_IFACE:-vdcinth}" +# Temporary root-namespace veth peer name. It is moved into the client namespace +# and then renamed to OPENWRT_CLIENT_IFACE, because common names such as +# eth0 already exist in the devcontainer root namespace. +OPENWRT_CLIENT_PEER_IFACE="${OPENWRT_CLIENT_PEER_IFACE:-vdcintp}" +OPENWRT_CLIENT_IFACE="${OPENWRT_CLIENT_IFACE:-eth0}" +OPENWRT_CLIENT_VLAN="${OPENWRT_CLIENT_VLAN:-100}" +OPENWRT_CLIENT_CIDR_PREFIX="${OPENWRT_CLIENT_CIDR_PREFIX:-172.28.100.}" +OPENWRT_CLIENT_ROUTER="${OPENWRT_CLIENT_ROUTER:-172.28.100.1}" +OPENWRT_CLIENT_PUBLIC_DNS_NAME="${OPENWRT_CLIENT_PUBLIC_DNS_NAME:-openwrt.org}" +OPENWRT_CLIENT_PUBLIC_WEB_URL="${OPENWRT_CLIENT_PUBLIC_WEB_URL:-http://example.com/}" + +case "$(uname -m)" in + x86_64|amd64) + OPENWRT_ARCH="x86_64" + OPENWRT_TARGET_PATH="x86/64" + OPENWRT_PREFIX="openwrt-${OPENWRT_VERSION}-x86-64" + OPENWRT_IMAGE_NAME="${OPENWRT_PREFIX}-generic-ext4-combined.img.gz" + OPENWRT_QEMU="${OPENWRT_QEMU:-qemu-system-x86_64}" + ;; + aarch64|arm64) + OPENWRT_ARCH="aarch64" + OPENWRT_TARGET_PATH="armsr/armv8" + OPENWRT_PREFIX="openwrt-${OPENWRT_VERSION}-armsr-armv8" + OPENWRT_IMAGE_NAME="${OPENWRT_PREFIX}-generic-ext4-combined-efi.img.gz" + OPENWRT_QEMU="${OPENWRT_QEMU:-qemu-system-aarch64}" + ;; + *) + echo "unsupported host architecture: $(uname -m)" >&2 + exit 1 + ;; +esac + +OPENWRT_BASE_URL="https://downloads.openwrt.org/releases/${OPENWRT_VERSION}/targets/${OPENWRT_TARGET_PATH}" + +OPENWRT_DIR="${VM_DIR}" +OPENWRT_IMAGE_DIR="${OPENWRT_DIR}/images" +OPENWRT_WORK_DIR="${OPENWRT_DIR}/work" +OPENWRT_IMAGE_GZ="${OPENWRT_IMAGE_DIR}/${OPENWRT_IMAGE_NAME}" +OPENWRT_IMAGE_RAW="${OPENWRT_IMAGE_DIR}/${OPENWRT_IMAGE_NAME%.gz}" +OPENWRT_SHA256SUMS_NAME="sha256sums" +OPENWRT_SHA256SUMS="${OPENWRT_IMAGE_DIR}/sha256sums-${OPENWRT_VERSION}-${OPENWRT_ARCH}" +OPENWRT_WORK_DISK="${OPENWRT_WORK_DIR}/openwrt-${OPENWRT_ARCH}.qcow2" +OPENWRT_PID="${OPENWRT_WORK_DIR}/qemu.pid" +OPENWRT_LOG="${OPENWRT_WORK_DIR}/qemu.log" + +OPENWRT_PROVISION_MARKER="${OPENWRT_PROVISION_MARKER:-/etc/devicecode-vm-provisioned}" +OPENWRT_PROVISION_FORCE="${OPENWRT_PROVISION_FORCE:-0}" diff --git a/tests/integration/openwrt_vm/fixtures/devicecode_vm_mwan_intent.lua b/tests/integration/openwrt_vm/fixtures/devicecode_vm_mwan_intent.lua new file mode 100644 index 00000000..c90e6b3d --- /dev/null +++ b/tests/integration/openwrt_vm/fixtures/devicecode_vm_mwan_intent.lua @@ -0,0 +1,84 @@ +-- Shared OpenWrt VM intent used by real /etc/config + mwan3 integration tests. +-- The VM exposes eth0 as the QEMU management/LAN NIC and eth1..eth3 as +-- deterministic QEMU user-mode WAN NICs when OPENWRT_VM_WAN_IFACES=3. + +local M = {} + +M.wans = { + { id = 'wan', device = 'eth1', gateway = '172.31.1.2', route_metric = 11, weight = 50 }, + { id = 'wanb', device = 'eth2', gateway = '172.31.2.2', route_metric = 12, weight = 30 }, + { id = 'wanc', device = 'eth3', gateway = '172.31.3.2', route_metric = 13, weight = 20 }, +} + +function M.intent() + return { + schema = 'devicecode.net.intent/1', + rev = 24010, + generation = 24010, + segments = { + lan = { + kind = 'lan', + addressing = { ipv4 = { mode = 'static', cidr = '192.168.1.1/24' } }, + dhcp = { enabled = true, start = 100, limit = 150, leasetime = '12h' }, + dns = { local_server = true, domain = 'vm.bigbox.test' }, + firewall = { zone = 'lan' }, + }, + }, + interfaces = { + lan = { + kind = 'bridge', role = 'lan', segment = 'lan', members = { 'eth0' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.1.1/24' } }, + firewall = { zone = 'lan' }, + }, + wan = { + kind = 'ethernet', role = 'wan', endpoint = { ifname = 'eth1' }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, + dhcp = { enabled = false }, firewall = { zone = 'wan' }, + }, + wanb = { + kind = 'ethernet', role = 'wan', endpoint = { ifname = 'eth2' }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, + dhcp = { enabled = false }, firewall = { zone = 'wan' }, + }, + wanc = { + kind = 'ethernet', role = 'wan', endpoint = { ifname = 'eth3' }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, + dhcp = { enabled = false }, firewall = { zone = 'wan' }, + }, + }, + dns = { + enabled = true, + domain = 'vm.bigbox.test', + upstreams = { '1.1.1.1', '8.8.8.8' }, + cache = { size = 1000 }, + records = { router = { name = 'config.vm.bigbox.test', address = '192.168.1.1' } }, + }, + dhcp = { defaults = { authoritative = true } }, + firewall = { + defaults = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT' }, + zones = { + lan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + wan = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT', masq = true, mtu_fix = true }, + }, + policies = { lan_to_wan = { src = 'lan', dest = 'wan' } }, + }, + routing = { routes = {} }, + wan = { + enabled = true, + load_balancing = { policy = 'balanced', speedtests = true }, + last_resort = 'unreachable', + health = { family = 'ipv4', reliability = 1, count = 1, timeout = 1, interval = 2, up = 1, down = 1, initial_state = 'online' }, + rules = { + https = { family = 'ipv4', proto = 'tcp', dest_port = '443', policy = 'balanced', sticky = true }, + }, + members = { + wan = { interface = 'wan', mwan_metric = 1, weight = 50, track_ip = '172.31.1.2' }, + wanb = { interface = 'wanb', mwan_metric = 1, weight = 30, track_ip = '172.31.2.2' }, + wanc = { interface = 'wanc', mwan_metric = 1, weight = 20, track_ip = '172.31.3.2' }, + }, + }, + vpn = {}, diagnostics = {}, + } +end + +return M diff --git a/tests/integration/openwrt_vm/fixtures/mwan3-balanced b/tests/integration/openwrt_vm/fixtures/mwan3-balanced new file mode 100644 index 00000000..612d60a4 --- /dev/null +++ b/tests/integration/openwrt_vm/fixtures/mwan3-balanced @@ -0,0 +1,63 @@ +# Three-WAN deterministic MWAN3 fixture for OpenWrt VM tests. +# The VM default attaches three extra QEMU user-network NICs, normally seen as: +# wan -> eth1 +# wanb -> eth2 +# wanc -> eth3 +# No track_ip is configured so the fixture does not depend on external +# connectivity; the live-weight tests exercise policy-chain updates. + +config globals 'globals' + option mmx_mask '0x3F00' + +config interface 'wan' + option enabled '1' + option family 'ipv4' + option initial_state 'online' + +config interface 'wanb' + option enabled '1' + option family 'ipv4' + option initial_state 'online' + +config interface 'wanc' + option enabled '1' + option family 'ipv4' + option initial_state 'online' + +config member 'wan_m1_w1' + option interface 'wan' + option metric '1' + option weight '1' + +config member 'wanb_m1_w1' + option interface 'wanb' + option metric '1' + option weight '1' + +config member 'wanc_m1_w1' + option interface 'wanc' + option metric '1' + option weight '1' + +config policy 'wan_only' + list use_member 'wan_m1_w1' + option last_resort 'unreachable' + +config policy 'wanb_only' + list use_member 'wanb_m1_w1' + option last_resort 'unreachable' + +config policy 'wanc_only' + list use_member 'wanc_m1_w1' + option last_resort 'unreachable' + +config policy 'balanced' + list use_member 'wan_m1_w1' + list use_member 'wanb_m1_w1' + list use_member 'wanc_m1_w1' + option last_resort 'unreachable' + +config rule 'default_rule_v4' + option dest_ip '0.0.0.0/0' + option use_policy 'balanced' + option family 'ipv4' diff --git a/tests/integration/openwrt_vm/images/.gitignore b/tests/integration/openwrt_vm/images/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/tests/integration/openwrt_vm/images/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/integration/openwrt_vm/readme.md b/tests/integration/openwrt_vm/readme.md new file mode 100644 index 00000000..0bb780fb --- /dev/null +++ b/tests/integration/openwrt_vm/readme.md @@ -0,0 +1,319 @@ +# OpenWrt VM integration tests + +This directory provides an opt-in OpenWrt VM test lane for Devicecode integration work. + +It is intended for HAL and networking tests that need a real OpenWrt userspace and kernel networking stack, including UCI, `fw4`/nftables, `tc`, IFB and veth devices. It is not part of the normal unit test path. + +## Layout + +```text +tests/integration/openwrt_vm + env.sh + Makefile + scripts/ + tests/ +```` + +The VM uses QEMU. A QEMU user-mode NIC is used for SSH and provisioning. By default, three additional QEMU user-mode NICs are attached for deterministic WAN fixtures, normally appearing inside OpenWrt as `eth1`, `eth2` and `eth3`. Optional tap NICs may be added separately for stronger host-bridged dataplane tests. + +## Requirements + +On the host: + +```text +qemu-system-x86_64 or qemu-system-aarch64 +qemu-img +curl +gzip +ssh +scp +sha256sum or shasum +``` + +On AArch64 hosts, QEMU EFI firmware is also required. + +Check the local setup with: + +```sh +make preflight +``` + +## Basic workflow + +From this directory: + +```sh +make fetch +make reset +make run +make provision +make test +``` + +Or run the full lane: + +```sh +make openwrt-vm-test +``` + +The default test target currently checks: + +```text +OpenWrt baseline tools and state +veth, IFB, HTB, fq_codel and ingress qdisc support +Lua UCI and Devicecode UCI manager behaviour, including async activation ownership +network provider apply/snapshot/observer behaviour, including async OpenWrt activation contract +static scan for fibres-unaware OS/IO calls in NET/OpenWrt HAL paths +VLAN/MWAN/shaping provider behaviour +MWAN3 live-weight updates via iptables-restore --noflush +SCP copy to and from the VM +``` + +## Common commands + +```sh +make run # start the VM +make wait # wait until SSH is ready +make ssh # open a root SSH session +make smoke # show basic VM state +make provision # install packages needed by the tests +make test # run current VM tests +make logs # show recent serial log output +make stop # stop the VM +make clean # stop and remove the work disk +``` + +Force package provisioning to run again: + +```sh +OPENWRT_PROVISION_FORCE=1 make provision +``` + +Use a shorter SSH wait while iterating: + +```sh +OPENWRT_SSH_WAIT_S=30 make smoke +``` + +## Image and disk handling + +Images are downloaded into: + +```text +images/ +``` + +The downloaded image is verified against OpenWrt `sha256sums` before use. + +The writable VM disk is a qcow2 overlay in: + +```text +work/ +``` + +Reset it with: + +```sh +make reset +``` + +This discards changes made inside the VM. + +## Dataplane testing + +The default tests create veth and IFB devices inside the VM. This is enough for many kernel traffic-control tests. + +For the default MWAN fixture, three user-mode WAN NICs are attached automatically. Override the count with `OPENWRT_VM_WAN_IFACES` if needed: + +```sh +OPENWRT_VM_WAN_IFACES=3 make run +``` + +For tests requiring host-side dataplane interfaces, provide tap devices explicitly. Each tap becomes an additional virtio NIC after the management and default WAN NICs, which allows MWAN/VLAN experiments with multiple host-bridged WAN or LAN surfaces: + +```sh +OPENWRT_TAP_IFACES="tap-dc0 tap-dc1 tap-dc2" make run +``` + +The management SSH NIC remains separate from any tap dataplane NICs. The MWAN live-weight tests are still self-contained by default, but the tap path is available for fuller end-to-end routing and distribution tests. + +## Notes + +This lane is intentionally separate from the normal unit suite. It is for tests where container-based execution is not representative, especially OpenWrt HAL work involving UCI, networking reloads, nftables, traffic control, IFB, veth and future shaping or bandwidth-estimation tests. + +Keep service logic tests outside this lane where possible. Use this VM lane for behaviour that depends on OpenWrt or kernel networking semantics. +Additional real-VM network tests apply a Devicecode-generated config directly to `/etc/config` while preserving the QEMU management LAN on `eth0` and using the deterministic VM WAN NICs `eth1`, `eth2` and `eth3`. They assert both the generated UCI shape and that mwan3 is active over the VM WAN links. + + + +### Render generated default OpenWrt config files + +To see the exact `/etc/config/*` files generated from `src/configs/bigbox-v1-cm-2.json` without modifying the VM's real `/etc/config`, run: + +```sh +make render-default-configs +``` + +The files are copied to: + +```text +tests/integration/openwrt_vm/work/generated-default-etc-config/ +``` + +For a stdout dump as well, run: + +```sh +make print-default-configs +``` + +The renderer runs the real OpenWrt provider against a temporary UCI confdir. By default it uses `eth0` as the segment trunk and no GSM uplink facts, so only wired WAN is realised. To render with modem interfaces realised, set: + +```sh +DEVICECODE_RENDER_GSM_PRIMARY_IFNAME=wwan0 DEVICECODE_RENDER_GSM_SECONDARY_IFNAME=wwan1 make print-default-configs +``` + +### Host bridge client on internal VLAN 100 + +For debugging real client behaviour on the internal segment, an optional target +builds a host-side Linux bridge dataplane and starts the VM with a tap-backed +trunk NIC: + +```sh +make test-openwrt-int-bridge-client-dhcp-dns +``` + +This target is intentionally not part of `make test`. It needs host networking +privileges because it creates Linux bridges, veth pairs, network namespaces and +QEMU tap devices. Prefer `make network-lab-test` in CI or from an unprivileged +devcontainer; run this target directly only on a host that grants those +networking permissions. The harness will try to provision common Debian/Ubuntu +packages such as `iproute2`, `udhcpc`, `dnsutils`, `wget` and `tcpdump` when +they are missing. + +The fixture uses: + +```text +Linux bridge: brdcint +QEMU tap trunk: tapdcint, VLAN 100 tagged toward the VM +client namespace: dcintc +client port: vdcinth as an untagged access port on VLAN 100 +OpenWrt trunk: eth4 by default, derived from OPENWRT_VM_WAN_IFACES + 1 +OpenWrt segment: int / br-int / vl-int / 172.28.100.1/24 +``` + +The test applies a small Devicecode-generated OpenWrt config, obtains a DHCP +lease from a host network namespace over VLAN 100, and checks router reachability, +public-IP reachability, local DNS, public DNS and a public HTTP fetch from that +namespace. On failure it dumps both host bridge/client state and OpenWrt network, +DHCP and dnsmasq diagnostics. + +Remove the host-side fixture with: + +```sh +make teardown-bridge-client-fabric +``` + + +### Network-lab VM tier + +The host-bridge/client targets need privileges that ordinary CI containers +usually should not have: Linux bridges, tap devices, veth pairs and network +namespaces. The `network-lab` tier runs those tests inside a disposable Linux VM +instead, so the top-level devcontainer and fast CI jobs can remain unprivileged. + +Default lab workflow: + +```sh +make network-lab-test +``` + +This will: + +```text +1. boot a Debian cloud-image VM with QEMU user-mode SSH forwarding +2. provision qemu, iproute2, tcpdump, dnsutils and related tools inside the lab +3. rsync the repository into the lab VM +4. run the privileged OpenWrt dataplane targets inside the lab VM +``` + +The default lab targets are: + +```text +test-openwrt-int-bridge-client-dhcp-dns +test-openwrt-dnsmasq-multi-instance-resilience +``` + +Run a specific target inside the lab with: + +```sh +./scripts/network-lab-test test-openwrt-dnsmasq-multi-instance-resilience +``` + +Useful lab controls: + +```sh +make network-lab-start # boot the lab VM +make network-lab-wait # wait for lab SSH +make network-lab-provision # install lab packages +make network-lab-sync # rsync the repo into the lab +make network-lab-ssh # open a shell in the lab VM +make network-lab-stop # stop the lab VM +``` + +Important environment knobs: + +```text +NETWORK_LAB_SSH_PORT=2242 +NETWORK_LAB_MEM=4096M +NETWORK_LAB_CPUS=2 +NETWORK_LAB_KVM=auto +NETWORK_LAB_ARCH=auto # native by default: amd64 on x86_64, arm64 on aarch64 +NETWORK_LAB_TEST_TARGETS="test-openwrt-int-bridge-client-dhcp-dns test-openwrt-dnsmasq-multi-instance-resilience" +NETWORK_LAB_OPENWRT_SSH_WAIT_S=600 # nested OpenWrt VM boot budget inside the lab +NETWORK_LAB_OPENWRT_KVM=auto # passed through to the nested OpenWrt VM lane +NETWORK_LAB_BASE_IMAGE_URL=auto # Debian genericcloud image matching NETWORK_LAB_ARCH +``` + +The lab VM is deliberately a separate integration tier. Keep fast unit/provider +tests outside it; use it only for tests where host kernel networking permissions +or platform isolation matter. + +The OpenWrt VM runs nested inside this lab VM. If KVM is unavailable either to +the outer lab VM or to the nested OpenWrt VM, the lab remains useful as a +permission boundary but boots much more slowly; the lab therefore uses a larger +nested OpenWrt SSH wait budget than the direct local `openwrt-vm-test` lane. + + +### Network-lab image/package footprint + +The network-lab uses a Debian `genericcloud` image matching the host CPU by +default: `amd64` on x86_64 hosts and `arm64` on AArch64 hosts. This keeps the +lab VM native and lets the nested OpenWrt VM choose the matching OpenWrt target +through the normal `env.sh` architecture detection. + +The lab provisioning intentionally installs packages with `--no-install-recommends`. +Some GUI-looking libraries can still appear because they are hard dependencies of +Debian's QEMU packages, not because the lab image is a desktop image. + +The lab scripts also normalise `PATH` to include `/usr/sbin` and `/sbin`. On +minimal cloud images, tools such as iproute2's `bridge` can be installed but not +visible to the non-login SSH user unless those directories are added explicitly. + +### Full service graph with mocked HAL capabilities + +`make test-devicecode-full-stack-mock-hal` copies the current source tree into +the OpenWrt VM and runs the real Devicecode services in-process against a bus +with mocked HAL capability providers. It deliberately excludes the production +`hal` service and replaces the HAL boundary with public capability topics for +filesystem, network, control-store, artifact-store, time and platform. The test +boots config, device, fabric, gsm, http, metrics, monitor, net, system, time, +ui, update, wifi and wired from their normal source modules. + +The fixture asserts that configuration is loaded through the filesystem +capability, the broad service graph reaches retained running/ready state, wired +public state is projected, HTTP capability state is retained, UI runs with raw +HAL topics excluded by default, and network apply requests are serialised by the +mock HAL boundary while config churn is in flight. This lane is intended to +catch shared-infrastructure regressions in service lifecycle, retained publish, +capability dependency handling, config watch and cancellation semantics under a +real OpenWrt Lua runtime. diff --git a/tests/integration/openwrt_vm/scripts/backup-openwrt-config-state b/tests/integration/openwrt_vm/scripts/backup-openwrt-config-state new file mode 100755 index 00000000..de435a14 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/backup-openwrt-config-state @@ -0,0 +1,22 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +SSH="$SCRIPT_DIR/ssh" +STATE="${DEVICECODE_CONFIG_STATE:-/tmp/devicecode-openwrt-config-backup}" + +"$SSH" "CONFIG_STATE='$STATE' sh -s" <<'REMOTE_BACKUP' +set -eu +rm -rf "$CONFIG_STATE" +mkdir -p "$CONFIG_STATE" +: > "$CONFIG_STATE/present" +: > "$CONFIG_STATE/absent" +for name in network dhcp firewall mwan3; do + if [ -e "/etc/config/$name" ]; then + cp -a "/etc/config/$name" "$CONFIG_STATE/$name" + echo "$name" >> "$CONFIG_STATE/present" + else + echo "$name" >> "$CONFIG_STATE/absent" + fi +done +REMOTE_BACKUP diff --git a/tests/integration/openwrt_vm/scripts/cleanup-shaping-state b/tests/integration/openwrt_vm/scripts/cleanup-shaping-state new file mode 100755 index 00000000..3766b89f --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/cleanup-shaping-state @@ -0,0 +1,65 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +SSH="$VM_DIR/scripts/ssh" + +if [ "${DC_SHAPING_CLEANUP_DISABLED:-0}" = 1 ]; then + exit 0 +fi + +"$SSH" 'set +e +# Remove Devicecode-owned shaping mangle chains and jumps. Loop deletions so +# interrupted/retried tests cannot leave duplicate jump rules behind. +for tbl in mangle; do + for chain in OUTPUT FORWARD POSTROUTING PREROUTING; do + while iptables -t "$tbl" -D "$chain" -j DEVICECODE_SHAPING_OUTPUT 2>/dev/null; do :; done + while iptables -t "$tbl" -D "$chain" -j DEVICECODE_SHAPING_FORWARD 2>/dev/null; do :; done + done + iptables -t "$tbl" -F DEVICECODE_SHAPING_OUTPUT 2>/dev/null + iptables -t "$tbl" -F DEVICECODE_SHAPING_FORWARD 2>/dev/null + iptables -t "$tbl" -X DEVICECODE_SHAPING_OUTPUT 2>/dev/null + iptables -t "$tbl" -X DEVICECODE_SHAPING_FORWARD 2>/dev/null +done + +# Remove shaping qdiscs from deterministic VM test devices. Avoid broad globs +# over production-looking interfaces; these names are owned by the VM tests. +for dev in \ + br-jan vl-jan ifb_br_jan ifb_vl_jan \ + dcjantrunk0 dcjantrunk0p dc-jan-host dc-jan-client0 \ + dcshape-budget dcshape-budgetp \ + dcs-bflat dcs-bflatp \ + dcs-bagg dcs-baggp \ + dcshape-borrow dcshape-borrowp \ + dcshape-idem dcshape-idemp ifb_dcs_idem \ + dcshape-old dcshape-oldp ifb_dcshape_old \ + dcwan0 dcwan-up0 dcwan-cap dcwan-capp \ + dcclient-host dcclient0 ifb_dcwan0 ifb_dcwan_cap ifb_dcwan_c; do + tc qdisc del dev "$dev" root 2>/dev/null + tc qdisc del dev "$dev" ingress 2>/dev/null +done + +# Remove test namespaces and test links. Deleting the root-side veth usually +# removes the peer as well, but peers are listed here for interrupted runs. +for ns in dcwan-upstream dcwan-client dcwan-cap-peer dcshape-idem-peer dc-jan-shape-client; do + ip netns del "$ns" 2>/dev/null +done +for dev in \ + dcjantrunk0 dcjantrunk0p dc-jan-host dc-jan-client0 \ + dcshape-budget dcshape-budgetp \ + dcs-bflat dcs-bflatp \ + dcs-bagg dcs-baggp \ + dcshape-borrow dcshape-borrowp \ + dcshape-idem dcshape-idemp \ + dcshape-old dcshape-oldp \ + dcwan0 dcwan-up0 dcclient-host dcclient0 dcwan-cap dcwan-capp \ + ifb_br_jan ifb_vl_jan ifb_dcs_idem ifb_dcshape_old ifb_dcwan0 ifb_dcwan_cap ifb_dcwan_c; do + ip link del "$dev" 2>/dev/null +done + +# Best-effort cleanups for tests that inserted temporary routing/filtering rules. +while iptables -D FORWARD -i dcclient-host -o dcwan0 -j ACCEPT 2>/dev/null; do :; done +while iptables -D FORWARD -i dcwan0 -o dcclient-host -j ACCEPT 2>/dev/null; do :; done +while iptables -t mangle -D OUTPUT -o dcwan0 -m comment --comment devicecode-test-client-mark -j MARK --set-xmark 0x00200000/0x00f00000 2>/dev/null; do :; done +' diff --git a/tests/integration/openwrt_vm/scripts/ensure-devicecode-staged b/tests/integration/openwrt_vm/scripts/ensure-devicecode-staged new file mode 100755 index 00000000..688549a4 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/ensure-devicecode-staged @@ -0,0 +1,32 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="${DEVICECODE_REMOTE_STAGE:-/tmp/devicecode-current}" +WORK="$VM_DIR/work" +STAMP_FILE="$WORK/devicecode-stage.stamp" + +mkdir -p "$WORK" + +# Use metadata rather than file contents so the check is cheap during local +# development. It is deliberately conservative: touching a source file causes +# a re-stage. +STAMP="$({ cd "$ROOT_DIR" && find src vendor -type f -printf '%p %s %T@\n' | sort; } | sha256sum | awk '{print $1}')" +printf '%s\n' "$STAMP" > "$STAMP_FILE" + +if [ "${DEVICECODE_STAGE_FORCE:-0}" != 1 ] && \ + "$SSH" "test -f '$REMOTE/.devicecode-stage-stamp' && grep -qx '$STAMP' '$REMOTE/.devicecode-stage-stamp'" >/dev/null 2>&1; then + printf '%s\n' "$REMOTE" + exit 0 +fi + +printf '[openwrt-vm] staging Devicecode source to %s\n' "$REMOTE" >&2 +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SSH" "printf '%s\\n' '$STAMP' > '$REMOTE/.devicecode-stage-stamp'" +printf '%s\n' "$REMOTE" diff --git a/tests/integration/openwrt_vm/scripts/ensure-jan-client-dhcp-dns-deps b/tests/integration/openwrt_vm/scripts/ensure-jan-client-dhcp-dns-deps new file mode 100755 index 00000000..57ecc1e8 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/ensure-jan-client-dhcp-dns-deps @@ -0,0 +1,75 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +"$SCRIPT_DIR/wait-ssh" + +echo "[openwrt-vm] checking jan client network-data-plane test dependencies" + +"$SCRIPT_DIR/ssh" ' + set -eu + + fail_missing() { + echo "jan client dependency check failed: $*" >&2 + echo "Run: make provision-force" >&2 + exit 1 + } + + cleanup_probe() { + ip netns del dcdepns >/dev/null 2>&1 || true + ip link del dcdep0 2>/dev/null || true + ip link del dcdep0p 2>/dev/null || true + ip link del dcdep0.32 2>/dev/null || true + ip link del dcdepvlan 2>/dev/null || true + ip link del dcdepifb 2>/dev/null || true + } + trap cleanup_probe EXIT INT TERM + + missing_cmd="" + for x in ip udhcpc nslookup pgrep tc; do + command -v "$x" >/dev/null 2>&1 || missing_cmd="$missing_cmd $x" + done + [ -z "$missing_cmd" ] || fail_missing "missing commands:$missing_cmd" + + # Load modules if available. Provisioning is responsible for installing + # packages; this check must not install packages, so missing capability is a + # deterministic provision failure rather than a hidden per-test mutation. + modprobe ifb 2>/dev/null || true + modprobe 8021q 2>/dev/null || true + modprobe sch_htb 2>/dev/null || true + modprobe sch_ingress 2>/dev/null || true + modprobe sch_fq_codel 2>/dev/null || true + modprobe cls_u32 2>/dev/null || true + modprobe act_mirred 2>/dev/null || true + + cleanup_probe + ip link add dcdep0 type veth peer name dcdep0p >/dev/null 2>&1 || \ + fail_missing "veth devices are unavailable; expected kmod-veth/ip-full" + + ip link set dcdep0 up >/dev/null 2>&1 || true + ip link add link dcdep0 name dcdepvlan type vlan id 32 >/dev/null 2>&1 || \ + fail_missing "8021q VLAN devices are unavailable; expected built-in VLAN support or kmod-8021q" + + ip link add dcdepifb type ifb >/dev/null 2>&1 || \ + fail_missing "ifb devices are unavailable; expected kmod-ifb" + ip link set dcdepifb up >/dev/null 2>&1 || true + tc qdisc add dev dcdepifb root handle 1: htb default 1 >/dev/null 2>&1 || \ + fail_missing "HTB qdisc is unavailable; expected tc-full/kmod-sched/kmod-sched-core" + tc qdisc add dev dcdep0 ingress >/dev/null 2>&1 || \ + fail_missing "ingress qdisc is unavailable; expected tc-full/kmod-sched-core" + + if ! ip netns add dcdepns >/dev/null 2>&1; then + echo "[openwrt-vm] network namespaces unavailable; jan test will use same-namespace veth fallback" >&2 + else + ip netns del dcdepns >/dev/null 2>&1 || true + fi + + cleanup_probe +' + +echo "[openwrt-vm] jan client network-data-plane dependencies ready" diff --git a/tests/integration/openwrt_vm/scripts/ensure-mwan3 b/tests/integration/openwrt_vm/scripts/ensure-mwan3 new file mode 100755 index 00000000..abc63c05 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/ensure-mwan3 @@ -0,0 +1,24 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +"$SCRIPT_DIR/wait-ssh" + +"$SCRIPT_DIR/ssh" ' + set -eu + missing="" + for x in mwan3 iptables-save iptables-restore; do + command -v "$x" >/dev/null 2>&1 || missing="$missing $x" + done + [ -x /etc/init.d/mwan3 ] || missing="$missing /etc/init.d/mwan3" + if [ -n "$missing" ]; then + echo "[openwrt-vm] missing MWAN dependencies:$missing" >&2 + echo "[openwrt-vm] run: make provision-force" >&2 + exit 1 + fi +' diff --git a/tests/integration/openwrt_vm/scripts/fetch-image b/tests/integration/openwrt_vm/scripts/fetch-image new file mode 100755 index 00000000..4b0a5beb --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/fetch-image @@ -0,0 +1,48 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +mkdir -p "${OPENWRT_IMAGE_DIR}" + +if [ ! -f "${OPENWRT_IMAGE_GZ}" ]; then + echo "[openwrt-vm] fetching ${OPENWRT_IMAGE_NAME}" + curl -fL "${OPENWRT_BASE_URL}/${OPENWRT_IMAGE_NAME}" -o "${OPENWRT_IMAGE_GZ}" +fi + +"$SCRIPT_DIR/verify-image" + +if [ ! -f "${OPENWRT_IMAGE_RAW}" ]; then + echo "[openwrt-vm] decompressing ${OPENWRT_IMAGE_NAME}" + tmp="${OPENWRT_IMAGE_RAW}.tmp" + rm -f "$tmp" + + set +e + gzip -dc "${OPENWRT_IMAGE_GZ}" > "$tmp" + rc=$? + set -e + + if [ "$rc" -ne 0 ] && [ "$rc" -ne 2 ]; then + rm -f "$tmp" + echo "[openwrt-vm] gzip failed with exit code $rc" >&2 + exit "$rc" + fi + + if [ ! -s "$tmp" ]; then + rm -f "$tmp" + echo "[openwrt-vm] decompressed image is empty" >&2 + exit 1 + fi + + if [ "$rc" -eq 2 ]; then + echo "[openwrt-vm] gzip reported trailing data; using decompressed image anyway" >&2 + fi + + mv "$tmp" "${OPENWRT_IMAGE_RAW}" +fi + +echo "[openwrt-vm] image ready: ${OPENWRT_IMAGE_RAW}" diff --git a/tests/integration/openwrt_vm/scripts/network-lab-create-seed b/tests/integration/openwrt_vm/scripts/network-lab-create-seed new file mode 100755 index 00000000..696977e4 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/network-lab-create-seed @@ -0,0 +1,59 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/network-lab-env" + +mkdir -p "$NETWORK_LAB_DIR" + +if [ ! -f "$NETWORK_LAB_SSH_KEY" ]; then + echo "[network-lab] creating SSH key: $NETWORK_LAB_SSH_KEY" + ssh-keygen -q -t ed25519 -N '' -f "$NETWORK_LAB_SSH_KEY" +fi + +pubkey="$(cat "${NETWORK_LAB_SSH_KEY}.pub")" +seed_dir="${NETWORK_LAB_DIR}/seed" +rm -rf "$seed_dir" +mkdir -p "$seed_dir" + +cat > "${seed_dir}/user-data" < "${seed_dir}/meta-data" </dev/null 2>&1; then + cloud-localds "$NETWORK_LAB_SEED" "${seed_dir}/user-data" "${seed_dir}/meta-data" +elif command -v genisoimage >/dev/null 2>&1; then + genisoimage -quiet -output "$NETWORK_LAB_SEED" -volid cidata -joliet -rock "${seed_dir}/user-data" "${seed_dir}/meta-data" +elif command -v mkisofs >/dev/null 2>&1; then + mkisofs -quiet -output "$NETWORK_LAB_SEED" -volid cidata -joliet -rock "${seed_dir}/user-data" "${seed_dir}/meta-data" +elif command -v xorrisofs >/dev/null 2>&1; then + xorrisofs -quiet -output "$NETWORK_LAB_SEED" -volid cidata -joliet -rock "${seed_dir}/user-data" "${seed_dir}/meta-data" +else + cat >&2 <<'MSG' +[network-lab] missing cloud-init seed tool. +[network-lab] Install one of: cloud-localds, genisoimage, mkisofs, xorrisofs. +MSG + exit 1 +fi + +echo "[network-lab] seed image ready: $NETWORK_LAB_SEED" diff --git a/tests/integration/openwrt_vm/scripts/network-lab-env b/tests/integration/openwrt_vm/scripts/network-lab-env new file mode 100755 index 00000000..08e301cb --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/network-lab-env @@ -0,0 +1,80 @@ +#!/usr/bin/env sh +# Shared configuration for the optional OpenWrt network-lab VM. +# +# The lab VM is a privilege boundary for dataplane integration tests that need +# bridges, tap devices, veth pairs and network namespaces. Ordinary CI jobs can +# stay unprivileged and delegate only these tests to the lab. + +if [ -z "${VM_DIR:-}" ]; then + VM_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +fi + +# Network-lab commands run as a non-login cloud user. Keep /usr/sbin and /sbin +# visible so iproute2's `bridge`, modprobe-adjacent helpers, etc. are found. +PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}" +export PATH + +OPENWRT_DIR="${VM_DIR}" +OPENWRT_WORK_DIR="${OPENWRT_WORK_DIR:-${OPENWRT_DIR}/work}" + +NETWORK_LAB_DIR="${NETWORK_LAB_DIR:-${OPENWRT_WORK_DIR}/network-lab}" +NETWORK_LAB_IMAGE_DIR="${NETWORK_LAB_IMAGE_DIR:-${NETWORK_LAB_DIR}/images}" + +if [ -z "${NETWORK_LAB_ARCH:-}" ] || [ "${NETWORK_LAB_ARCH:-}" = "auto" ]; then + case "$(uname -m)" in + x86_64|amd64) NETWORK_LAB_ARCH="amd64" ;; + aarch64|arm64) NETWORK_LAB_ARCH="arm64" ;; + *) + echo "[network-lab] unsupported host architecture for lab VM: $(uname -m)" >&2 + exit 1 + ;; + esac +fi + +case "$NETWORK_LAB_ARCH" in + amd64) + NETWORK_LAB_BASE_IMAGE_NAME="${NETWORK_LAB_BASE_IMAGE_NAME:-debian-12-genericcloud-amd64.qcow2}" + NETWORK_LAB_QEMU="${NETWORK_LAB_QEMU:-qemu-system-x86_64}" + ;; + arm64) + NETWORK_LAB_BASE_IMAGE_NAME="${NETWORK_LAB_BASE_IMAGE_NAME:-debian-12-genericcloud-arm64.qcow2}" + NETWORK_LAB_QEMU="${NETWORK_LAB_QEMU:-qemu-system-aarch64}" + ;; + *) + echo "[network-lab] unsupported NETWORK_LAB_ARCH=$NETWORK_LAB_ARCH" >&2 + exit 1 + ;; +esac + +if [ "${NETWORK_LAB_BASE_IMAGE_URL:-}" = "auto" ]; then + unset NETWORK_LAB_BASE_IMAGE_URL +fi +NETWORK_LAB_BASE_IMAGE_URL="${NETWORK_LAB_BASE_IMAGE_URL:-https://cloud.debian.org/images/cloud/bookworm/latest/${NETWORK_LAB_BASE_IMAGE_NAME}}" +NETWORK_LAB_BASE_IMAGE="${NETWORK_LAB_BASE_IMAGE:-${NETWORK_LAB_IMAGE_DIR}/${NETWORK_LAB_BASE_IMAGE_NAME}}" +NETWORK_LAB_DISK="${NETWORK_LAB_DISK:-${NETWORK_LAB_DIR}/network-lab-${NETWORK_LAB_ARCH}.qcow2}" +NETWORK_LAB_SEED="${NETWORK_LAB_SEED:-${NETWORK_LAB_DIR}/seed.iso}" +NETWORK_LAB_PID="${NETWORK_LAB_PID:-${NETWORK_LAB_DIR}/qemu.pid}" +NETWORK_LAB_LOG="${NETWORK_LAB_LOG:-${NETWORK_LAB_DIR}/qemu.log}" +NETWORK_LAB_SSH_KEY="${NETWORK_LAB_SSH_KEY:-${NETWORK_LAB_DIR}/id_ed25519}" +NETWORK_LAB_SSH_PORT="${NETWORK_LAB_SSH_PORT:-2242}" +NETWORK_LAB_SSH_WAIT_S="${NETWORK_LAB_SSH_WAIT_S:-180}" +NETWORK_LAB_USER="${NETWORK_LAB_USER:-dc}" +NETWORK_LAB_HOST="${NETWORK_LAB_HOST:-127.0.0.1}" +NETWORK_LAB_HOSTNAME="${NETWORK_LAB_HOSTNAME:-dc-network-lab}" +NETWORK_LAB_MEM="${NETWORK_LAB_MEM:-4096M}" +NETWORK_LAB_CPUS="${NETWORK_LAB_CPUS:-2}" +NETWORK_LAB_DISK_SIZE="${NETWORK_LAB_DISK_SIZE:-30G}" +NETWORK_LAB_KVM="${NETWORK_LAB_KVM:-auto}" +NETWORK_LAB_REPO_DIR="${NETWORK_LAB_REPO_DIR:-/home/${NETWORK_LAB_USER}/dc-lua}" +NETWORK_LAB_TEST_TARGETS="${NETWORK_LAB_TEST_TARGETS:-test-openwrt-int-bridge-client-dhcp-dns test-openwrt-dnsmasq-multi-instance-resilience}" +# The OpenWrt VM runs nested inside the lab VM. When nested KVM is unavailable, +# booting through QEMU emulation can be much slower than the direct local +# openwrt-vm-test lane, so use a lab-specific default wait budget. +NETWORK_LAB_OPENWRT_SSH_WAIT_S="${NETWORK_LAB_OPENWRT_SSH_WAIT_S:-600}" +NETWORK_LAB_OPENWRT_KVM="${NETWORK_LAB_OPENWRT_KVM:-auto}" +NETWORK_LAB_RSYNC_EXCLUDES="${NETWORK_LAB_RSYNC_EXCLUDES:-.git tests/integration/openwrt_vm/work tests/integration/openwrt_vm/images}" + +NETWORK_LAB_SSH_OPTS="-p ${NETWORK_LAB_SSH_PORT} -i ${NETWORK_LAB_SSH_KEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5" +NETWORK_LAB_SSH="ssh ${NETWORK_LAB_SSH_OPTS} ${NETWORK_LAB_USER}@${NETWORK_LAB_HOST}" +NETWORK_LAB_SCP="scp ${NETWORK_LAB_SSH_OPTS}" +NETWORK_LAB_RSYNC_RSH="ssh ${NETWORK_LAB_SSH_OPTS}" diff --git a/tests/integration/openwrt_vm/scripts/network-lab-fetch-image b/tests/integration/openwrt_vm/scripts/network-lab-fetch-image new file mode 100755 index 00000000..34400620 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/network-lab-fetch-image @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/network-lab-env" + +mkdir -p "$NETWORK_LAB_IMAGE_DIR" + +if [ ! -f "$NETWORK_LAB_BASE_IMAGE" ]; then + echo "[network-lab] fetching base image: $NETWORK_LAB_BASE_IMAGE_URL" + tmp="${NETWORK_LAB_BASE_IMAGE}.tmp" + rm -f "$tmp" + curl -fL "$NETWORK_LAB_BASE_IMAGE_URL" -o "$tmp" + mv "$tmp" "$NETWORK_LAB_BASE_IMAGE" +fi + +echo "[network-lab] base image ready: $NETWORK_LAB_BASE_IMAGE" diff --git a/tests/integration/openwrt_vm/scripts/network-lab-provision b/tests/integration/openwrt_vm/scripts/network-lab-provision new file mode 100755 index 00000000..07836e2f --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/network-lab-provision @@ -0,0 +1,63 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/network-lab-env" + +sh "$SCRIPT_DIR/network-lab-wait" + +$NETWORK_LAB_SSH 'set -eu +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}" +if [ -e /etc/devicecode-network-lab-provisioned ]; then + echo "[network-lab] provision already complete" + exit 0 +fi +arch="$(dpkg --print-architecture)" +case "$arch" in + amd64) + qemu_packages="qemu-system-x86" + ;; + arm64) + qemu_packages="qemu-system-arm qemu-efi-aarch64" + ;; + *) + echo "[network-lab] unsupported lab architecture: $arch" >&2 + exit 1 + ;; +esac +sudo apt-get update +# shellcheck disable=SC2086 +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + gzip \ + openssh-client \ + rsync \ + socat \ + $qemu_packages \ + qemu-utils \ + make \ + iproute2 \ + iputils-ping \ + busybox \ + dnsutils \ + wget \ + tcpdump \ + nftables \ + sudo +command -v ip >/dev/null 2>&1 || { echo "[network-lab] missing ip after iproute2 install" >&2; exit 1; } +command -v bridge >/dev/null 2>&1 || { echo "[network-lab] missing bridge after iproute2 install; PATH=$PATH" >&2; exit 1; } +sudo modprobe tun 2>/dev/null || true +if [ ! -e /dev/net/tun ]; then + sudo mkdir -p /dev/net + sudo mknod /dev/net/tun c 10 200 2>/dev/null || true + sudo chmod 0666 /dev/net/tun 2>/dev/null || true +fi +probe="dclab$$" +sudo ip link add name "$probe" type bridge +sudo ip link del "$probe" +sudo touch /etc/devicecode-network-lab-provisioned +echo "[network-lab] provision complete" +' diff --git a/tests/integration/openwrt_vm/scripts/network-lab-ssh b/tests/integration/openwrt_vm/scripts/network-lab-ssh new file mode 100755 index 00000000..b56b4010 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/network-lab-ssh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/network-lab-env" + +exec $NETWORK_LAB_SSH "$@" diff --git a/tests/integration/openwrt_vm/scripts/network-lab-start b/tests/integration/openwrt_vm/scripts/network-lab-start new file mode 100755 index 00000000..aab7f7d3 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/network-lab-start @@ -0,0 +1,103 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/network-lab-env" + +if [ -f "$NETWORK_LAB_PID" ] && kill -0 "$(cat "$NETWORK_LAB_PID")" 2>/dev/null; then + echo "[network-lab] already running with pid $(cat "$NETWORK_LAB_PID")" + exit 0 +fi + +if command -v ss >/dev/null 2>&1 && ss -H -ltn "sport = :${NETWORK_LAB_SSH_PORT}" 2>/dev/null | grep -q .; then + echo "[network-lab] SSH port ${NETWORK_LAB_SSH_PORT} is already in use" >&2 + echo "[network-lab] run ./scripts/network-lab-stop, or set NETWORK_LAB_SSH_PORT to a free port" >&2 + exit 1 +fi + +if [ ! -f "$NETWORK_LAB_BASE_IMAGE" ]; then + sh "$SCRIPT_DIR/network-lab-fetch-image" +fi + +if [ ! -f "$NETWORK_LAB_SEED" ] || [ ! -f "$NETWORK_LAB_SSH_KEY" ]; then + sh "$SCRIPT_DIR/network-lab-create-seed" +fi + +mkdir -p "$NETWORK_LAB_DIR" + +if [ ! -f "$NETWORK_LAB_DISK" ]; then + echo "[network-lab] creating overlay disk: $NETWORK_LAB_DISK" + qemu-img create -f qcow2 -F qcow2 -b "$NETWORK_LAB_BASE_IMAGE" "$NETWORK_LAB_DISK" "$NETWORK_LAB_DISK_SIZE" >/dev/null +fi + +rm -f "$NETWORK_LAB_PID" "$NETWORK_LAB_LOG" + +set -- \ + -m "$NETWORK_LAB_MEM" \ + -smp "$NETWORK_LAB_CPUS" \ + -drive "file=${NETWORK_LAB_DISK},format=qcow2,if=virtio" \ + -drive "file=${NETWORK_LAB_SEED},format=raw,media=cdrom,readonly=on" \ + -netdev "user,id=net0,hostfwd=tcp::${NETWORK_LAB_SSH_PORT}-:22" \ + -device virtio-net-pci,netdev=net0 \ + -display none \ + -monitor none \ + -daemonize \ + -pidfile "$NETWORK_LAB_PID" \ + -serial "file:${NETWORK_LAB_LOG}" + +kvm_args="" +case "$NETWORK_LAB_KVM" in + 1) + kvm_args=1 + ;; + auto) + if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + kvm_args=1 + fi + ;; + 0|off|false) + ;; + *) + echo "[network-lab] invalid NETWORK_LAB_KVM=$NETWORK_LAB_KVM" >&2 + exit 1 + ;; +esac + +case "$NETWORK_LAB_ARCH" in + amd64) + if [ -n "$kvm_args" ]; then + set -- -enable-kvm -cpu host "$@" + fi + ;; + arm64) + firmware="" + for p in \ + /usr/share/AAVMF/AAVMF_CODE.fd \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd \ + /usr/share/edk2/aarch64/QEMU_EFI.fd + do + if [ -f "$p" ]; then + firmware="$p" + break + fi + done + if [ -z "$firmware" ]; then + echo "[network-lab] missing AArch64 EFI firmware for arm64 lab VM" >&2 + exit 1 + fi + if [ -n "$kvm_args" ]; then + set -- -enable-kvm -machine virt -cpu host -bios "$firmware" "$@" + else + set -- -machine virt -cpu cortex-a72 -bios "$firmware" "$@" + fi + ;; + *) + echo "[network-lab] unsupported NETWORK_LAB_ARCH=$NETWORK_LAB_ARCH" >&2 + exit 1 + ;; +esac + +echo "[network-lab] starting $NETWORK_LAB_ARCH VM on SSH port $NETWORK_LAB_SSH_PORT" +exec "$NETWORK_LAB_QEMU" "$@" diff --git a/tests/integration/openwrt_vm/scripts/network-lab-stop b/tests/integration/openwrt_vm/scripts/network-lab-stop new file mode 100755 index 00000000..19390ed5 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/network-lab-stop @@ -0,0 +1,46 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/network-lab-env" + +stop_pid() { + pid="$1" + label="$2" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + echo "[network-lab] stopping ${label} pid ${pid}" + kill "$pid" 2>/dev/null || true + for _ in 1 2 3 4 5; do + if ! kill -0 "$pid" 2>/dev/null; then break; fi + sleep 1 + done + if kill -0 "$pid" 2>/dev/null; then + kill -9 "$pid" 2>/dev/null || true + fi + fi +} + +if [ -f "$NETWORK_LAB_PID" ]; then + pid="$(cat "$NETWORK_LAB_PID")" + stop_pid "$pid" "recorded" + rm -f "$NETWORK_LAB_PID" +fi + +# Failed or interrupted lab starts can leave QEMU alive after the pidfile is +# removed. Clean up any qemu-system process owned by this user that still owns +# the lab SSH host-forward port. +if command -v ss >/dev/null 2>&1; then + port_pids="$(ss -H -ltnp "sport = :${NETWORK_LAB_SSH_PORT}" 2>/dev/null \ + | sed -n 's/.*pid=\([0-9][0-9]*\).*/\1/p' \ + | sort -u)" + for pid in $port_pids; do + cmd="$(ps -p "$pid" -o comm= 2>/dev/null || true)" + case "$cmd" in + qemu-system-*) + stop_pid "$pid" "stale port ${NETWORK_LAB_SSH_PORT}" + ;; + esac + done +fi diff --git a/tests/integration/openwrt_vm/scripts/network-lab-sync b/tests/integration/openwrt_vm/scripts/network-lab-sync new file mode 100755 index 00000000..96ca5c40 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/network-lab-sync @@ -0,0 +1,20 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +REPO_ROOT="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/network-lab-env" + +sh "$SCRIPT_DIR/network-lab-wait" + +exclude_args="" +for x in $NETWORK_LAB_RSYNC_EXCLUDES; do + exclude_args="$exclude_args --exclude=$x" +done + +$NETWORK_LAB_SSH "mkdir -p '$NETWORK_LAB_REPO_DIR'" +# shellcheck disable=SC2086 +rsync -az --delete $exclude_args -e "$NETWORK_LAB_RSYNC_RSH" "$REPO_ROOT/" "${NETWORK_LAB_USER}@${NETWORK_LAB_HOST}:${NETWORK_LAB_REPO_DIR}/" +echo "[network-lab] synced repo to ${NETWORK_LAB_REPO_DIR}" diff --git a/tests/integration/openwrt_vm/scripts/network-lab-test b/tests/integration/openwrt_vm/scripts/network-lab-test new file mode 100755 index 00000000..741efb6d --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/network-lab-test @@ -0,0 +1,42 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/network-lab-env" + +sh "$SCRIPT_DIR/network-lab-stop" || true +sh "$SCRIPT_DIR/network-lab-start" +sh "$SCRIPT_DIR/network-lab-provision" +sh "$SCRIPT_DIR/network-lab-sync" + +targets="$NETWORK_LAB_TEST_TARGETS" +if [ "$#" -gt 0 ]; then + targets="$*" +fi + +echo "[network-lab] running targets: $targets" +echo "[network-lab] nested OpenWrt SSH wait: ${NETWORK_LAB_OPENWRT_SSH_WAIT_S}s" +# Pass the target list as positional arguments rather than interpolating it into +# remote shell code. The exported OPENWRT_* values affect the nested OpenWrt VM +# lane only; they do not change the outer lab VM lifecycle. +# shellcheck disable=SC2086 # intentional splitting of Make target names +$NETWORK_LAB_SSH 'sh -s' -- \ + "$NETWORK_LAB_OPENWRT_SSH_WAIT_S" \ + "$NETWORK_LAB_OPENWRT_KVM" \ + "$NETWORK_LAB_REPO_DIR" \ + $targets <<'NETWORK_LAB_RUN_TARGETS' +set -eu +openwrt_ssh_wait_s="$1" +openwrt_kvm="$2" +repo_dir="$3" +shift 3 +export OPENWRT_SSH_WAIT_S="$openwrt_ssh_wait_s" +export OPENWRT_KVM="$openwrt_kvm" +cd "$repo_dir/tests/integration/openwrt_vm" +make preflight +for target do + make "$target" +done +NETWORK_LAB_RUN_TARGETS diff --git a/tests/integration/openwrt_vm/scripts/network-lab-wait b/tests/integration/openwrt_vm/scripts/network-lab-wait new file mode 100755 index 00000000..35c0acb2 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/network-lab-wait @@ -0,0 +1,24 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/network-lab-env" + +end=$(( $(date +%s) + NETWORK_LAB_SSH_WAIT_S )) +echo "[network-lab] waiting for SSH on ${NETWORK_LAB_HOST}:${NETWORK_LAB_SSH_PORT}" +while [ "$(date +%s)" -lt "$end" ]; do + if $NETWORK_LAB_SSH 'true' >/dev/null 2>&1; then + echo "[network-lab] SSH ready" + $NETWORK_LAB_SSH 'cloud-init status --wait >/dev/null 2>&1 || true' + exit 0 + fi + sleep 2 +done + +echo "[network-lab] timeout waiting for SSH" >&2 +if [ -f "$NETWORK_LAB_LOG" ]; then + tail -n 120 "$NETWORK_LAB_LOG" >&2 || true +fi +exit 1 diff --git a/tests/integration/openwrt_vm/scripts/preflight b/tests/integration/openwrt_vm/scripts/preflight new file mode 100755 index 00000000..1c647467 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/preflight @@ -0,0 +1,78 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +missing=0 + +need_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "[openwrt-vm] missing required command: $1" >&2 + missing=1 + fi +} + +need_cmd "$OPENWRT_QEMU" +need_cmd qemu-img +need_cmd curl +need_cmd gzip +need_cmd ssh +need_cmd scp + +if ! command -v sha256sum >/dev/null 2>&1 && ! command -v shasum >/dev/null 2>&1; then + echo "[openwrt-vm] missing required checksum tool: sha256sum or shasum" >&2 + missing=1 +fi + +if [ "$OPENWRT_ARCH" = "aarch64" ]; then + found="" + for p in \ + /usr/share/AAVMF/AAVMF_CODE.fd \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd \ + /usr/share/edk2/aarch64/QEMU_EFI.fd + do + if [ -f "$p" ]; then + found="$p" + break + fi + done + if [ -z "$found" ]; then + echo "[openwrt-vm] missing AArch64 EFI firmware" >&2 + missing=1 + else + echo "[openwrt-vm] AArch64 EFI firmware: $found" + fi +fi + +if [ "$OPENWRT_KVM" = "auto" ] || [ "$OPENWRT_KVM" = "1" ]; then + if [ -e /dev/kvm ]; then + if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + echo "[openwrt-vm] KVM available" + else + echo "[openwrt-vm] /dev/kvm exists but is not accessible to this user" >&2 + if [ "$OPENWRT_KVM" = "1" ]; then missing=1; fi + fi + else + echo "[openwrt-vm] KVM not available; QEMU will use emulation" + if [ "$OPENWRT_KVM" = "1" ]; then missing=1; fi + fi +fi + +if [ -n "${OPENWRT_TAP_IFACES:-}" ]; then + for tap in $OPENWRT_TAP_IFACES; do + if ! ip link show "$tap" >/dev/null 2>&1; then + echo "[openwrt-vm] configured tap device not present: $tap" >&2 + missing=1 + fi + done +fi + +if [ "$missing" -ne 0 ]; then + exit 1 +fi + +echo "[openwrt-vm] preflight ok" diff --git a/tests/integration/openwrt_vm/scripts/provision b/tests/integration/openwrt_vm/scripts/provision new file mode 100755 index 00000000..35ef424f --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/provision @@ -0,0 +1,102 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +"$SCRIPT_DIR/wait-ssh" + +if [ "${OPENWRT_PROVISION_FORCE}" != "1" ]; then + if "$SCRIPT_DIR/ssh" "test -f '${OPENWRT_PROVISION_MARKER}'" >/dev/null 2>&1; then + echo "[openwrt-vm] provision already complete; set OPENWRT_PROVISION_FORCE=1 to run again" + exit 0 + fi +fi + +echo "[openwrt-vm] provisioning base packages" + +"$SCRIPT_DIR/ssh" 'sh -s' <<'OPENWRT_PROVISION_BASE' +set -eu + +echo '[openwrt-vm] configuring temporary outbound route' +ip route replace default via 192.168.1.2 dev br-lan || true + +mkdir -p /tmp/resolv.conf.d +printf 'nameserver 192.168.1.3\nnameserver 1.1.1.1\n' > /tmp/resolv.conf.d/resolv.conf.auto +printf 'nameserver 192.168.1.3\nnameserver 1.1.1.1\n' > /tmp/resolv.conf + +echo '[openwrt-vm] checking connectivity' +ping -c 1 -W 3 192.168.1.2 >/dev/null || { + echo 'cannot reach QEMU user-network gateway 192.168.1.2' >&2 + ip addr >&2 + ip route >&2 + exit 1 +} + +opkg_update_with_retry() { + tries=0 + while [ "$tries" -lt 5 ]; do + tries=$((tries + 1)) + if opkg update; then + return 0 + fi + echo "[openwrt-vm] opkg update failed on attempt $tries; retrying" >&2 + ip route replace default via 192.168.1.2 dev br-lan || true + printf 'nameserver 192.168.1.3\nnameserver 1.1.1.1\n' > /tmp/resolv.conf + sleep $((tries * 2)) + done + echo '[openwrt-vm] opkg update failed after retries' >&2 + ip addr >&2 || true + ip route >&2 || true + cat /tmp/resolv.conf >&2 || true + return 1 +} + +opkg_update_with_retry + +# Install MWAN3 in a separate phase. Immediately after this SSH step returns, +# provision installs a known-good three-WAN fixture before any further package +# downloads can observe or trip over a half-configured mwan3 setup. +opkg install tc-full kmod-sched kmod-sched-core kmod-sched-ctinfo kmod-ifb kmod-veth ip-full lua libuci-lua + +# Some OpenWrt targets expose 8021q as built-in kernel support and do not +# publish a separate kmod-8021q package. VLAN capability is checked by the +# jan DHCP/DNS dependency script, so keep this package best-effort here. +opkg install kmod-8021q || true +opkg install mwan3 iptables-nft ip6tables-nft +OPENWRT_PROVISION_BASE + +# Install a valid three-WAN network/firewall/mwan3 fixture immediately after +# mwan3 is installed. This prevents the rest of provisioning from running with +# a package-default or otherwise invalid multi-WAN configuration. +"$VM_DIR/tests/setup_mwan3_three_wan_fixture.sh" + +"$SCRIPT_DIR/ssh" 'sh -s' -- "$OPENWRT_PROVISION_MARKER" <<'OPENWRT_PROVISION_EXTRA' +set -eu +marker="$1" + +# Keep provisioning traffic on the management network even after the WAN DHCP +# clients have added their own defaults. +ip route replace default via 192.168.1.2 dev br-lan || true + +# Dependencies used by vendored fibers/bus/trie and future real command-backed tests. +# Package names vary across OpenWrt feeds/releases, so keep these best-effort. +(opkg install luaposix || true) +(opkg install coreutils-stty || true) +(opkg install procps-ng-pgrep || true) +(opkg install lua-bit32 || true) +(opkg install lua-cjson || true) + +# Optional but useful for performance tests. Do not fail provisioning if unavailable +# on a given target/release. +opkg install iperf3 || true +opkg install tcpdump || true +opkg install bind-dig || true + +touch "$marker" +OPENWRT_PROVISION_EXTRA + +echo "[openwrt-vm] provision complete" diff --git a/tests/integration/openwrt_vm/scripts/render-default-configs b/tests/integration/openwrt_vm/scripts/render-default-configs new file mode 100755 index 00000000..81790656 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/render-default-configs @@ -0,0 +1,194 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +SCP_FROM="$VM_DIR/scripts/scp-from" +REMOTE="/tmp/devicecode-render-default-configs" +REMOTE_OUT="$REMOTE/out" +WORK="$VM_DIR/work/render-default-configs" +OUT_DIR="${DEVICECODE_RENDER_OUT_DIR:-$VM_DIR/work/generated-default-etc-config}" +TRUNK_IFNAME="${DEVICECODE_RENDER_TRUNK_IFNAME:-eth0}" +PRINT="${DEVICECODE_RENDER_PRINT:-1}" + +mkdir -p "$WORK" "$OUT_DIR" + +cat > "$WORK/run_devicecode_render_default_configs.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local cjson = require 'cjson.safe' +local provider_loader = require 'services.hal.backends.network.provider' +local net_config = require 'services.net.config' +local intent_realiser = require 'services.net.intent_realiser' + +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function read_file(path) + local f, err = io.open(path, 'rb') + if not f then fail('read failed ' .. path .. ': ' .. tostring(err)) end + local data = f:read('*a') + f:close() + return data +end +local function write_file(path, data) + local f, err = io.open(path, 'wb') + if not f then fail('write failed ' .. path .. ': ' .. tostring(err)) end + f:write(data or '') + f:close() +end +local function shell_quote(s) + s = tostring(s or '') + return "'" .. s:gsub("'", "'\\''") .. "'" +end +local function mkdir_p(path) + local ok = os.execute('mkdir -p ' .. shell_quote(path)) + if ok ~= true and ok ~= 0 then fail('mkdir failed: ' .. path) end +end +local function rm_rf(path) + local ok = os.execute('rm -rf ' .. shell_quote(path)) + if ok ~= true and ok ~= 0 then fail('rm -rf failed: ' .. path) end +end +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.05)) + end + if pred() then return true end + fail(label or 'condition not satisfied') +end + +local function maybe_gsm_sources() + local sources = { gsm_uplinks = {} } + local function add(role, env) + local ifname = os.getenv(env) + if type(ifname) == 'string' and ifname ~= '' then + sources.gsm_uplinks[role] = { + schema = 'devicecode.gsm.uplink/1', + id = role, + role = role, + state = 'connected', + connected = true, + available = true, + linux = { ifname = ifname }, + } + end + end + add('primary', 'DEVICECODE_RENDER_GSM_PRIMARY_IFNAME') + add('secondary', 'DEVICECODE_RENDER_GSM_SECONDARY_IFNAME') + return sources +end + +local root = os.getenv('DEVICECODE_RENDER_ROOT') or '.' +local confdir = os.getenv('DEVICECODE_RENDER_CONFDIR') or '/tmp/devicecode-render-default-configs/conf' +local savedir = os.getenv('DEVICECODE_RENDER_SAVEDIR') or '/tmp/devicecode-render-default-configs/save' +local outdir = os.getenv('DEVICECODE_RENDER_OUTDIR') or '/tmp/devicecode-render-default-configs/out' +local trunk_ifname = os.getenv('DEVICECODE_RENDER_TRUNK_IFNAME') or 'eth0' +local print_configs = os.getenv('DEVICECODE_RENDER_PRINT') ~= '0' + +rm_rf(confdir) +rm_rf(savedir) +rm_rf(outdir) +mkdir_p(confdir) +mkdir_p(savedir) +mkdir_p(outdir) + +fibers.run(function() + local doc = cjson.decode(read_file(root .. '/src/configs/bigbox-v1-cm-2.json')) + if type(doc) ~= 'table' then fail('default config JSON did not decode') end + local payload = net_config.extract_payload(doc.net) + local base_intent, err = net_config.normalise(payload, { generation = doc.net and doc.net.rev or 1 }) + if not base_intent then fail('default NET config normalise failed: ' .. tostring(err)) end + local apply_intent = intent_realiser.realise(base_intent, maybe_gsm_sources()) + + local activation_cmds = {} + local shaping_cmds = {} + local provider = assert(provider_loader.new({ + provider = 'openwrt', + confdir = confdir, + savedir = savedir, + debounce_s = 0.01, + platform = { segment_trunk = { ifname = trunk_ifname, protected = true } }, + run_cmd = function(argv) + activation_cmds[#activation_cmds + 1] = table.concat(argv or {}, ' ') + return true, nil + end, + shaper_run_cmd = function(argv) + shaping_cmds[#shaping_cmds + 1] = table.concat(argv or {}, ' ') + return true, nil + end, + }, {})) + + local valid = perform(provider:validate_op({ intent = apply_intent })) + if not (valid and valid.ok == true) then fail('validate failed: ' .. tostring(valid and valid.err)) end + local result = perform(provider:apply_op({ intent = apply_intent, opts = { generation = doc.net and doc.net.rev or 1, apply_id = 'render-default-configs' } })) + if not (result and result.ok == true) then fail('apply failed: ' .. tostring(result and result.err)) end + local mgr = provider._uci_manager + wait_until(function() + local st = mgr and mgr.activation_status and mgr:activation_status() or nil + return st and (st.state == 'done' or st.state == 'idle') + end, 5, 'activation runner should settle') + + local pkgs = { 'network', 'dhcp', 'firewall', 'mwan3' } + for _, pkg in ipairs(pkgs) do + local data = read_file(confdir .. '/' .. pkg) + write_file(outdir .. '/' .. pkg, data) + if print_configs then + io.stdout:write('\n==> ', pkg, ' <==\n') + io.stdout:write(data) + if data:sub(-1) ~= '\n' then io.stdout:write('\n') end + end + end + + local manifest = { + schema = 'devicecode.openwrt.generated-configs/1', + source_config = 'src/configs/bigbox-v1-cm-2.json', + config_rev = doc.net and doc.net.rev or nil, + trunk_ifname = trunk_ifname, + realised_wan_members = {}, + activation_cmds = activation_cmds, + shaping_cmds = shaping_cmds, + openwrt_names = result.openwrt_names, + note = 'Generated into a temporary UCI confdir; no real /etc/config files were modified by this renderer.', + } + for id, member in pairs((apply_intent.wan and apply_intent.wan.members) or {}) do + manifest.realised_wan_members[#manifest.realised_wan_members + 1] = { + id = id, + interface = member.interface or id, + route_metric = member.route_metric, + mwan_metric = member.mwan_metric, + weight = member.weight, + } + end + table.sort(manifest.realised_wan_members, function(a, b) return tostring(a.id) < tostring(b.id) end) + write_file(outdir .. '/manifest.json', assert(cjson.encode(manifest))) + + provider:terminate('render complete') +end) +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_devicecode_render_default_configs.lua" "$REMOTE/run_devicecode_render_default_configs.lua" + +echo "[openwrt-vm] rendering default Devicecode /etc/config files into $OUT_DIR" +"$SSH" "cd '$REMOTE' && DEVICECODE_RENDER_ROOT='$REMOTE' DEVICECODE_RENDER_CONFDIR='$REMOTE/conf' DEVICECODE_RENDER_SAVEDIR='$REMOTE/save' DEVICECODE_RENDER_OUTDIR='$REMOTE_OUT' DEVICECODE_RENDER_TRUNK_IFNAME='$TRUNK_IFNAME' DEVICECODE_RENDER_PRINT='$PRINT' DEVICECODE_RENDER_GSM_PRIMARY_IFNAME='${DEVICECODE_RENDER_GSM_PRIMARY_IFNAME:-}' DEVICECODE_RENDER_GSM_SECONDARY_IFNAME='${DEVICECODE_RENDER_GSM_SECONDARY_IFNAME:-}' lua ./run_devicecode_render_default_configs.lua" +rm -rf "$OUT_DIR" +mkdir -p "$(dirname "$OUT_DIR")" +"$SCP_FROM" "$REMOTE_OUT" "$OUT_DIR" +printf '[openwrt-vm] generated configs copied to %s\n' "$OUT_DIR" +printf '[openwrt-vm] files:\n' +find "$OUT_DIR" -maxdepth 1 -type f | while IFS= read -r f; do basename "$f"; done | sort | sed 's/^/ /' diff --git a/tests/integration/openwrt_vm/scripts/reset-disk b/tests/integration/openwrt_vm/scripts/reset-disk new file mode 100755 index 00000000..b91e44c9 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/reset-disk @@ -0,0 +1,21 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +"$SCRIPT_DIR/fetch-image" + +mkdir -p "${OPENWRT_WORK_DIR}" +rm -f "${OPENWRT_WORK_DISK}" + +qemu-img create \ + -f qcow2 \ + -F raw \ + -b "${OPENWRT_IMAGE_RAW}" \ + "${OPENWRT_WORK_DISK}" + +echo "[openwrt-vm] reset disk: ${OPENWRT_WORK_DISK}" diff --git a/tests/integration/openwrt_vm/scripts/restore-openwrt-config-state b/tests/integration/openwrt_vm/scripts/restore-openwrt-config-state new file mode 100755 index 00000000..d53d7f86 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/restore-openwrt-config-state @@ -0,0 +1,37 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +SSH="$SCRIPT_DIR/ssh" +WAIT_SSH="$SCRIPT_DIR/wait-ssh" +STATE="${DEVICECODE_CONFIG_STATE:-/tmp/devicecode-openwrt-config-backup}" +ACTIVATE="${DEVICECODE_RESTORE_ACTIVATE:-1}" + +"$SSH" "CONFIG_STATE='$STATE' RESTORE_ACTIVATE='$ACTIVATE' sh -s" <<'REMOTE_RESTORE' +set +e +if [ -d "$CONFIG_STATE" ]; then + while read name; do + [ -n "$name" ] && [ -e "$CONFIG_STATE/$name" ] && cp -a "$CONFIG_STATE/$name" "/etc/config/$name" + done < "$CONFIG_STATE/present" + while read name; do + [ -n "$name" ] && rm -f "/etc/config/$name" + done < "$CONFIG_STATE/absent" + if [ "${RESTORE_ACTIVATE:-1}" = 1 ]; then + ( + sleep 1 + /etc/init.d/network reload >/tmp/devicecode-config-restore-network.log 2>&1 + /etc/init.d/firewall restart >/tmp/devicecode-config-restore-firewall.log 2>&1 + ) /tmp/devicecode-config-restore-wrapper.log 2>&1 & + echo "restore-launched" + else + echo "restore-files-only" + fi +else + echo "no saved config state at $CONFIG_STATE" >&2 +fi +REMOTE_RESTORE + +if [ "${DEVICECODE_RESTORE_NOWAIT:-0}" != 1 ] && [ "${ACTIVATE:-1}" = 1 ]; then + sleep 2 + "$WAIT_SSH" +fi diff --git a/tests/integration/openwrt_vm/scripts/run-vm b/tests/integration/openwrt_vm/scripts/run-vm new file mode 100755 index 00000000..3f9d7829 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/run-vm @@ -0,0 +1,173 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +if [ -f "${OPENWRT_PID}" ] && kill -0 "$(cat "${OPENWRT_PID}")" 2>/dev/null; then + echo "[openwrt-vm] already running with pid $(cat "${OPENWRT_PID}")" + exit 0 +fi + +if command -v ss >/dev/null 2>&1 && ss -H -ltn "sport = :${OPENWRT_SSH_PORT}" 2>/dev/null | grep -q .; then + echo "[openwrt-vm] SSH port ${OPENWRT_SSH_PORT} is already in use" >&2 + echo "[openwrt-vm] run ./scripts/stop-vm, or set OPENWRT_SSH_PORT to a free port" >&2 + exit 1 +fi + +if [ ! -f "${OPENWRT_WORK_DISK}" ]; then + "$SCRIPT_DIR/reset-disk" +fi + +mkdir -p "${OPENWRT_WORK_DIR}" +rm -f "${OPENWRT_LOG}" "${OPENWRT_PID}" + +NETDEV="user,id=mgmt0,net=192.168.1.0/24,hostfwd=tcp::${OPENWRT_SSH_PORT}-192.168.1.1:22" + +# Debian's qemu-system-arm package does not install the iPXE option ROMs +# unless recommended packages are enabled. We boot from disk, not PXE, so disable +# the virtio-net PCI option ROMs on AArch64 instead of requiring ipxe-qemu. +NET_ROMFILE_OPT="" +if [ "${OPENWRT_ARCH}" = "aarch64" ]; then + NET_ROMFILE_OPT=",romfile=" +fi + +add_common_args() { + set -- \ + -m "${OPENWRT_VM_MEM}" \ + -smp "${OPENWRT_VM_CPUS}" \ + -drive "file=${OPENWRT_WORK_DISK},format=qcow2,if=virtio" \ + -netdev "${NETDEV}" \ + -device "virtio-net-pci,netdev=mgmt0${NET_ROMFILE_OPT}" \ + -display none \ + -monitor none \ + -daemonize \ + -pidfile "${OPENWRT_PID}" \ + -serial "file:${OPENWRT_LOG}" + + idx=0 + while [ "$idx" -lt "${OPENWRT_VM_WAN_IFACES:-3}" ]; do + idx=$((idx + 1)) + mac="$(printf '02:dc:bb:01:00:%02x' "$idx")" + set -- "$@" \ + -netdev "user,id=wan${idx},net=172.31.${idx}.0/24" \ + -device "virtio-net-pci,netdev=wan${idx},mac=${mac}${NET_ROMFILE_OPT}" + done + + idx=0 + for tap in ${OPENWRT_TAP_IFACES:-}; do + idx=$((idx + 1)) + mac="$(printf '02:dc:bb:00:00:%02x' "$idx")" + set -- "$@" \ + -netdev "tap,id=data${idx},ifname=${tap},script=no,downscript=no" \ + -device "virtio-net-pci,netdev=data${idx},mac=${mac}${NET_ROMFILE_OPT}" + done + + if [ "$OPENWRT_ARCH" = "x86_64" ]; then + case "$OPENWRT_KVM" in + 1) + set -- -enable-kvm "$@" + ;; + auto) + if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + set -- -enable-kvm "$@" + fi + ;; + esac + fi + + exec "$OPENWRT_QEMU" "$@" +} + +if [ "${OPENWRT_ARCH}" = "x86_64" ]; then + add_common_args +fi + +AAVMF_CODE="" +for p in \ + /usr/share/AAVMF/AAVMF_CODE.fd \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd \ + /usr/share/edk2/aarch64/QEMU_EFI.fd +do + if [ -f "$p" ]; then + AAVMF_CODE="$p" + break + fi +done + +if [ -z "${AAVMF_CODE}" ]; then + echo "could not find AArch64 EFI firmware" >&2 + exit 1 +fi + +aarch64_kvm_args="" +case "${OPENWRT_KVM}" in + 1) + if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + aarch64_kvm_args=1 + else + echo "[openwrt-vm] OPENWRT_KVM=1 but /dev/kvm is not accessible" >&2 + exit 1 + fi + ;; + auto) + if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + aarch64_kvm_args=1 + fi + ;; + 0|off|false) + ;; + *) + echo "[openwrt-vm] invalid OPENWRT_KVM=${OPENWRT_KVM}" >&2 + exit 1 + ;; +esac + +if [ -n "$aarch64_kvm_args" ]; then + set -- \ + -enable-kvm \ + -machine virt \ + -cpu host \ + -bios "${AAVMF_CODE}" +else + set -- \ + -machine virt \ + -cpu cortex-a72 \ + -bios "${AAVMF_CODE}" +fi + +# Rebuild the common tail after the AArch64 machine-specific prefix. +set -- "$@" \ + -m "${OPENWRT_VM_MEM}" \ + -smp "${OPENWRT_VM_CPUS}" \ + -drive "file=${OPENWRT_WORK_DISK},format=qcow2,if=virtio" \ + -netdev "${NETDEV}" \ + -device "virtio-net-pci,netdev=mgmt0${NET_ROMFILE_OPT}" \ + -display none \ + -monitor none \ + -daemonize \ + -pidfile "${OPENWRT_PID}" \ + -serial "file:${OPENWRT_LOG}" + +idx=0 +while [ "$idx" -lt "${OPENWRT_VM_WAN_IFACES:-3}" ]; do + idx=$((idx + 1)) + mac="$(printf '02:dc:bb:01:00:%02x' "$idx")" + set -- "$@" \ + -netdev "user,id=wan${idx},net=172.31.${idx}.0/24" \ + -device "virtio-net-pci,netdev=wan${idx},mac=${mac}${NET_ROMFILE_OPT}" +done + +idx=0 +for tap in ${OPENWRT_TAP_IFACES:-}; do + idx=$((idx + 1)) + mac="$(printf '02:dc:bb:00:00:%02x' "$idx")" + set -- "$@" \ + -netdev "tap,id=data${idx},ifname=${tap},script=no,downscript=no" \ + -device "virtio-net-pci,netdev=data${idx},mac=${mac}${NET_ROMFILE_OPT}" +done + +exec "${OPENWRT_QEMU}" "$@" diff --git a/tests/integration/openwrt_vm/scripts/scp-from b/tests/integration/openwrt_vm/scripts/scp-from new file mode 100755 index 00000000..4c990ad1 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/scp-from @@ -0,0 +1,23 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +if [ "$#" -ne 2 ]; then + echo "usage: $0 REMOTE_PATH LOCAL_PATH" >&2 + exit 2 +fi + +exec scp \ + -q \ + -O \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -P "${OPENWRT_SSH_PORT}" \ + -r \ + "root@127.0.0.1:$1" "$2" diff --git a/tests/integration/openwrt_vm/scripts/scp-to b/tests/integration/openwrt_vm/scripts/scp-to new file mode 100755 index 00000000..9a30ae50 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/scp-to @@ -0,0 +1,23 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +if [ "$#" -ne 2 ]; then + echo "usage: $0 LOCAL_PATH REMOTE_PATH" >&2 + exit 2 +fi + +exec scp \ + -q \ + -O \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -P "${OPENWRT_SSH_PORT}" \ + -r \ + "$1" "root@127.0.0.1:$2" diff --git a/tests/integration/openwrt_vm/scripts/setup-bridge-client-fabric b/tests/integration/openwrt_vm/scripts/setup-bridge-client-fabric new file mode 100755 index 00000000..aece647f --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/setup-bridge-client-fabric @@ -0,0 +1,232 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "[openwrt-vm] need root privileges for: $*" >&2 + exit 1 + fi +} + +apt_install_best_effort() { + if command -v apt-get >/dev/null 2>&1; then + as_root apt-get update + as_root env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$@" || true + fi +} + +need_cmd() { + cmd="$1" + shift + if ! command -v "$cmd" >/dev/null 2>&1; then + apt_install_best_effort "$@" + fi + command -v "$cmd" >/dev/null 2>&1 || { + echo "[openwrt-vm] missing required host command: $cmd" >&2 + exit 1 + } +} + + +print_privilege_diagnostics() { + { + echo "[openwrt-vm] host privilege diagnostics:" + echo " user: $(id 2>/dev/null || true)" + echo " uname: $(uname -a 2>/dev/null || true)" + echo " ip: $(command -v ip 2>/dev/null || echo missing)" + echo " bridge: $(command -v bridge 2>/dev/null || echo missing)" + echo " /dev/net/tun: $(ls -l /dev/net/tun 2>/dev/null || echo missing)" + if [ -r /proc/self/status ]; then + grep '^Cap' /proc/self/status | sed 's/^/ self /' || true + fi + if command -v sudo >/dev/null 2>&1; then + sudo sh -c 'grep "^Cap" /proc/self/status' 2>/dev/null | sed 's/^/ sudo /' || true + fi + } >&2 +} + +fail_privilege() { + what="$1" + log="${2:-}" + echo "[openwrt-vm] host-side bridge client fabric failed preflight: $what" >&2 + if [ -n "$log" ] && [ -s "$log" ]; then + echo "[openwrt-vm] failing command output:" >&2 + sed 's/^/[openwrt-vm] /' "$log" >&2 || true + fi + print_privilege_diagnostics + cat >&2 <<'EOF' +[openwrt-vm] This test creates host Linux bridges, veth pairs, network namespaces +[openwrt-vm] and QEMU tap devices. Run it on a host with NET_ADMIN/tun access, +[openwrt-vm] or delegate it to the isolated network-lab VM: +[openwrt-vm] make network-lab-test +[openwrt-vm] For local direct execution outside the lab, provide the needed host +[openwrt-vm] networking privileges explicitly. +EOF + exit 1 +} + +probe_or_fail() { + what="$1" + shift + log="${OPENWRT_WORK_DIR}/host-preflight-${what}.log" + mkdir -p "$OPENWRT_WORK_DIR" + rm -f "$log" + if as_root "$@" >"$log" 2>&1; then + rm -f "$log" + return 0 + fi + fail_privilege "$what" "$log" +} + +ensure_net_admin() { + probe="dcbr$$" + probe_or_fail "bridge-create" ip link add name "$probe" type bridge + as_root ip link del "$probe" >/dev/null 2>&1 || true +} + +ensure_netns() { + probe="dcns$$" + probe_or_fail "netns-create" ip netns add "$probe" + as_root ip netns del "$probe" >/dev/null 2>&1 || true +} + +ensure_tun() { + if [ ! -c /dev/net/tun ]; then + fail_privilege "tun-device-missing" "" + fi + probe="dctap$$" + probe_or_fail "tap-create" ip tuntap add dev "$probe" mode tap user "$(id -u)" + as_root ip tuntap del dev "$probe" mode tap >/dev/null 2>&1 || true +} + + +validate_ifname() { + name="$1" + label="$2" + case "$name" in + '') + echo "[openwrt-vm] empty interface name for $label" >&2 + exit 1 + ;; + esac + if [ "${#name}" -gt 15 ]; then + echo "[openwrt-vm] interface name for $label is too long for Linux IFNAMSIZ: $name" >&2 + echo "[openwrt-vm] choose <=15 characters, e.g. set $label to a shorter value" >&2 + exit 1 + fi +} + +need_cmd ip iproute2 +need_cmd bridge iproute2 +validate_ifname "$OPENWRT_CLIENT_BRIDGE" OPENWRT_CLIENT_BRIDGE +validate_ifname "$OPENWRT_CLIENT_TAP" OPENWRT_CLIENT_TAP +validate_ifname "$OPENWRT_CLIENT_HOST_IFACE" OPENWRT_CLIENT_HOST_IFACE +validate_ifname "$OPENWRT_CLIENT_PEER_IFACE" OPENWRT_CLIENT_PEER_IFACE +validate_ifname "$OPENWRT_CLIENT_IFACE" OPENWRT_CLIENT_IFACE +ensure_net_admin +ensure_netns +ensure_tun + +# Prefer the normal Debian busybox package for a host-side udhcpc applet. Do +# not request busybox-static alongside busybox: those packages conflict on +# Debian/bookworm and can break provisioning before the dataplane is created. +if ! command -v udhcpc >/dev/null 2>&1; then + apt_install_best_effort busybox + if ! command -v busybox >/dev/null 2>&1 || ! busybox udhcpc --help >/dev/null 2>&1; then + apt_install_best_effort udhcpc + fi +fi +if ! command -v nslookup >/dev/null 2>&1; then + apt_install_best_effort dnsutils bind9-dnsutils busybox +fi +if ! command -v wget >/dev/null 2>&1 && ! command -v curl >/dev/null 2>&1; then + apt_install_best_effort wget curl +fi +if ! command -v tcpdump >/dev/null 2>&1; then + apt_install_best_effort tcpdump +fi + +cleanup_client_bits() { + as_root ip netns del "$OPENWRT_CLIENT_NS" 2>/dev/null || true + as_root rm -rf "/etc/netns/$OPENWRT_CLIENT_NS" 2>/dev/null || true + as_root ip link del "$OPENWRT_CLIENT_HOST_IFACE" 2>/dev/null || true + as_root ip link del "$OPENWRT_CLIENT_PEER_IFACE" 2>/dev/null || true +} + +cleanup_client_bits + +# Linux bridge datapath: QEMU tap as a VLAN-100 trunk, host veth client as +# an untagged access port on VLAN 100. +if ip link show dev "$OPENWRT_CLIENT_TAP" >/dev/null 2>&1; then + as_root ip link set "$OPENWRT_CLIENT_TAP" nomaster 2>/dev/null || true + as_root ip link set "$OPENWRT_CLIENT_TAP" down 2>/dev/null || true + as_root ip tuntap del dev "$OPENWRT_CLIENT_TAP" mode tap 2>/dev/null || true + as_root ip link del "$OPENWRT_CLIENT_TAP" 2>/dev/null || true +fi +if ip link show dev "$OPENWRT_CLIENT_BRIDGE" >/dev/null 2>&1; then + as_root ip link set "$OPENWRT_CLIENT_BRIDGE" down 2>/dev/null || true + as_root ip link del "$OPENWRT_CLIENT_BRIDGE" 2>/dev/null || true +fi + +as_root ip link add name "$OPENWRT_CLIENT_BRIDGE" type bridge +# Some Docker/LinuxKit kernels reject bridge options during RTM_NEWLINK with +# "Attribute failed policy validation". Create the bridge first, then enable +# VLAN filtering through the dedicated bridge setter (or sysfs as a fallback). +if ! as_root ip link set dev "$OPENWRT_CLIENT_BRIDGE" type bridge vlan_filtering 1 >/dev/null 2>&1; then + if [ -w "/sys/class/net/$OPENWRT_CLIENT_BRIDGE/bridge/vlan_filtering" ] || as_root test -w "/sys/class/net/$OPENWRT_CLIENT_BRIDGE/bridge/vlan_filtering"; then + if [ "$(id -u)" -eq 0 ]; then + printf '1\n' > "/sys/class/net/$OPENWRT_CLIENT_BRIDGE/bridge/vlan_filtering" + else + printf '1\n' | sudo tee "/sys/class/net/$OPENWRT_CLIENT_BRIDGE/bridge/vlan_filtering" >/dev/null + fi + else + echo "[openwrt-vm] failed to enable VLAN filtering on $OPENWRT_CLIENT_BRIDGE" >&2 + exit 1 + fi +fi +as_root ip link set "$OPENWRT_CLIENT_BRIDGE" up +as_root ip link set "$OPENWRT_CLIENT_BRIDGE" promisc on 2>/dev/null || true + +as_root ip tuntap add dev "$OPENWRT_CLIENT_TAP" mode tap user "$(id -u)" +as_root ip link set "$OPENWRT_CLIENT_TAP" master "$OPENWRT_CLIENT_BRIDGE" +as_root ip link set "$OPENWRT_CLIENT_TAP" up +as_root ip link set "$OPENWRT_CLIENT_TAP" promisc on 2>/dev/null || true +as_root bridge vlan del dev "$OPENWRT_CLIENT_TAP" vid 1 2>/dev/null || true +as_root bridge vlan del dev "$OPENWRT_CLIENT_TAP" vid "$OPENWRT_CLIENT_VLAN" 2>/dev/null || true +as_root bridge vlan add dev "$OPENWRT_CLIENT_TAP" vid "$OPENWRT_CLIENT_VLAN" + +as_root ip netns add "$OPENWRT_CLIENT_NS" +as_root ip link del "$OPENWRT_CLIENT_HOST_IFACE" 2>/dev/null || true +as_root ip link del "$OPENWRT_CLIENT_PEER_IFACE" 2>/dev/null || true +as_root ip link add "$OPENWRT_CLIENT_HOST_IFACE" type veth peer name "$OPENWRT_CLIENT_PEER_IFACE" +as_root ip link set "$OPENWRT_CLIENT_PEER_IFACE" netns "$OPENWRT_CLIENT_NS" +as_root ip netns exec "$OPENWRT_CLIENT_NS" ip link set "$OPENWRT_CLIENT_PEER_IFACE" name "$OPENWRT_CLIENT_IFACE" +as_root ip link set "$OPENWRT_CLIENT_HOST_IFACE" master "$OPENWRT_CLIENT_BRIDGE" +as_root ip link set "$OPENWRT_CLIENT_HOST_IFACE" up +as_root ip link set "$OPENWRT_CLIENT_HOST_IFACE" promisc on 2>/dev/null || true +as_root bridge vlan del dev "$OPENWRT_CLIENT_HOST_IFACE" vid 1 2>/dev/null || true +as_root bridge vlan del dev "$OPENWRT_CLIENT_HOST_IFACE" vid "$OPENWRT_CLIENT_VLAN" 2>/dev/null || true +as_root bridge vlan add dev "$OPENWRT_CLIENT_HOST_IFACE" vid "$OPENWRT_CLIENT_VLAN" pvid untagged +as_root ip netns exec "$OPENWRT_CLIENT_NS" ip link set lo up +as_root ip netns exec "$OPENWRT_CLIENT_NS" ip link set "$OPENWRT_CLIENT_IFACE" up + +mkdir -p "$OPENWRT_WORK_DIR" +printf '%s\n' linux-bridge > "$OPENWRT_WORK_DIR/int-client-fabric.kind" + +cat </dev/null; then + echo "[openwrt-vm] stopping ${label} pid ${pid}" + kill "$pid" || true + sleep 1 + if kill -0 "$pid" 2>/dev/null; then + kill -9 "$pid" || true + fi + fi +} + +if [ -f "${OPENWRT_PID}" ]; then + pid="$(cat "${OPENWRT_PID}")" + stop_pid "$pid" "recorded" + rm -f "${OPENWRT_PID}" +fi + +# Some failed or interrupted runs can leave QEMU alive after the pidfile is +# removed. If the management SSH port is still owned by a qemu-system process +# belonging to this user, stop it too so the next run can bind hostfwd again. +if command -v ss >/dev/null 2>&1; then + port_pids="$(ss -H -ltnp "sport = :${OPENWRT_SSH_PORT}" 2>/dev/null | sed -n 's/.*pid=\([0-9][0-9]*\).*/\1/p' | sort -u)" + for pid in $port_pids; do + cmd="$(ps -p "$pid" -o comm= 2>/dev/null || true)" + case "$cmd" in + qemu-system-*) + stop_pid "$pid" "stale port ${OPENWRT_SSH_PORT}" + ;; + esac + done +fi diff --git a/tests/integration/openwrt_vm/scripts/teardown-bridge-client-fabric b/tests/integration/openwrt_vm/scripts/teardown-bridge-client-fabric new file mode 100755 index 00000000..8c8072a9 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/teardown-bridge-client-fabric @@ -0,0 +1,35 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "[openwrt-vm] need root privileges for: $*" >&2 + exit 1 + fi +} + +as_root ip netns del "$OPENWRT_CLIENT_NS" 2>/dev/null || true +as_root rm -rf "/etc/netns/$OPENWRT_CLIENT_NS" 2>/dev/null || true +as_root ip link del "$OPENWRT_CLIENT_HOST_IFACE" 2>/dev/null || true +as_root ip link del "$OPENWRT_CLIENT_PEER_IFACE" 2>/dev/null || true +as_root ip link set "$OPENWRT_CLIENT_TAP" nomaster 2>/dev/null || true +as_root ip link set "$OPENWRT_CLIENT_TAP" down 2>/dev/null || true +as_root ip tuntap del dev "$OPENWRT_CLIENT_TAP" mode tap 2>/dev/null || true +as_root ip link del "$OPENWRT_CLIENT_TAP" 2>/dev/null || true +if ip link show dev "$OPENWRT_CLIENT_BRIDGE" >/dev/null 2>&1; then + as_root ip link set "$OPENWRT_CLIENT_BRIDGE" down 2>/dev/null || true + as_root ip link del "$OPENWRT_CLIENT_BRIDGE" 2>/dev/null || true +fi +rm -f "$OPENWRT_WORK_DIR/int-client-fabric.kind" 2>/dev/null || true + +echo "[openwrt-vm] client fabric removed" diff --git a/tests/integration/openwrt_vm/scripts/verify-image b/tests/integration/openwrt_vm/scripts/verify-image new file mode 100755 index 00000000..4cd7c357 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/verify-image @@ -0,0 +1,49 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +mkdir -p "${OPENWRT_IMAGE_DIR}" + +if [ ! -f "${OPENWRT_SHA256SUMS}" ]; then + echo "[openwrt-vm] fetching ${OPENWRT_SHA256SUMS_NAME}" + curl -fL "${OPENWRT_BASE_URL}/${OPENWRT_SHA256SUMS_NAME}" -o "${OPENWRT_SHA256SUMS}" +fi + +if [ ! -f "${OPENWRT_IMAGE_GZ}" ]; then + echo "[openwrt-vm] image is missing: ${OPENWRT_IMAGE_GZ}" >&2 + exit 1 +fi + +expected="$(awk -v name="${OPENWRT_IMAGE_NAME}" ' + { + file=$2 + sub(/^\*/, "", file) + if (file == name) { print $1; found=1; exit } + } + END { if (!found) exit 1 } +' "${OPENWRT_SHA256SUMS}" 2>/dev/null || true)" + +if [ -z "$expected" ]; then + echo "[openwrt-vm] ${OPENWRT_IMAGE_NAME} not found in ${OPENWRT_SHA256SUMS}" >&2 + exit 1 +fi + +if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "${OPENWRT_IMAGE_GZ}" | awk '{ print $1 }')" +else + actual="$(shasum -a 256 "${OPENWRT_IMAGE_GZ}" | awk '{ print $1 }')" +fi + +if [ "$actual" != "$expected" ]; then + echo "[openwrt-vm] checksum mismatch for ${OPENWRT_IMAGE_NAME}" >&2 + echo "[openwrt-vm] expected: $expected" >&2 + echo "[openwrt-vm] actual: $actual" >&2 + exit 1 +fi + +echo "[openwrt-vm] checksum ok: ${OPENWRT_IMAGE_NAME}" diff --git a/tests/integration/openwrt_vm/scripts/wait-ssh b/tests/integration/openwrt_vm/scripts/wait-ssh new file mode 100755 index 00000000..77ff5464 --- /dev/null +++ b/tests/integration/openwrt_vm/scripts/wait-ssh @@ -0,0 +1,30 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +deadline="${OPENWRT_SSH_WAIT_S}" +start="$(date +%s)" + +echo "[openwrt-vm] waiting for SSH on 127.0.0.1:${OPENWRT_SSH_PORT}" + +while true; do + if "$SCRIPT_DIR/ssh" 'true' >/dev/null 2>&1; then + echo "[openwrt-vm] SSH ready" + exit 0 + fi + + now="$(date +%s)" + if [ $((now - start)) -ge "$deadline" ]; then + echo "[openwrt-vm] SSH did not become ready within ${deadline}s" >&2 + echo "[openwrt-vm] last serial log lines:" >&2 + tail -n 80 "${OPENWRT_LOG}" >&2 || true + exit 1 + fi + + sleep 2 +done diff --git a/tests/integration/openwrt_vm/tests/setup_devicecode_vm_mwan_generated_config.sh b/tests/integration/openwrt_vm/tests/setup_devicecode_vm_mwan_generated_config.sh new file mode 100755 index 00000000..c34a6325 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/setup_devicecode_vm_mwan_generated_config.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-vm-real-mwan-config" +mkdir -p "$VM_DIR/work" + +ACTIVATE="${DEVICECODE_VM_MWAN_CONFIG_ACTIVATE:-0}" +if [ "$ACTIVATE" = 1 ]; then + MODE=active +else + MODE=config +fi +DONE="/tmp/devicecode-vm-real-mwan-config.$MODE.done" +STATUS="/tmp/devicecode-vm-real-mwan-config.$MODE.status" +LOG="/tmp/devicecode-vm-real-mwan-config.$MODE.log" +MARKER="/tmp/devicecode-vm-real-mwan-config.$MODE.ok" +CONFIG_VERSION="devicecode-vm-real-mwan-config-v2-https-sticky" +VERSION="/tmp/devicecode-vm-real-mwan-config.$MODE.version" + +echo "[openwrt-vm] applying Devicecode generated MWAN config mode=$MODE" + +if [ "${DEVICECODE_VM_MWAN_CONFIG_FORCE:-0}" != 1 ]; then + if [ "$ACTIVATE" = 1 ]; then + CHECK_MWAN="mwan3 status >/tmp/devicecode-vm-real-mwan-status.log 2>&1 && grep -q 'balanced:' /tmp/devicecode-vm-real-mwan-status.log && grep -Eq 'S[[:space:]]+https' /tmp/devicecode-vm-real-mwan-status.log" + else + CHECK_MWAN="true" + fi + if "$SSH" "test -f '$MARKER' && test -f '$VERSION' && [ \"\$(cat '$VERSION' 2>/dev/null)\" = '$CONFIG_VERSION' ] && [ \"\$(uci -q get network.wan.device 2>/dev/null)\" = eth1 ] && [ \"\$(uci -q get network.wanb.device 2>/dev/null)\" = eth2 ] && [ \"\$(uci -q get network.wanc.device 2>/dev/null)\" = eth3 ] && [ \"\$(uci -q get network.wan.metric 2>/dev/null)\" = 11 ] && [ \"\$(uci -q get network.wanb.metric 2>/dev/null)\" = 12 ] && [ \"\$(uci -q get network.wanc.metric 2>/dev/null)\" = 13 ] && $CHECK_MWAN" >/dev/null 2>&1; then + echo "[openwrt-vm] Devicecode generated MWAN config already installed mode=$MODE" + exit 0 + fi +fi + +cat > "$VM_DIR/work/run_devicecode_vm_real_mwan_apply.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + './fixtures/?.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local exec = require 'fibers.io.exec' +local provider_loader = require 'services.hal.backends.network.provider' +local vm_intent = require 'devicecode_vm_mwan_intent' + +local perform = fibers.perform +local function fail(msg) error(msg, 2) end +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.25)) + end + if pred() then return true end + fail(label or 'condition was not satisfied before timeout') +end +local function capture(...) + local cmd = exec.command(...) + local out, st, code, sig, err = perform(cmd:combined_output_op()) + if st == 'exited' and code == 0 then return out or '' end + fail('command failed: ' .. table.concat({ ... }, ' ') .. ' ' .. tostring(err or out or st or sig or code)) +end +local function ok_cmd(...) + local cmd = exec.command(...) + local _out, st, code = perform(cmd:combined_output_op()) + return st == 'exited' and code == 0 +end +local function contains(s, needle) + return type(s) == 'string' and s:find(needle, 1, true) ~= nil +end + +fibers.run(function() + local activate = os.getenv('DEVICECODE_VM_MWAN_CONFIG_ACTIVATE') == '1' + for _, wan in ipairs(vm_intent.wans) do + if not ok_cmd('ip', 'link', 'show', wan.device) then + fail('missing VM WAN interface ' .. wan.device .. '; run the VM with OPENWRT_VM_WAN_IFACES=3') + end + end + + local provider_config = { provider = 'openwrt', debounce_s = 0.05 } + if not activate then + -- Config-shape tests need real /etc/config files but not disruptive service + -- reloads. Keep the activation path exercised while making commands inert. + provider_config.run_cmd = function(argv) + print('config-only: skipped activation command ' .. table.concat(argv or {}, ' ')) + return true, nil + end + end + + local provider = assert(provider_loader.new(provider_config, {})) + local intent = vm_intent.intent() + local valid = perform(provider:validate_op({ intent = intent })) + if not (valid and valid.ok == true) then fail('validate failed: ' .. tostring(valid and valid.err)) end + local result = perform(provider:apply_op({ intent = intent, opts = { generation = 24010, apply_id = 'openwrt-vm-real-mwan' } })) + if not (result and result.ok == true) then fail('apply failed: ' .. tostring(result and result.err)) end + + local mgr = provider._uci_manager + wait_until(function() + local st = mgr and mgr.activation_status and mgr:activation_status() or nil + return st and (st.state == 'done' or st.state == 'idle') + end, 30, 'activation runner should complete OpenWrt activation work') + + if activate then + wait_until(function() + local status = capture('mwan3', 'status') + return contains(status, 'balanced:') and contains(status, 'wan (') and contains(status, 'wanb (') and contains(status, 'wanc (') + end, 30, 'mwan3 should expose generated balanced policy') + end + + provider:terminate('test complete') +end) + +print('devicecode VM real MWAN apply: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE/fixtures'; rm -f '$DONE' '$STATUS' '$LOG' '$MARKER' '$VERSION'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$VM_DIR/fixtures/devicecode_vm_mwan_intent.lua" "$REMOTE/fixtures/devicecode_vm_mwan_intent.lua" +"$SCP_TO" "$VM_DIR/work/run_devicecode_vm_real_mwan_apply.lua" "$REMOTE/run_devicecode_vm_real_mwan_apply.lua" +echo "[openwrt-vm] running Devicecode generated MWAN config apply mode=$MODE" +if ! "$SSH" "cd '$REMOTE' && DEVICECODE_VM_MWAN_CONFIG_ACTIVATE=$ACTIVATE lua ./run_devicecode_vm_real_mwan_apply.lua >'$LOG' 2>&1"; then + "$SSH" "cat '$LOG' 2>/dev/null || true" >&2 || true + echo "Devicecode generated MWAN config apply failed mode=$MODE" >&2 + exit 1 +fi +"$SSH" "printf '%s\n' '$CONFIG_VERSION' > '$VERSION'; touch '$MARKER'; cat '$LOG'" diff --git a/tests/integration/openwrt_vm/tests/setup_mwan3_three_wan_fixture.sh b/tests/integration/openwrt_vm/tests/setup_mwan3_three_wan_fixture.sh new file mode 100755 index 00000000..11e19a33 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/setup_mwan3_three_wan_fixture.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +FIXTURE="$VM_DIR/fixtures/mwan3-balanced" +REMOTE_FIXTURE="/tmp/devicecode-mwan3-balanced.fixture" + +"$SCP_TO" "$FIXTURE" "$REMOTE_FIXTURE" + +"$SSH" 'sh -s' <<'REMOTE' +set -eu +MARKER=/tmp/devicecode-mwan3-three-wan-fixture.ok +if [ "${DEVICECODE_MWAN3_FIXTURE_FORCE:-0}" != 1 ] && [ -f "$MARKER" ]; then + if [ "$(uci -q get network.wan.device 2>/dev/null)" = eth1 ] && \ + [ "$(uci -q get network.wanb.device 2>/dev/null)" = eth2 ] && \ + [ "$(uci -q get network.wanc.device 2>/dev/null)" = eth3 ] && \ + [ -x /etc/init.d/mwan3 ] && \ + mwan3 status >/tmp/devicecode-mwan-live-status.log 2>&1 && \ + iptables-save -t mangle | grep -q '^:mwan3_policy_balanced ' && \ + iptables-save -t mangle | grep -q '^:mwan3_iface_in_wan ' && \ + iptables-save -t mangle | grep -q '^:mwan3_iface_in_wanb ' && \ + iptables-save -t mangle | grep -q '^:mwan3_iface_in_wanc '; then + echo 'three-WAN MWAN3 fixture already installed' + exit 0 + fi +fi + +echo 'installing three-WAN MWAN3 fixture' + +cleanup_devicecode_shaping_state() { + # Tests may be run individually or after an interrupted prior run. Keep the + # MWAN fixture independent of Devicecode shaping experiments by removing any + # Devicecode-owned qdiscs, IFBs and mangle chains before reloading network. + for dev in eth1 eth2 eth3; do + tc qdisc del dev "$dev" root >/dev/null 2>&1 || true + tc qdisc del dev "$dev" ingress >/dev/null 2>&1 || true + ifb="ifb_$(echo "$dev" | sed 's/[^A-Za-z0-9_]/_/g')" + ip link del "$ifb" >/dev/null 2>&1 || true + done + if command -v iptables >/dev/null 2>&1; then + while iptables -t mangle -D OUTPUT -j DEVICECODE_SHAPING_OUTPUT >/dev/null 2>&1; do :; done + while iptables -t mangle -D FORWARD -j DEVICECODE_SHAPING_FORWARD >/dev/null 2>&1; do :; done + iptables -t mangle -F DEVICECODE_SHAPING_OUTPUT >/dev/null 2>&1 || true + iptables -t mangle -F DEVICECODE_SHAPING_FORWARD >/dev/null 2>&1 || true + iptables -t mangle -X DEVICECODE_SHAPING_OUTPUT >/dev/null 2>&1 || true + iptables -t mangle -X DEVICECODE_SHAPING_FORWARD >/dev/null 2>&1 || true + fi +} + +run_logged_step() { + label="$1"; log="$2"; shift 2 + echo "[mwan-fixture] $label" + if "$@" >"$log" 2>&1; then + return 0 + fi + rc="$?" + echo "[mwan-fixture] failed: $label" >&2 + cat "$log" >&2 || true + logread 2>/dev/null | tail -80 >&2 || true + ps w >&2 || true + exit "$rc" +} + +cleanup_devicecode_shaping_state +for dev in eth1 eth2 eth3; do + ip link show "$dev" >/dev/null 2>&1 || { + echo "missing VM WAN interface: $dev" >&2 + echo 'reset/restart the OpenWrt VM with the default OPENWRT_VM_WAN_IFACES=3' >&2 + exit 1 + } +done + +uci -q batch <<'EOF' +delete network.wan +set network.wan=interface +set network.wan.device='eth1' +set network.wan.proto='dhcp' +set network.wan.metric='10' + +delete network.wanb +set network.wanb=interface +set network.wanb.device='eth2' +set network.wanb.proto='dhcp' +set network.wanb.metric='20' + +delete network.wanc +set network.wanc=interface +set network.wanc.device='eth3' +set network.wanc.proto='dhcp' +set network.wanc.metric='30' + +commit network +EOF + +WAN_ZONE="$({ uci show firewall || true; } | sed -n "s/^\(firewall\.[^=]*\)=zone$/\1/p" | while read sec; do [ "$(uci -q get "$sec.name")" = wan ] && echo "$sec" && break; done)" +[ -n "$WAN_ZONE" ] || { echo 'wan firewall zone not found' >&2; exit 1; } +uci -q delete "$WAN_ZONE.network" || true +uci add_list "$WAN_ZONE.network=wan" +uci add_list "$WAN_ZONE.network=wanb" +uci add_list "$WAN_ZONE.network=wanc" +uci commit firewall + +cp /tmp/devicecode-mwan3-balanced.fixture /etc/config/mwan3 + +run_logged_step 'network reload' /tmp/devicecode-mwan-live-network.log /etc/init.d/network reload +run_logged_step 'firewall restart' /tmp/devicecode-mwan-live-firewall.log /etc/init.d/firewall restart +/etc/init.d/mwan3 enable >/tmp/devicecode-mwan-live-enable.log 2>&1 || true +run_logged_step 'mwan3 restart' /tmp/devicecode-mwan-live-restart.log /etc/init.d/mwan3 restart +sleep 2 + +mwan3 status >/tmp/devicecode-mwan-live-status.log 2>&1 || { cat /tmp/devicecode-mwan-live-status.log; exit 1; } +touch "$MARKER" +iptables-save -t mangle | grep -q '^:mwan3_policy_balanced ' +iptables-save -t mangle | grep -q '^:mwan3_iface_in_wan ' +iptables-save -t mangle | grep -q '^:mwan3_iface_in_wanb ' +iptables-save -t mangle | grep -q '^:mwan3_iface_in_wanc ' +REMOTE diff --git a/tests/integration/openwrt_vm/tests/test_devicecode_bigbox_phase1_broken_trunk.sh b/tests/integration/openwrt_vm/tests/test_devicecode_bigbox_phase1_broken_trunk.sh new file mode 100755 index 00000000..5cce0020 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_devicecode_bigbox_phase1_broken_trunk.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-bigbox-phase1-broken-trunk-test" +WORK="$VM_DIR/work/bigbox-phase1-broken-trunk-test" + +mkdir -p "$WORK" +cat > "$WORK/run_devicecode_bigbox_phase1_broken_trunk.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local bus = require 'bus' +local wired_config = require 'services.wired.config' +local wired_service = require 'services.wired.service' +local wired_publisher = require 'services.wired.publisher' +local wired_topics = require 'services.wired.topics' + +local function fail(msg) error(msg, 2) end +local function eq(a, b, msg) if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function ok(v, msg) if v ~= true then fail(msg or 'expected true') end end +local function retained_payload(conn, topic) + local v = conn:retained_view({ '#' }) + local msg = v:get(topic) + v:close() + if not msg then fail('missing retained topic ' .. table.concat(topic, '/')) end + return msg.payload +end +local function find_violation(list, kind, fields) + for _, v in ipairs(list or {}) do + if v.kind == kind then + local match = true + for k, expected in pairs(fields or {}) do if v[k] ~= expected then match = false; break end end + if match then return v end + end + end + return nil +end + +local segments = { + mgmt = { kind = 'system', protected = true, vlan = { id = 10 } }, + switch_control = { kind = 'system', protected = true, vlan = { id = 11 } }, + fabric = { kind = 'system', protected = true, vlan = { id = 12 } }, + lan = { kind = 'user', vlan = { id = 100 } }, + guest = { kind = 'guest', vlan = { id = 101 } }, +} +local intent, err = wired_config.normalise({ + schema = wired_config.SCHEMA, + surfaces = { + ['switch-uplink-cm5'] = { + kind = 'switch-port', role = 'internal-trunk', protected = true, + attachment = { mode = 'trunk', required_segments = { 'mgmt', 'switch_control', 'fabric' }, user_segments = 'all-realised-user-segments' }, + }, + ['lan-1'] = { + kind = 'ethernet-port', role = 'access', + attachment = { mode = 'access', segment = 'lan' }, + }, + }, +}) +if not intent then fail(err) end + +fibers.run(function() + local b = bus.new() + local conn = b:connect({ origin_base = { kind = 'vm-test' } }) + local snap = { + net = { segments = segments }, + config_intent = intent, + assembly = { surfaces = { + ['switch-uplink-cm5'] = { component = 'switch-main', observed_surface = 'GE8' }, + ['lan-1'] = { component = 'switch-main', observed_surface = 'GE1' }, + } }, + observations = { + ['switch-main'] = { + status = { state = 'available', available = true, mode = 'read_only' }, + surfaces = { + ['GE8'] = { + observed_surface = 'GE8', kind = 'switch-port', capabilities = { trunk = true, access = false }, + link = { state = 'up', speed_mbps = 1000 }, + -- Missing switch_control VLAN 11 and guest VLAN 101. + attachment = { mode = 'trunk', vlans = { 10, 12, 100 } }, + }, + ['GE1'] = { + observed_surface = 'GE1', kind = 'ethernet-port', capabilities = { access = true, trunk = true }, + attachment = { mode = 'access', vlan = 100 }, + }, + }, + }, + }, + stats = {}, + } + wired_service._test.rebuild_derived(snap) + local ok_pub, perr = wired_publisher.publish_all_now(conn, snap, wired_publisher.new_state()) + if ok_pub ~= true then fail(perr or 'wired publish failed') end + + eq(snap.state, 'degraded', 'snap state') + ok(find_violation(snap.violations, 'missing_required_segment_carriage', { surface_id = 'switch-uplink-cm5', segment = 'switch_control', vlan = 11 }) ~= nil, 'missing switch_control violation') + ok(find_violation(snap.violations, 'missing_user_segment_carriage', { surface_id = 'switch-uplink-cm5', segment = 'guest', vlan = 101 }) ~= nil, 'missing guest violation') + + local published = retained_payload(conn, wired_topics.violations()) + ok(find_violation(published, 'missing_required_segment_carriage', { surface_id = 'switch-uplink-cm5', segment = 'switch_control', vlan = 11 }) ~= nil, 'published missing switch_control violation') + local uplink = retained_payload(conn, wired_topics.surface('switch-uplink-cm5')) + eq(uplink.availability.state, 'available', 'uplink link still observed available') +end) + +print('devicecode Big Box phase 1 broken trunk: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_devicecode_bigbox_phase1_broken_trunk.lua" "$REMOTE/run_devicecode_bigbox_phase1_broken_trunk.lua" +"$SSH" "cd '$REMOTE' && lua ./run_devicecode_bigbox_phase1_broken_trunk.lua" diff --git a/tests/integration/openwrt_vm/tests/test_devicecode_bigbox_phase1_composition.sh b/tests/integration/openwrt_vm/tests/test_devicecode_bigbox_phase1_composition.sh new file mode 100755 index 00000000..f3e51295 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_devicecode_bigbox_phase1_composition.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-bigbox-phase1-composition-test" +WORK="$VM_DIR/work/bigbox-phase1-composition-test" + +mkdir -p "$WORK" +cat > "$WORK/run_devicecode_bigbox_phase1_composition.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local bus = require 'bus' +local device_config = require 'services.device.config' +local device_model = require 'services.device.model' +local device_publisher = require 'services.device.publisher' +local wired_config = require 'services.wired.config' +local wired_service = require 'services.wired.service' +local wired_publisher = require 'services.wired.publisher' +local wired_topics = require 'services.wired.topics' + +local function fail(msg) error(msg, 2) end +local function eq(a, b, msg) if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function ok(v, msg) if v ~= true then fail(msg or 'expected true') end end +local function retained_payload(conn, topic) + local v = conn:retained_view({ '#' }) + local msg = v:get(topic) + v:close() + if not msg then fail('missing retained topic ' .. table.concat(topic, '/')) end + return msg.payload +end + +local function build_device_assembly(conn) + local catalogue, err = device_config.to_catalogue({ + schema = device_config.SCHEMA, + assembly = { + product = 'big-box', + components = { + ['switch-main'] = { kind = 'switch', role = 'wired-fabric' }, + ['cm5-local-wired'] = { kind = 'direct-nic', role = 'controller-wired-port' }, + }, + links = { + ['cm5-switch'] = { + kind = 'wired', role = 'controller-switch-uplink', internal = true, + a = { component = 'cm5-local-wired', observed_surface = 'eth0' }, + b = { component = 'switch-main', observed_surface = 'GE8' }, + }, + }, + surfaces = { + ['cm5-eth0'] = { component = 'cm5-local-wired', observed_surface = 'eth0', exposure = 'internal' }, + ['switch-uplink-cm5'] = { component = 'switch-main', observed_surface = 'GE8', exposure = 'internal' }, + ['lan-1'] = { component = 'switch-main', observed_surface = 'GE1', exposure = 'external' }, + ['lan-2'] = { component = 'switch-main', observed_surface = 'GE2', exposure = 'external' }, + }, + }, + components = { + ['switch-main'] = { + kind = 'switch', module = 'switch', class = 'member', role = 'wired-fabric', member = 'switch-main', + facts = { wired_observation_status = { 'raw', 'host', 'wired', 'provider', 'switch-main', 'status' } }, + }, + }, + }) + if not catalogue then fail(err) end + + local model = device_model.new() + ok((select(1, model:apply_catalogue(1, catalogue))) ~= nil, 'catalogue apply') + local snap = model:snapshot() + local ok_pub, perr = device_publisher.publish_summary_now(conn, snap, { emit_event = false }) + if ok_pub ~= true then fail(perr or 'device assembly publish failed') end +end + +local function retain_raw_wired_facts(conn) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'cm5-local-wired', 'status' }, { state = 'available', available = true }) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'cm5-local-wired', 'state', 'surfaces' }, { surfaces = { + eth0 = { capabilities = { trunk = true }, link = { state = 'up', speed_mbps = 1000 }, attachment = { mode = 'trunk', vlans = { 10, 11, 12, 100, 101 } } }, + } }) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'cm5-local-wired', 'state', 'topology' }, {}) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'status' }, { state = 'available', available = true, mode = 'read_only', driver = 'rtl8380m_http' }) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'surfaces' }, { surfaces = { + ['GE8'] = { + observed_surface = 'GE8', kind = 'switch-port', + capabilities = { trunk = true, access = false, poe = false }, + link = { state = 'up', speed_mbps = 1000 }, + attachment = { mode = 'trunk', vlans = { 10, 11, 12, 100, 101 } }, + }, + ['GE1'] = { + observed_surface = 'GE1', kind = 'ethernet-port', + capabilities = { access = true, trunk = true, poe = true }, + link = { state = 'up', speed_mbps = 1000 }, + attachment = { mode = 'access', vlan = 100 }, + }, + ['GE2'] = { + observed_surface = 'GE2', kind = 'ethernet-port', + capabilities = { access = true, trunk = true, poe = true }, + link = { state = 'down' }, + attachment = { mode = 'access', vlan = 101 }, + }, + } }) + conn:retain({ 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'topology' }, { trunks = { ['GE8'] = { vlans = { 10, 11, 12, 100, 101 } } } }) +end + +local function raw_provider(conn, component) + local status = retained_payload(conn, { 'raw', 'host', 'wired', 'provider', component, 'status' }) + local surfaces_payload = retained_payload(conn, { 'raw', 'host', 'wired', 'provider', component, 'state', 'surfaces' }) + local topology_payload = retained_payload(conn, { 'raw', 'host', 'wired', 'provider', component, 'state', 'topology' }) + return { status = status, surfaces = surfaces_payload.surfaces or surfaces_payload, topology = topology_payload } +end + +local function compose_wired(conn) + local assembly = retained_payload(conn, { 'state', 'device', 'assembly' }) + + local intent, err = wired_config.normalise({ + schema = wired_config.SCHEMA, + surfaces = { + ['cm5-eth0'] = { + kind = 'direct-nic', role = 'internal-trunk', protected = true, + attachment = { mode = 'trunk', required_segments = { 'mgmt', 'switch_control', 'fabric' }, user_segments = 'all-realised-user-segments' }, + }, + ['switch-uplink-cm5'] = { + kind = 'switch-port', role = 'internal-trunk', protected = true, + attachment = { mode = 'trunk', required_segments = { 'mgmt', 'switch_control', 'fabric' }, user_segments = 'all-realised-user-segments' }, + }, + ['lan-1'] = { + kind = 'ethernet-port', role = 'access', name = 'LAN 1', capabilities = { poe = true }, + attachment = { mode = 'access', segment = 'lan' }, + }, + ['lan-2'] = { + kind = 'ethernet-port', role = 'access', name = 'LAN 2', capabilities = { poe = true }, + attachment = { mode = 'access', segment = 'guest' }, + }, + }, + }) + if not intent then fail(err) end + + local snap = { + assembly = assembly, + net = { segments = { + mgmt = { kind = 'system', protected = true, vlan = { id = 10 } }, + switch_control = { kind = 'system', protected = true, vlan = { id = 11 } }, + fabric = { kind = 'system', protected = true, vlan = { id = 12 } }, + lan = { kind = 'user', vlan = { id = 100 } }, + guest = { kind = 'guest', vlan = { id = 101 } }, + } }, + config_intent = intent, + observations = { + ['cm5-local-wired'] = raw_provider(conn, 'cm5-local-wired'), + ['switch-main'] = raw_provider(conn, 'switch-main'), + }, + stats = {}, + } + wired_service._test.rebuild_derived(snap) + local ok_pub, perr = wired_publisher.publish_all_now(conn, snap, wired_publisher.new_state()) + if ok_pub ~= true then fail(perr or 'wired publish failed') end + return snap +end + +fibers.run(function() + local b = bus.new() + local conn = b:connect({ origin_base = { kind = 'vm-test' } }) + conn:retain({ 'state', 'net', 'segments' }, { rev = 1, segments = { + mgmt = { kind = 'system', protected = true, vlan = { id = 10 } }, + switch_control = { kind = 'system', protected = true, vlan = { id = 11 } }, + fabric = { kind = 'system', protected = true, vlan = { id = 12 } }, + lan = { kind = 'user', vlan = { id = 100 } }, + guest = { kind = 'guest', vlan = { id = 101 } }, + } }) + + build_device_assembly(conn) + retain_raw_wired_facts(conn) + local assembly = retained_payload(conn, { 'state', 'device', 'assembly' }) + eq(assembly.surfaces['lan-1'].observed_surface, 'GE1', 'device assembly maps lan-1') + + local snap = compose_wired(conn) + eq(snap.state, 'running', 'wired composed state') + eq(#(snap.violations or {}), 0, 'wired violations') + + local lan1 = retained_payload(conn, wired_topics.surface('lan-1')) + eq(lan1.surface_id, 'lan-1', 'lan-1 surface id') + eq(lan1.attachment.segment, 'lan', 'lan-1 segment') + eq(lan1.availability.state, 'available', 'lan-1 available') + + local uplink = retained_payload(conn, wired_topics.surface('switch-uplink-cm5')) + eq(uplink.availability.state, 'available', 'switch uplink available') + local violations = retained_payload(conn, wired_topics.violations()) + eq(#(violations.violations or {}), 0, 'published violations empty') +end) + +print('devicecode Big Box phase 1 composition: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_devicecode_bigbox_phase1_composition.lua" "$REMOTE/run_devicecode_bigbox_phase1_composition.lua" +"$SSH" "cd '$REMOTE' && lua ./run_devicecode_bigbox_phase1_composition.lua" diff --git a/tests/integration/openwrt_vm/tests/test_devicecode_exec_containment.sh b/tests/integration/openwrt_vm/tests/test_devicecode_exec_containment.sh new file mode 100755 index 00000000..c5441e76 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_devicecode_exec_containment.sh @@ -0,0 +1,410 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-exec-containment-test" +WORK="$VM_DIR/work/exec-containment-test" + +mkdir -p "$WORK" + +cat > "$WORK/run_devicecode_exec_containment.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local exec = require 'fibers.io.exec' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' + +local perform = fibers.perform + +math.randomseed(os.time()) + +local function fail(msg) + error(msg, 2) +end + +local function read_file(path) + local f = io.open(path, 'r') + if not f then return nil end + local s = f:read('*a') + f:close() + return s +end + +local function file_exists(path) + local f = io.open(path, 'r') + if f then f:close(); return true end + return false +end + +local function pid_alive(pid) + pid = tonumber(pid) + if not pid then return false end + local ok = os.execute(('kill -0 %d >/dev/null 2>&1'):format(pid)) + return ok == true or ok == 0 +end + +local function kill_pid(pid) + pid = tonumber(pid) + if pid then os.execute(('kill -KILL %d >/dev/null 2>&1 || true'):format(pid)) end +end + +local function wait_until(pred, timeout_s) + local deadline = fibers.now() + timeout_s + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.05)) + end + return pred() +end + +local function wait_for_proc(proc, timeout_s, label) + local is_exit, status, code, sig, err = perform(op.boolean_choice( + proc:run_op(), + sleep.sleep_op(timeout_s) + )) + if not is_exit then + pcall(function() proc:kill(9) end) + fail(label .. ' did not exit within ' .. tostring(timeout_s) .. 's') + end + return status, code, sig, err +end + +local function read_line_with_timeout(stream, timeout_s, label) + local is_line, line, err = perform(op.boolean_choice( + stream:read_line_op(), + sleep.sleep_op(timeout_s) + )) + if not is_line then + fail(label .. ' did not produce a line within ' .. tostring(timeout_s) .. 's') + end + if not line then + fail(label .. ' closed before producing a line: ' .. tostring(err)) + end + return line +end +local function test_process_group_shutdown_kills_grandchild() + print('running: process_group_shutdown_kills_grandchild_on_openwrt') + + if type(exec.supports) == 'function' and not exec.supports('process_group') then + print('skipping process_group test: backend does not advertise process_group') + return + end + + local marker = ('/tmp/dc-exec-pg-%d-%d'):format(os.time(), math.random(1000000)) + os.remove(marker) + + local script = [[ +trap '' TERM +( trap '' TERM; while :; do sleep 1; done ) & +echo $! > "$1" +while :; do sleep 1; done +]] + + local proc = exec.command { + 'sh', '-c', script, 'sh', marker, + stdin = 'null', stdout = 'pipe', stderr = 'stdout', + flags = { process_group = true }, + } + + local out, start_err = proc:stdout_stream() + assert(out, 'failed to start process-group command: ' .. tostring(start_err)) + assert(wait_until(function() return file_exists(marker) end, 3.0), 'grandchild pid marker was not written') + + local grandchild = tonumber((read_file(marker) or ''):match('(%d+)')) + assert(grandchild, 'could not parse grandchild pid marker') + assert(pid_alive(grandchild), 'grandchild was not alive before shutdown') + + local status, _code, _sig, err = perform(proc:shutdown_op(0.15)) + assert(status == 'exited' or status == 'signalled', + 'process-group command did not reach a terminal state: ' .. tostring(status) .. ' err=' .. tostring(err)) + + local dead = wait_until(function() return not pid_alive(grandchild) end, 2.0) + if not dead then kill_pid(grandchild) end + assert(dead, 'grandchild remained alive after process-group shutdown: pid=' .. tostring(grandchild)) + os.remove(marker) +end + +local function test_signal_bridge_sigterm_cleans_owned_grandchild() + print('running: signal_bridge_sigterm_cleans_owned_grandchild_on_openwrt') + + if type(exec.supports) == 'function' and not exec.supports('process_group') then + print('skipping signal bridge process cleanup test: backend does not advertise process_group') + return + end + + local marker = ('/tmp/dc-signal-bridge-%d-%d'):format(os.time(), math.random(1000000)) + os.remove(marker) + + local proc = exec.command { + 'lua', './signal_bridge_child.lua', marker, + stdin = 'null', stdout = 'pipe', stderr = 'stdout', + } + local out, start_err = proc:stdout_stream() + assert(out, 'failed to start signal bridge child: ' .. tostring(start_err)) + + local line = read_line_with_timeout(out, 5.0, 'signal bridge child') + local grandchild = tonumber(line:match('READY%s+(%d+)')) + assert(grandchild, 'unexpected signal bridge child line: ' .. tostring(line)) + assert(pid_alive(grandchild), 'owned grandchild was not alive before SIGTERM') + + local ok, kill_err = proc:kill(15) + assert(ok, 'failed to send SIGTERM to signal bridge child: ' .. tostring(kill_err)) + + local status, code, sig, err = wait_for_proc(proc, 6.0, 'signal bridge child') + assert(status == 'exited' or status == 'signalled', + 'signal bridge child did not terminate cleanly: status=' .. tostring(status) .. + ' code=' .. tostring(code) .. ' sig=' .. tostring(sig) .. ' err=' .. tostring(err)) + + local dead = wait_until(function() return not pid_alive(grandchild) end, 3.0) + if not dead then kill_pid(grandchild) end + assert(dead, 'owned grandchild remained alive after SIGTERM/root-scope cancellation: pid=' .. tostring(grandchild)) + os.remove(marker) +end + +local function test_parent_death_signal_cleans_helper() + print('running: parent_death_signal_cleans_helper_on_openwrt') + + if type(exec.supports) == 'function' and not exec.supports('parent_death_signal') then + print('skipping parent_death_signal test: backend does not advertise parent_death_signal') + return + end + + local marker = ('/tmp/dc-parent-death-%d-%d'):format(os.time(), math.random(1000000)) + os.remove(marker) + + local proc = exec.command { + 'lua', './parent_death_launcher.lua', marker, + stdin = 'null', stdout = 'pipe', stderr = 'stdout', + } + local out, start_err = proc:stdout_stream() + assert(out, 'failed to start parent-death launcher: ' .. tostring(start_err)) + + local line = read_line_with_timeout(out, 5.0, 'parent-death launcher') + if line:match('^SKIP') then + print(line) + wait_for_proc(proc, 3.0, 'parent-death launcher skip') + return + end + + local helper = tonumber(line:match('LAUNCHED%s+(%d+)')) + assert(helper, 'unexpected parent-death launcher line: ' .. tostring(line)) + assert(pid_alive(helper), 'parent-death helper was not alive before parent exit') + + local status, code, sig, err = wait_for_proc(proc, 3.0, 'parent-death launcher') + assert(status == 'exited' and code == 0, + 'parent-death launcher should exit 0: status=' .. tostring(status) .. + ' code=' .. tostring(code) .. ' sig=' .. tostring(sig) .. ' err=' .. tostring(err)) + + local dead = wait_until(function() return not pid_alive(helper) end, 3.0) + if not dead then kill_pid(helper) end + assert(dead, 'parent-death-owned helper remained alive after launcher parent exit: pid=' .. tostring(helper)) + os.remove(marker) +end + +fibers.run(function() + local features = type(exec.features) == 'function' and exec.features() or {} + print(('exec backend features: process_group=%s parent_death_signal=%s') + :format(tostring(features.process_group), tostring(features.parent_death_signal))) + + test_process_group_shutdown_kills_grandchild() + test_signal_bridge_sigterm_cleans_owned_grandchild() + test_parent_death_signal_cleans_helper() +end) + +print('devicecode exec containment on OpenWrt VM: ok') +LUA + +cat > "$WORK/signal_bridge_child.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local exec = require 'fibers.io.exec' +local sleep = require 'fibers.sleep' +local signal_bridge = require 'devicecode.signal_bridge' + +local perform = fibers.perform +local marker = assert(arg and arg[1], 'marker path required') + +local function read_file(path) + local f = io.open(path, 'r') + if not f then return nil end + local s = f:read('*a') + f:close() + return s +end + +local function file_exists(path) + local f = io.open(path, 'r') + if f then f:close(); return true end + return false +end + +local function wait_until(pred, timeout_s) + local deadline = fibers.now() + timeout_s + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.05)) + end + return pred() +end + +local function owned_flags() + local flags = { process_group = true } + if type(exec.supports) == 'function' and exec.supports('parent_death_signal') then + flags.parent_death_signal = 'TERM' + end + return flags +end + +local ok, err = xpcall(function() + fibers.run(function(scope) + local sig_ok, sig_err = signal_bridge.install(scope, { TERM = true, INT = true }) + assert(sig_ok, 'signal bridge install failed: ' .. tostring(sig_err)) + + os.remove(marker) + + local script = [[ +trap '' TERM +( trap '' TERM; while :; do sleep 1; done ) & +echo $! > "$1" +while :; do sleep 1; done +]] + + local cmd = exec.command { + 'sh', '-c', script, 'sh', marker, + stdin = 'null', stdout = 'null', stderr = 'null', + flags = owned_flags(), + } + + local spawned, spawn_err = scope:spawn(function() + perform(cmd:run_op()) + end) + assert(spawned, 'failed to spawn command waiter: ' .. tostring(spawn_err)) + + assert(wait_until(function() return file_exists(marker) end, 3.0), 'owned grandchild marker was not written') + local grandchild = tonumber((read_file(marker) or ''):match('(%d+)')) + assert(grandchild, 'could not parse owned grandchild pid') + + io.stdout:write('READY ' .. tostring(grandchild) .. '\n') + io.stdout:flush() + + while true do + perform(sleep.sleep_op(1.0)) + end + end) +end, tostring) + +if not ok then + if tostring(err):match('signal:TERM') or tostring(err):match('signal:INT') or tostring(err):match('scope cancelled') then + os.exit(0) + end + io.stderr:write('signal bridge child failed: ' .. tostring(err) .. '\n') + os.exit(1) +end +LUA + +cat > "$WORK/parent_death_launcher.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local exec = require 'fibers.io.exec' +local sleep = require 'fibers.sleep' + +local perform = fibers.perform +local marker = assert(arg and arg[1], 'marker path required') + +local function read_file(path) + local f = io.open(path, 'r') + if not f then return nil end + local s = f:read('*a') + f:close() + return s +end + +local function file_exists(path) + local f = io.open(path, 'r') + if f then f:close(); return true end + return false +end + +local function wait_until(pred, timeout_s) + local deadline = fibers.now() + timeout_s + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.05)) + end + return pred() +end + +fibers.run(function() + if type(exec.supports) == 'function' and not exec.supports('parent_death_signal') then + print('SKIP parent_death_signal unsupported by selected backend') + return + end + + os.remove(marker) + + local script = [[ +echo $$ > "$1" +while :; do sleep 1; done +]] + + local cmd = exec.command { + 'sh', '-c', script, 'sh', marker, + stdin = 'null', stdout = 'pipe', stderr = 'null', + flags = { parent_death_signal = 'TERM' }, + } + + local out, start_err = cmd:stdout_stream() + if not out then + local msg = tostring(start_err) + if msg:match('parent_death_signal') and msg:match('not supported') then + print('SKIP parent_death_signal unsupported by selected backend: ' .. msg) + return + end + error('failed to start parent-death helper: ' .. msg) + end + assert(wait_until(function() return file_exists(marker) end, 3.0), 'parent-death helper marker was not written') + + local helper = tonumber((read_file(marker) or ''):match('(%d+)')) + assert(helper, 'could not parse parent-death helper pid') + + io.stdout:write('LAUNCHED ' .. tostring(helper) .. '\n') + io.stdout:flush() + + -- Deliberately bypass Fibers scope unwinding and command finalisers. The + -- helper must still die because the exec backend was asked for parent_death_signal. + os.exit(0) +end) +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_devicecode_exec_containment.lua" "$REMOTE/run_devicecode_exec_containment.lua" +"$SCP_TO" "$WORK/signal_bridge_child.lua" "$REMOTE/signal_bridge_child.lua" +"$SCP_TO" "$WORK/parent_death_launcher.lua" "$REMOTE/parent_death_launcher.lua" +"$SSH" "cd '$REMOTE' && lua ./run_devicecode_exec_containment.lua" diff --git a/tests/integration/openwrt_vm/tests/test_devicecode_full_stack_mock_hal.sh b/tests/integration/openwrt_vm/tests/test_devicecode_full_stack_mock_hal.sh new file mode 100755 index 00000000..fac24913 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_devicecode_full_stack_mock_hal.sh @@ -0,0 +1,523 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-full-stack-mock-hal-test" +WORK="$VM_DIR/work/full-stack-mock-hal-test" + +mkdir -p "$WORK" + +cat > "$WORK/run_devicecode_full_stack_mock_hal.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + './vendor/?.lua', './vendor/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local safe = require 'coxpcall' +local busmod = require 'bus' + +local json_ok, json = pcall(require, 'cjson.safe') +if not json_ok or not json then json = require 'cjson' end + +-- The OpenWrt VM dependency set deliberately stays small and does not +-- include lua-http. The real HTTP service still needs lua-http's URI/header +-- modules at require time, so provide a tiny URI/header-compatible boundary for +-- this full-stack mock-HAL test. The test does not exercise real HTTP transport. +package.preload['http.util'] = package.preload['http.util'] or function () + local M = {} + function M.split_authority(authority, scheme) + authority = tostring(authority or '') + local host, port + if authority:sub(1, 1) == '[' then + host, port = authority:match('^%[([^%]]+)%]:(%d+)$') + if not host then host = authority:match('^%[([^%]]+)%]$') end + else + host, port = authority:match('^([^:]+):(%d+)$') + if not host then host = authority end + end + if host == '' then error('invalid authority') end + port = tonumber(port) or ((scheme == 'https' or scheme == 'wss') and 443 or 80) + return host, port + end + return M +end + +local function mock_headers_new(fields) + local h = { _items = {}, _order = {} } + local function norm(k) return tostring(k or ''):lower() end + function h:get(k) return self._items[norm(k)] end + function h:upsert(k, v) + k = norm(k) + if self._items[k] == nil then self._order[#self._order + 1] = k end + self._items[k] = tostring(v or '') + return true + end + function h:append(k, v) return self:upsert(k, v) end + function h:delete(k) self._items[norm(k)] = nil; return true end + function h:remove(k) return self:delete(k) end + function h:clone() + local c = mock_headers_new() + for _, k in ipairs(self._order) do if self._items[k] ~= nil then c:upsert(k, self._items[k]) end end + return c + end + function h:each() + local i = 0 + return function () + while true do + i = i + 1 + local k = self._order[i] + if k == nil then return nil end + if self._items[k] ~= nil then return k, self._items[k] end + end + end + end + for k, v in pairs(fields or {}) do h:upsert(k, v) end + return h +end + +package.preload['http.headers'] = package.preload['http.headers'] or function () + return { new = function () return mock_headers_new() end } +end + +package.preload['http.request'] = package.preload['http.request'] or function () + local M = {} + function M.new_from_uri(uri) + local scheme, rest = tostring(uri or ''):match('^([%a][%w+.-]*)://(.+)$') + if not scheme or rest == '' then return nil, 'scheme not valid' end + local authority, path = rest:match('^([^/]*)(/.*)$') + if not authority then authority, path = rest, '/' end + if authority == '' then return nil, 'invalid authority' end + local headers = mock_headers_new({ + [':scheme'] = scheme:lower(), + [':authority'] = authority, + [':path'] = path, + [':method'] = 'GET', + }) + return { + headers = headers, + set_body = function (self, body) self.body = body; return true end, + go = function () return nil, 'mock_http_transport_unavailable' end, + } + end + return M +end + +-- The OpenWrt VM dependency set also omits lua-openssl. The Wi-Fi +-- service requires openssl.digest at load time for deterministic guest/user id +-- generation. The full-stack mock-HAL lane disables Wi-Fi radios/SSIDs, but we +-- still load the real Wi-Fi service, so provide a small deterministic digest +-- boundary for this test harness. This is not a cryptographic implementation. +package.preload['openssl.digest'] = package.preload['openssl.digest'] or function () + local M = {} + local function hex32(n) + return string.format('%08x', n % 0x100000000) + end + local function simple_digest(_algo, input) + input = tostring(input or '') + local h1, h2, h3, h4 = 2166136261, 16777619, 3141592653, 2718281829 + for i = 1, #input do + local b = input:byte(i) + h1 = ((h1 + b) * 16777619) % 0x100000000 + h2 = ((h2 + b) * 109951) % 0x100000000 + h3 = ((h3 + ((b * i) % 0x100000000)) * 65599) % 0x100000000 + h4 = ((h4 + h1 + h2 + h3 + b) * 33) % 0x100000000 + end + return hex32(h1) .. hex32(h2) .. hex32(h3) .. hex32(h4) .. hex32((h1 + h3) % 0x100000000) .. hex32((h2 + h4) % 0x100000000) .. hex32((h1 + h4) % 0x100000000) .. hex32((h2 + h3) % 0x100000000) + end + function M.digest(algo, input) return simple_digest(algo, input) end + function M.new(algo) + local chunks = {} + return { + update = function (_, part) chunks[#chunks + 1] = tostring(part or ''); return true end, + final = function (_, part) + if part ~= nil then chunks[#chunks + 1] = tostring(part) end + local s = table.concat(chunks) + return { + tohex = function () return simple_digest(algo, s) end, + } + end, + } + end + return M +end + +local config_service = require 'services.config' +local device_service = require 'services.device' +local fabric_service = require 'services.fabric' +local gsm_service = require 'services.gsm' +local http_service = require 'services.http' +local metrics_service = require 'services.metrics' +local monitor_service = require 'services.monitor' +local net_service = require 'services.net' +local system_service = require 'services.system' +local time_service = require 'services.time' +local ui_service = require 'services.ui' +local update_service = require 'services.update' +local wifi_service = require 'services.wifi' +local wired_service = require 'services.wired' + +local perform = fibers.perform + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_eq(a, b, msg) if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end return v end + +local function read_file(path) + local f = assert(io.open(path, 'rb')) + local s = assert(f:read('*a')) + f:close() + return s +end + +local function copy(v, seen) + if type(v) ~= 'table' then return v end + seen = seen or {} + if seen[v] then return seen[v] end + local out = {} + seen[v] = out + for k, subv in pairs(v) do out[copy(k, seen)] = copy(subv, seen) end + return out +end + +local function wait_until(label, pred, timeout_s) + local deadline = fibers.now() + (timeout_s or 2.0) + local last + while fibers.now() < deadline do + local ok, value = safe.pcall(pred) + if ok and value then return value end + if not ok then last = value end + perform(sleep.sleep_op(0.01)) + end + local ok, value = safe.pcall(pred) + if ok and value then return value end + fail(label .. (last and (': ' .. tostring(last)) or '')) +end + +local function retained_payload(conn, topic) + local view = conn:retained_view(topic) + local msg = view:get(topic) + view:close() + return msg and msg.payload or nil +end + +local function wait_retained(conn, topic, pred, label, timeout_s) + return wait_until(label or ('retained ' .. table.concat(topic, '/')), function() + local p = retained_payload(conn, topic) + if p ~= nil and (not pred or pred(p)) then return p end + return nil + end, timeout_s or 2.0) +end + +local function fake_http_driver() + return { + started = false, + terminated = nil, + start = function(self) self.started = true; return true end, + terminate = function(self, reason) self.terminated = reason or true; return true end, + run_op = function(_, _, fn) return fibers.guard(function() return fibers.always(fn()) end) end, + } +end + +local function bind_cap(scope, conn, class, id, methods, opts) + opts = opts or {} + conn:retain({ 'cap', class, id, 'state' }, 'added') + conn:retain({ 'cap', class, id, 'status' }, { state = 'available', available = true, reason = 'mock_hal' }) + local offerings = {} + for method in pairs(methods or {}) do offerings[method] = true end + conn:retain({ 'cap', class, id, 'meta' }, { offerings = offerings, mock = true }) + + local endpoints = {} + for method, handler in pairs(methods or {}) do + local ep = assert(conn:bind({ 'cap', class, id, 'rpc', method }, { queue_len = opts.queue_len or 64 })) + endpoints[#endpoints + 1] = ep + local ok, err = scope:spawn(function() + while true do + local req = perform(ep:recv_op()) + if req == nil then return end + local function run_one() + local ok_h, reply = safe.pcall(handler, req.payload or {}, req) + if not ok_h then reply = { ok = false, reason = tostring(reply) } end + if type(reply) ~= 'table' then reply = { ok = false, reason = tostring(reply) } end + local ok_reply = safe.pcall(function() req:reply(reply) end) + if not ok_reply then + -- The operation may have been deliberately detached by the caller; + -- the mock HAL still records completion through its own counters. + end + end + if opts.concurrent == true then + scope:spawn(run_one) + else + run_one() + end + end + end) + assert_true(ok, err) + end + return endpoints +end + +local function normalise_service_config(base) + local cfg = copy(base) + -- Keep the production shape where useful, but avoid hardware-dependent work. + cfg.fabric.data.links = {} + cfg.gsm.data.modems.known = {} + cfg.wifi.data.radios = {} + cfg.wifi.data.ssids = {} + cfg.wifi.data.band_steering = nil + cfg.ui.data.http = cfg.ui.data.http or {} + cfg.ui.data.http.enabled = false + cfg.ui.data.updates = { + upload = { enabled = false, max_bytes = 1, require_auth = false, component = 'mcu', create_job = false, start_job = false }, + commit = { require_auth = false }, + } + cfg.update.data.components = {} + cfg.update.data.bundled = { enabled = false, components = {} } + cfg.metrics.data.cloud_url = 'http://127.0.0.1/' + cfg.metrics.data.templates = {} + cfg.metrics.data.pipelines = {} + cfg.system.data.alarms = {} + return cfg +end + +local function encoded_services_config() + local raw = assert(json.decode(read_file('./configs/bigbox-v1-cm-2.json'))) + return assert(json.encode(normalise_service_config(raw))) +end + +local function setup_mock_hal(scope, conn, counters) + counters.fs_reads = {} + counters.fs_writes = {} + counters.network_apply_calls = 0 + counters.network_apply_active = 0 + counters.network_apply_max_active = 0 + counters.network_apply_generations = {} + + local service_blob = encoded_services_config() + local mainflux_blob = assert(json.encode({ thing_key = 'mock-thing', channels = {} })) + + conn:retain({ 'svc', 'hal', 'announce' }, { role = 'hal', backend = 'openwrt-vm-mock-hal' }) + conn:retain({ 'svc', 'hal', 'status' }, { service = 'hal', state = 'running', ready = true, mock = true }) + + bind_cap(scope, conn, 'fs', 'config', { + read = function(req) + local filename = req.filename or req.path or '' + counters.fs_reads[#counters.fs_reads + 1] = filename + return { ok = true, reason = service_blob } + end, + }) + + bind_cap(scope, conn, 'fs', 'credentials', { + read = function(req) + counters.fs_reads[#counters.fs_reads + 1] = req.filename or req.path or '' + return { ok = true, reason = mainflux_blob } + end, + }) + + bind_cap(scope, conn, 'fs', 'state', { + read = function(req) + counters.fs_reads[#counters.fs_reads + 1] = req.filename or req.path or '' + return { ok = false, reason = 'not found', code = 'not_found' } + end, + write = function(req) + counters.fs_writes[#counters.fs_writes + 1] = req + return { ok = true, reason = true } + end, + }) + + local store = {} + bind_cap(scope, conn, 'control-store', 'update', { + list = function(req) + local keys, prefix = {}, req.prefix or '' + for k in pairs(store) do if k:sub(1, #prefix) == prefix then keys[#keys + 1] = k end end + table.sort(keys) + return { ok = true, reason = keys } + end, + get = function(req) + if store[req.key] == nil then return { ok = false, reason = 'not found', code = 'not_found' } end + return { ok = true, reason = store[req.key] } + end, + put = function(req) store[req.key] = req.data; return { ok = true, reason = true } end, + delete = function(req) store[req.key] = nil; return { ok = true, reason = true } end, + }) + + bind_cap(scope, conn, 'artifact-store', 'main', { + put = function() return { ok = true, reason = { stored = true } } end, + get = function() return { ok = false, reason = 'not found', code = 'not_found' } end, + stat = function() return { ok = false, reason = 'not found', code = 'not_found' } end, + }) + + local network_gate = { block_next = false, blocked = nil } + counters.network_gate = network_gate + bind_cap(scope, conn, 'network-config', 'main', { + apply = function(req) + counters.network_apply_calls = counters.network_apply_calls + 1 + counters.network_apply_active = counters.network_apply_active + 1 + if counters.network_apply_active > counters.network_apply_max_active then + counters.network_apply_max_active = counters.network_apply_active + end + local gen = req.opts and req.opts.generation or req.intent and req.intent.generation or counters.network_apply_calls + counters.network_apply_generations[#counters.network_apply_generations + 1] = gen + if network_gate.block_next and network_gate.blocked == nil then + network_gate.blocked = gen + while network_gate.block_next do perform(sleep.sleep_op(0.01)) end + end + counters.network_apply_active = counters.network_apply_active - 1 + return { ok = true, reason = { ok = true, applied = true, changed = true, backend = 'mock-openwrt', generation = gen } } + end, + apply_live_weights = function() return { ok = true, reason = { ok = true, applied = true } } end, + apply_shaping = function() return { ok = true, reason = { ok = true, applied = true } } end, + }) + + bind_cap(scope, conn, 'network-state', 'main', { + watch = function() return { ok = true, reason = { ok = true, watch = true } } end, + }) + bind_cap(scope, conn, 'network-diagnostics', 'main', { + speedtest = function() return { ok = true, reason = { ok = true, skipped = true, reason = 'mock' } } end, + read_counters = function() return { ok = true, reason = { ok = true, counters = {} } } end, + }) + + bind_cap(scope, conn, 'time', 'mock', {}) + conn:retain({ 'cap', 'time', 'mock', 'state', 'synced' }, { synced = true, stratum = 1, accuracy_seconds = 0.01 }) + conn:publish({ 'cap', 'time', 'mock', 'event', 'synced' }, { synced = true, stratum = 1 }) + + bind_cap(scope, conn, 'platform', '1', { info = function() return { ok = true, reason = { model = 'openwrt-vm' } } end }) + + return counters +end + +local function spawn_service(root_scope, name, fn) + local child = assert(root_scope:child()) + local ok, err = child:spawn(function() + local ok_run, run_err = safe.pcall(fn) + if not ok_run then error(name .. ' failed: ' .. tostring(run_err), 0) end + end) + assert_true(ok, err) + return child +end + +local function service_status_ready_or_running(conn, name) + local p = retained_payload(conn, { 'svc', name, 'status' }) + if p and (p.state == 'running' or p.ready == true or p.state == 'waiting_for_hal' or p.state == 'waiting_for_job_store' or p.state == 'starting') then + return true + end + -- The VM lane starts services directly rather than through devicecode.main's + -- supervisor. Services that rely on supervisor-owned svc//status are + -- therefore accepted using their own public state projection instead. + if name == 'wired' then + return retained_payload(conn, { 'state', 'wired', 'summary' }) ~= nil + end + return false +end + +local function run_full_stack() + local bus = busmod.new() + local counters = {} + + fibers.run(function(root_scope) + local conn = bus:connect({ origin_base = { kind = 'openwrt-vm-full-stack-test' } }) + local hal_scope = assert(root_scope:child()) + setup_mock_hal(hal_scope, conn, counters) + + local scopes = {} + local function add(name, module, opts) + opts = opts or {} + opts.env = 'openwrt-vm-test' + scopes[#scopes + 1] = spawn_service(root_scope, name, function() module.start(conn, opts) end) + end + + add('config', config_service, { timings = { heartbeat_s = 60, hal_wait_timeout_s = 2 } }) + add('device', device_service, {}) + add('fabric', fabric_service, {}) + add('gsm', gsm_service, {}) + add('http', http_service, { driver = fake_http_driver(), observability = { stats_interval_s = 30, status_interval_s = 30 } }) + add('metrics', metrics_service, { heartbeat_s = 60 }) + add('monitor', monitor_service, {}) + add('net', net_service, {}) + add('system', system_service, {}) + add('time', time_service, {}) + add('ui', ui_service, {}) + add('update', update_service, {}) + add('wifi', wifi_service, {}) + add('wired', wired_service, {}) + + wait_retained(conn, { 'cfg', 'net' }, nil, 'config service should publish net config', 3) + wait_retained(conn, { 'svc', 'config', 'status' }, function(p) return p.state == 'running' end, 'config running', 3) + wait_retained(conn, { 'svc', 'http', 'status' }, function(p) return p.state == 'running' end, 'http running', 3) + wait_retained(conn, { 'cap', 'http', 'main', 'status' }, function(p) return p.available == true or p.state == 'available' or p.state == 'ready' end, 'http capability available', 3) + wait_retained(conn, { 'cap', 'http', 'main', 'state', 'stats' }, function(p) return p.ready == true end, 'http stats ready', 3) + wait_retained(conn, { 'svc', 'ui', 'status' }, function(p) return p.state == 'running' end, 'ui running', 3) + wait_retained(conn, { 'svc', 'update', 'status' }, function(p) return p.state == 'running' or p.ready == true or p.state == 'starting' end, 'update admitted', 3) + wait_retained(conn, { 'state', 'wired', 'summary' }, nil, 'wired summary retained', 3) + wait_retained(conn, { 'state', 'wired', 'surface', 'cm5-eth0' }, nil, 'wired configured surface retained', 3) + wait_retained(conn, { 'state', 'time', 'synced' }, function(p) return p == true end, 'time sync retained', 3) + wait_until('net should call mocked network apply', function() return counters.network_apply_calls >= 1 end, 3) + + for _, name in ipairs({ 'device', 'fabric', 'gsm', 'metrics', 'monitor', 'net', 'system', 'time', 'wifi', 'wired' }) do + assert_true(service_status_ready_or_running(conn, name), 'service did not publish acceptable status: ' .. name) + end + + -- Exercise service churn against a slow mock network apply. This guards the + -- full service graph against turning one config replacement into overlapping + -- host-mutating HAL requests. + local raw = assert(json.decode(encoded_services_config())) + raw.net.rev = 2 + raw.net.data.description = 'openwrt-vm mock-hal apply gate rev2' + counters.network_gate.block_next = true + conn:retain({ 'cfg', 'net' }, raw.net) + wait_until('second network apply should enter blocked mock HAL', function() + return counters.network_gate.blocked ~= nil + end, 3) + + local before_release_calls = counters.network_apply_calls + raw.net.rev = 3 + raw.net.data.description = 'openwrt-vm mock-hal apply gate rev3' + conn:retain({ 'cfg', 'net' }, raw.net) + perform(sleep.sleep_op(0.15)) + assert_eq(counters.network_apply_active, 1, 'mock HAL should keep one host-mutating network apply active while first is blocked') + assert_eq(counters.network_apply_max_active, 1, 'mock HAL network apply must be single-flight') + + counters.network_gate.block_next = false + wait_until('blocked network apply should drain', function() return counters.network_apply_active == 0 end, 3) + wait_until('latest pending network config should be applied after drain', function() + return counters.network_apply_calls > before_release_calls + end, 3) + assert_eq(counters.network_apply_max_active, 1, 'mock HAL network apply must remain single-flight after pending drain') + + -- Ensure UI default read model is not pulling raw HAL state into public UI state. + local raw_seen = false + local state_view = conn:retained_view({ 'state' }) + state_view:close() + local raw_msg = retained_payload(conn, { 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'counters' }) + if raw_msg ~= nil then raw_seen = true end + assert_true(raw_seen == false, 'mock stack should not depend on raw HAL topics for UI/readiness') + + for _, scope in ipairs(scopes) do scope:cancel('test complete') end + hal_scope:cancel('test complete') + for _, scope in ipairs(scopes) do perform(scope:join_op()) end + perform(hal_scope:join_op()) + end) + + print(('devicecode full stack mock HAL VM test: ok; services=14 network_apply_calls=%d fs_reads=%d'):format(counters.network_apply_calls or 0, #(counters.fs_reads or {}))) +end + +run_full_stack() +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$ROOT_DIR/src/configs" "$REMOTE/configs" +"$SCP_TO" "$WORK/run_devicecode_full_stack_mock_hal.lua" "$REMOTE/run_devicecode_full_stack_mock_hal.lua" +"$SSH" "cd '$REMOTE' && CONFIG_TARGET=openwrt-vm-full-stack lua ./run_devicecode_full_stack_mock_hal.lua" diff --git a/tests/integration/openwrt_vm/tests/test_devicecode_openwrt_no_blocking_os_io.sh b/tests/integration/openwrt_vm/tests/test_devicecode_openwrt_no_blocking_os_io.sh new file mode 100755 index 00000000..5bb261b3 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_devicecode_openwrt_no_blocking_os_io.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" + +cd "$ROOT_DIR" + +PATHS=" +src/services/hal/backends/openwrt +src/services/hal/backends/network/providers/openwrt +src/services/net +" + +matches="" +for path in $PATHS; do + if [ -d "$path" ]; then + found="$(grep -RInE '\b(os\.execute|io\.open|os\.remove|io\.popen|popen)\s*\(' "$path" 2>/dev/null || true)" + if [ -n "$found" ]; then + matches="${matches}${found} +" + fi + fi +done + +if [ -n "$matches" ]; then + printf '%s\n' 'fibres-unaware OS/IO calls found in OpenWrt NET/HAL paths:' >&2 + printf '%s\n' "$matches" >&2 + exit 1 +fi + +printf '%s\n' 'devicecode OpenWrt NET/HAL no blocking OS/IO scan: ok' diff --git a/tests/integration/openwrt_vm/tests/test_devicecode_uci_manager.sh b/tests/integration/openwrt_vm/tests/test_devicecode_uci_manager.sh new file mode 100755 index 00000000..2cf035f0 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_devicecode_uci_manager.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-uci-manager-test" +WORK="$VM_DIR/work/uci-manager-test" + +mkdir -p "$WORK" + +cat > "$WORK/run_devicecode_uci_manager.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', + './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', + './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', + './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', + './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local bus = require 'bus' +local trie = require 'trie' +local uci = require 'uci' +local uci_manager = require 'services.hal.backends.openwrt.uci_manager' +local compat = require 'services.hal.backends.common.uci' + +assert(type(bus.new) == 'function', 'vendored bus did not load') +assert(type(trie.new_pubsub) == 'function', 'vendored trie did not load') + +local perform = fibers.perform + +local function fail(msg) + error(msg, 2) +end + +local function eq(a, b, msg) + if a ~= b then + fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) + end +end + +local function assert_list(v, expected, label) + if type(v) ~= 'table' then + fail((label or 'list') .. ' should be a table, got ' .. type(v)) + end + eq(#v, #expected, (label or 'list') .. ' length') + for i = 1, #expected do + eq(v[i], expected[i], (label or 'list') .. '[' .. i .. ']') + end +end + +local function mkdir_p(path) + local ok = os.execute("mkdir -p '" .. path .. "'") + if ok ~= true and ok ~= 0 then fail('mkdir failed for ' .. path) end +end + +local tmp = '/tmp/dc-devicecode-uci-manager' +os.execute("rm -rf '" .. tmp .. "'") +local conf = tmp .. '/conf' +local save = tmp .. '/save' +mkdir_p(conf) +mkdir_p(save) +local f = assert(io.open(conf .. '/dcuci', 'w')) +f:write('# devicecode manager integration test\n') +f:close() + +local restarts = {} + +fibers.run(function(scope) + local mgr, merr = uci_manager.new({ + confdir = conf, + savedir = save, + allow_fake = false, + debounce_s = 0.02, + run_cmd = function(argv) + restarts[#restarts + 1] = table.concat(argv, ' ') + return true, nil + end, + }) + assert(mgr, merr) + local fake, note = mgr:fake_mode() + eq(fake, false, 'manager should use real libuci-lua') + assert(note == nil or note == false, 'unexpected fake-mode note: ' .. tostring(note)) + assert(mgr:start(scope)) + + local s = mgr:new_session() + s:set('dcuci', 'named', 'example') + s:set('dcuci', 'named', 'enabled', true) + s:set('dcuci', 'named', 'count', 7) + s:set('dcuci', 'named', 'listopt', { 'one', 'two' }) + local anon = s:add('dcuci', 'thing') + s:set('dcuci', anon, 'name', 'anonymous') + s:add_list('dcuci', 'named', 'listopt', 'three') + s:del_list('dcuci', 'named', 'listopt', 'one') + s:rename('dcuci', anon, 'anon_named') + s:reorder('dcuci', 'anon_named', 0) + + local ok, err = perform(s:commit_op('dcuci', { + { kind = 'reload', target = 'network' }, + { '/etc/init.d/network', 'reload' }, + { kind = 'reload', target = 'wifi' }, + })) + assert(ok, 'commit_op failed: ' .. tostring(err)) + + local s2 = mgr:new_session() + s2:rename('dcuci', 'named', 'count', 'renamed_count') + s2:delete('dcuci', 'named', 'enabled') + local ok2, err2 = s2:commit('dcuci') + assert(ok2, 'compat blocking commit failed: ' .. tostring(err2)) + + compat.bind_manager(mgr) + local cs = compat.new_session() + cs:set('dcuci', 'compat_named', 'example') + cs:set('dcuci', 'compat_named', 'flag', false) + local cok, cerr = cs:commit('dcuci') + assert(cok, 'common UCI compatibility commit failed: ' .. tostring(cerr)) + eq(compat.manager_owner(), 'bound', 'compat manager should be bound') + compat.clear_bound_manager() + + mgr:terminate('test complete') +end) + +eq(#restarts, 2, 'restart deduplication should leave network and wifi only') +eq(restarts[1], '/etc/init.d/network reload', 'first restart') +eq(restarts[2], '/sbin/wifi reload', 'second restart') + +local c = assert(uci.cursor(conf, save)) +if type(c.load) == 'function' then pcall(function() c:load('dcuci') end) end + +eq(c:get('dcuci', 'named', 'enabled'), nil, 'enabled should have been deleted') +eq(c:get('dcuci', 'named', 'renamed_count'), '7', 'renamed option') +assert_list(c:get('dcuci', 'named', 'listopt'), { 'two', 'three' }, 'listopt') +eq(c:get('dcuci', 'anon_named', 'name'), 'anonymous', 'renamed anonymous section') +eq(c:get('dcuci', 'compat_named', 'flag'), '0', 'compat boolean false') + +local seen_thing = false +assert(c:foreach('dcuci', 'thing', function(section) + if section['.name'] == 'anon_named' then + seen_thing = true + end +end)) +assert(seen_thing, 'foreach did not see renamed anonymous section') + +print('devicecode UCI manager on OpenWrt: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_devicecode_uci_manager.lua" "$REMOTE/run_devicecode_uci_manager.lua" +"$SSH" "cd '$REMOTE' && lua ./run_devicecode_uci_manager.lua" diff --git a/tests/integration/openwrt_vm/tests/test_devicecode_uci_manager_async_activation.sh b/tests/integration/openwrt_vm/tests/test_devicecode_uci_manager_async_activation.sh new file mode 100755 index 00000000..f0d05e3e --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_devicecode_uci_manager_async_activation.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-uci-manager-async-activation-test" +WORK="$VM_DIR/work/uci-manager-async-activation-test" + +mkdir -p "$WORK" + +cat > "$WORK/run_devicecode_uci_manager_async_activation.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', + './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', + './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', + './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', + './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' +local uci = require 'uci' +local uci_manager = require 'services.hal.backends.openwrt.uci_manager' + +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function eq(a, b, msg) + if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end +end +local function assert_true(v, msg) if not v then fail(msg or 'assertion failed') end end + +local function mkdir_p(path) + local ok = os.execute("mkdir -p '" .. path .. "'") + if ok ~= true and ok ~= 0 then fail('mkdir failed for ' .. path) end +end + +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.01)) + end + if pred() then return true end + fail(label or 'condition was not satisfied before timeout') +end + +local tmp = '/tmp/dc-devicecode-uci-manager-async-activation' +os.execute("rm -rf '" .. tmp .. "'") +local conf = tmp .. '/conf' +local save = tmp .. '/save' +mkdir_p(conf) +mkdir_p(save) +local f = assert(io.open(conf .. '/network', 'w')) +f:write('# devicecode async activation integration test\n') +f:close() + +local run_started = false +local run_finished = false +local restarts = {} +local unblock_tx, unblock_rx = mailbox.new(1, { full = 'reject_newest' }) +local result = nil + +fibers.run(function(scope) + local mgr, merr = uci_manager.new({ + confdir = conf, + savedir = save, + allow_fake = false, + debounce_s = 0.01, + run_cmd = function(argv) + restarts[#restarts + 1] = table.concat(argv, ' ') + run_started = true + perform(unblock_rx:recv_op()) + run_finished = true + return true, nil + end, + }) + assert(mgr, merr) + assert(mgr:start(scope)) + + scope:spawn(function() + result = perform(mgr:transaction_op({ + packages = { 'network' }, + records = { + { + config = 'network', + changes = { + { op = 'set', config = 'network', section = 'lan', option = 'interface' }, + { op = 'set', config = 'network', section = 'lan', option = 'proto', value = 'static' }, + }, + restart_cmds = { { kind = 'reload', target = 'network', wait = false } }, + }, + }, + rollback = true, + trace = { what = 'openwrt_vm_async_activation_contract', generation = 1 }, + })) + end) + + wait_until(function() return run_started == true end, 5, 'activation runner should start scheduled command') + eq(restarts[1], '/etc/init.d/network reload', 'activation command') + + wait_until(function() return result ~= nil end, 5, 'transaction should reply while async activation command is blocked') + assert_true(result.ok == true, 'transaction failed: ' .. tostring(result.err)) + assert_true(result.activation and result.activation.state == 'scheduled', 'activation should be scheduled') + eq(result.activation.commands, 1, 'scheduled command count') + eq(run_finished, false, 'transaction reply must not wait for activation command completion') + + local c = assert(uci.cursor(conf, save)) + if type(c.load) == 'function' then pcall(function() c:load('network') end) end + eq(c:get('network', 'lan'), 'interface', 'network.lan section committed before activation completion') + eq(c:get('network', 'lan', 'proto'), 'static', 'network.lan proto committed before activation completion') + + assert(unblock_tx:send(true)) + wait_until(function() return run_finished == true end, 1, 'activation command should finish after release') + eq(mgr:activation_status().state, 'done', 'activation status after command completion') + + mgr:terminate('test complete') +end) + +print('devicecode UCI manager async activation contract: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_devicecode_uci_manager_async_activation.lua" "$REMOTE/run_devicecode_uci_manager_async_activation.lua" +"$SSH" "cd '$REMOTE' && lua ./run_devicecode_uci_manager_async_activation.lua" diff --git a/tests/integration/openwrt_vm/tests/test_devicecode_wired_static_provider.sh b/tests/integration/openwrt_vm/tests/test_devicecode_wired_static_provider.sh new file mode 100755 index 00000000..04b51b19 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_devicecode_wired_static_provider.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-wired-static-provider-test" +WORK="$VM_DIR/work/wired-static-provider-test" + +mkdir -p "$WORK" +cat > "$WORK/run_devicecode_wired_static_provider.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local config = require 'services.wired.config' +local service = require 'services.wired.service' + +local function fail(msg) error(msg, 2) end +local function eq(a, b, msg) if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function ok(v, msg) if v ~= true then fail(msg or 'expected true') end end +local function normalised(raw) + local intent, err = config.normalise(raw) + if not intent then fail(err or 'config normalise failed') end + return intent +end +local function has_violation(snap, kind, fields) + for _, v in ipairs(snap.violations or {}) do + if v.kind == kind then + local match = true + for k, expected in pairs(fields or {}) do + if v[k] ~= expected then match = false; break end + end + if match then return v end + end + end + return nil +end + +local function has_required_vlan(trunk, segment, vlan) + for _, rec in ipairs((trunk and trunk.required_vlans) or {}) do + if rec.segment == segment and rec.vlan == vlan then return true end + end + return false +end + +local net_segments = { + mgmt = { kind = 'system', protected = true, vlan = { id = 10 } }, + switch_control = { kind = 'system', protected = true, vlan = { id = 11 } }, + fabric = { kind = 'system', protected = true, vlan = { id = 12 } }, + lan = { kind = 'user', vlan = { id = 100 } }, + guest = { kind = 'guest', vlan = { id = 101 } }, +} + +local wired_intent = normalised({ + schema = config.SCHEMA, + surfaces = { + ['switch-uplink-cm5'] = { + kind = 'switch-port', role = 'internal-trunk', protected = true, + attachment = { + mode = 'trunk', + required_segments = { 'mgmt', 'switch_control', 'fabric' }, + user_segments = 'all-realised-user-segments', + }, + }, + ['lan-1'] = { + kind = 'ethernet-port', role = 'access', capabilities = { poe = true }, + attachment = { mode = 'access', segment = 'lan' }, + }, + ['trunk-1'] = { + kind = 'ethernet-port', role = 'trunk', + attachment = { mode = 'trunk', segments = { 'lan', 'guest' } }, + }, + }, +}) + +local good = { + net = { segments = net_segments }, + config_intent = wired_intent, + assembly = { surfaces = { + ['switch-uplink-cm5'] = { component = 'switch-main', observed_surface = 'GE8' }, + ['lan-1'] = { component = 'switch-main', observed_surface = 'GE1' }, + ['trunk-1'] = { component = 'switch-main', observed_surface = 'GE2' }, + } }, + observations = { + ['switch-main'] = { + status = { state = 'available', available = true }, + surfaces = { + ['GE8'] = { + capabilities = { trunk = true, access = false, poe = false }, + attachment = { mode = 'trunk', vlans = { 10, 11, 12, 100, 101 } }, + link = { state = 'up', speed_mbps = 1000 }, + }, + ['GE1'] = { + capabilities = { access = true, trunk = true, poe = true }, + attachment = { mode = 'access', vlan = 100 }, + link = { state = 'up', speed_mbps = 1000 }, + }, + ['GE2'] = { + capabilities = { access = true, trunk = true, poe = false }, + attachment = { mode = 'trunk', vlans = { 100, 101 } }, + }, + }, + }, + }, + stats = {}, +} +service._test.rebuild_derived(good) +eq(good.state, 'running', 'good fixture should be running') +eq(#(good.violations or {}), 0, 'good fixture violations') +eq(good.surfaces['lan-1'].availability.state, 'available', 'lan-1 availability') +ok(has_required_vlan(good.topology.protected_trunks['switch-uplink-cm5'], 'guest', 101), 'guest expanded into protected trunk') +ok(has_required_vlan(good.topology.protected_trunks['switch-uplink-cm5'], 'lan', 100), 'lan expanded into protected trunk') + +local missing_provider = { + net = { segments = net_segments }, config_intent = wired_intent, assembly = good.assembly, observations = {}, stats = {}, +} +service._test.rebuild_derived(missing_provider) +ok(has_violation(missing_provider, 'protected_source_missing', { surface_id = 'switch-uplink-cm5', component = 'switch-main' }) ~= nil, 'protected source missing expected') + +local broken = { + net = { segments = net_segments }, + config_intent = wired_intent, + assembly = good.assembly, + observations = { + ['switch-main'] = { + status = { state = 'available', available = true }, + surfaces = { + ['GE8'] = { capabilities = { trunk = true }, attachment = { mode = 'trunk', vlans = { 10, 12, 100 } } }, + ['GE1'] = { capabilities = { access = false, trunk = true, poe = false }, attachment = { mode = 'access', vlan = 100 } }, + ['GE2'] = { capabilities = { access = true, trunk = false, poe = false }, attachment = { mode = 'access', vlan = 100 } }, + }, + }, + }, + stats = {}, +} +service._test.rebuild_derived(broken) +ok(has_violation(broken, 'missing_required_segment_carriage', { surface_id = 'switch-uplink-cm5', segment = 'switch_control', vlan = 11 }) ~= nil, 'missing switch_control expected') +ok(has_violation(broken, 'missing_user_segment_carriage', { surface_id = 'switch-uplink-cm5', segment = 'guest', vlan = 101 }) ~= nil, 'missing guest expected') +ok(has_violation(broken, 'observed_surface_does_not_support_access', { surface_id = 'lan-1', observed_surface = 'GE1' }) ~= nil, 'access capability violation expected') +ok(has_violation(broken, 'observed_surface_does_not_support_poe', { surface_id = 'lan-1', observed_surface = 'GE1' }) ~= nil, 'poe capability violation expected') +ok(has_violation(broken, 'observed_surface_does_not_support_trunk', { surface_id = 'trunk-1', observed_surface = 'GE2' }) ~= nil, 'trunk capability violation expected') +eq(broken.state, 'degraded', 'broken fixture state') + +print('devicecode wired static provider validation: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_devicecode_wired_static_provider.lua" "$REMOTE/run_devicecode_wired_static_provider.lua" +"$SSH" "cd '$REMOTE' && lua ./run_devicecode_wired_static_provider.lua" diff --git a/tests/integration/openwrt_vm/tests/test_lua_uci_baseline.sh b/tests/integration/openwrt_vm/tests/test_lua_uci_baseline.sh new file mode 100755 index 00000000..cb2e3f5a --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_lua_uci_baseline.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +SSH="$VM_DIR/scripts/ssh" + +"$SSH" 'command -v lua >/dev/null' +"$SSH" 'test -f /usr/lib/lua/uci.so -o -f /usr/lib/lua/5.1/uci.so -o -f /usr/lib/lua/5.4/uci.so' + +"$SSH" ' + set -eu + TMP="$(mktemp -d /tmp/dc-uci.XXXXXX)" + trap "rm -rf \"$TMP\"" EXIT + mkdir -p "$TMP/conf" "$TMP/save" + printf "# devicecode UCI VM test\n" > "$TMP/conf/dcuci" + CONF="$TMP/conf" SAVE="$TMP/save" lua - <<'"'"'LUA'"'"' +local uci = require "uci" +local conf = assert(os.getenv("CONF")) +local save = assert(os.getenv("SAVE")) +local c = assert(uci.cursor(conf, save)) +if type(c.load) == "function" then pcall(function() c:load("dcuci") end) end + +assert(c:set("dcuci", "named", "example")) +assert(c:set("dcuci", "named", "scalar", "value with spaces")) +assert(c:set("dcuci", "named", "enabled", "1")) +assert(c:set("dcuci", "named", "listopt", { "one", "two" })) + +local anon = assert(c:add("dcuci", "thing")) +assert(type(anon) == "string" and #anon > 0, "anonymous add returned no name") +assert(c:set("dcuci", anon, "name", "anonymous")) +if type(c.reorder) == "function" then assert(c:reorder("dcuci", anon, 0)) end + +local seen = false +assert(c:foreach("dcuci", "thing", function(s) + if s[".name"] == anon then seen = true end +end)) +assert(seen, "foreach did not see anonymous section") + +assert(c:delete("dcuci", "named", "scalar")) +assert(c:commit("dcuci")) + +local c2 = assert(uci.cursor(conf, save)) +local scalar = c2:get("dcuci", "named", "scalar") +assert(scalar == nil, "deleted scalar should be absent") +assert(c2:get("dcuci", "named", "enabled") == "1") +local list = c2:get("dcuci", "named", "listopt") +assert(type(list) == "table", "listopt should be a Lua table") +assert(list[1] == "one" and list[2] == "two", "listopt contents mismatch") +LUA +' + +echo "openwrt lua UCI baseline: ok" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_baseline.sh b/tests/integration/openwrt_vm/tests/test_openwrt_baseline.sh new file mode 100755 index 00000000..4c189453 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_baseline.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +SSH="$VM_DIR/scripts/ssh" + +"$SSH" 'command -v uci >/dev/null' +"$SSH" 'command -v ip >/dev/null' +"$SSH" 'command -v nft >/dev/null' +"$SSH" 'command -v tc >/dev/null' + +"$SSH" 'uci show network >/dev/null' +"$SSH" 'ip link show br-lan >/dev/null' +"$SSH" 'tc qdisc show >/dev/null' +"$SSH" 'nft list ruleset >/dev/null' + +"$SSH" ' + set -eu + ip link del dc_base_a 2>/dev/null || true + ip link del dc_base_ifb 2>/dev/null || true + trap "ip link del dc_base_a 2>/dev/null || true; ip link del dc_base_ifb 2>/dev/null || true" EXIT + + ip link add dc_base_a type veth peer name dc_base_b + ip link set dc_base_a up + ip link set dc_base_b up + ip link add dc_base_ifb type ifb + ip link set dc_base_ifb up + + tc qdisc add dev dc_base_a root handle 1: htb default 10 + tc class add dev dc_base_a parent 1: classid 1:10 htb rate 10mbit ceil 10mbit + tc qdisc add dev dc_base_a parent 1:10 fq_codel + tc qdisc show dev dc_base_a | grep -q htb + tc class show dev dc_base_a | grep -q "1:10" +' + +echo "openwrt baseline: ok" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_dnsmasq_multi_instance_resilience.sh b/tests/integration/openwrt_vm/tests/test_openwrt_dnsmasq_multi_instance_resilience.sh new file mode 100755 index 00000000..46db59d5 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_dnsmasq_multi_instance_resilience.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +WAIT_SSH="$VM_DIR/scripts/wait-ssh" +REMOTE="/tmp/devicecode-dnsmasq-multi-instance-resilience" +WORK="$VM_DIR/work/dnsmasq-multi-instance-resilience" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +fail() { + echo "openwrt dnsmasq multi-instance resilience test: $*" >&2 + dump_diag || true + exit 1 +} + +dump_diag() { + echo "--- OpenWrt dnsmasq multi-instance diagnostics ---" >&2 + "$SSH" 'set +e + echo "### link state"; ip -br link show; ip -br addr show + echo "### /etc/config/network"; sed -n "1,320p" /etc/config/network + echo "### /etc/config/dhcp"; sed -n "1,360p" /etc/config/dhcp + echo "### dnsmasq processes"; ps w | grep "[d]nsmasq" || true + echo "### UDP listeners"; (ss -lunp 2>/dev/null || netstat -lunp 2>/dev/null || true) | grep -E ":(53|67)\\b|dnsmasq" || true + echo "### dnsmasq generated"; for f in /var/etc/dnsmasq.conf* /tmp/etc/dnsmasq.conf*; do [ -f "$f" ] && { echo "### $f"; sed -n "1,220p" "$f"; }; done + echo "### dnsmasq log"; logread -e dnsmasq -e netifd | tail -n 260 + ' >&2 2>/dev/null || true + echo "--- end diagnostics ---" >&2 +} + +wait_for_router() { + label="$1" + timeout="$2" + shift 2 + cmd="$*" + deadline=$(( $(date +%s) + timeout )) + while [ "$(date +%s)" -le "$deadline" ]; do + if "$SSH" "$cmd" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + fail "timeout waiting for router $label" +} + +GUEST_TRUNK_IFACE="eth$((OPENWRT_VM_WAN_IFACES + 1))" +RESTART_ROUNDS="${OPENWRT_DNSMASQ_RESILIENCE_RESTARTS:-1}" +mkdir -p "$WORK" +cat > "$WORK/run_openwrt_dnsmasq_multi_instance_resilience.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local provider_loader = require 'services.hal.backends.network.provider' +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.05)) + end + if pred() then return true end + fail(label or 'condition was not satisfied before timeout') +end + +local trunk_ifname = os.getenv('DEVICECODE_TEST_TRUNK_IFACE') or 'eth4' + +local function vlan_segment(vid, cidr, zone, host_files, domain) + return { + kind = 'system', protected = true, + vlan = { id = vid }, + addressing = { ipv4 = { mode = 'static', cidr = cidr } }, + dhcp = { enabled = true, start = 10, limit = 240, leasetime = '12h' }, + dns = { local_server = true, domain = domain or 'bigbox.home', host_files = host_files or {} }, + firewall = { zone = zone or 'lan' }, + } +end + +local intent = { + schema = 'devicecode.net.intent/1', + rev = 1102024, + generation = 1102024, + segments = { + lan = { + kind = 'lan', + addressing = { ipv4 = { mode = 'static', cidr = '192.168.1.1/24' } }, + dhcp = { enabled = true, start = 100, limit = 150, leasetime = '12h' }, + dns = { local_server = true, domain = 'vm.bigbox.test' }, + firewall = { zone = 'lan' }, + }, + adm = vlan_segment(8, '172.28.8.1/24', 'lan', { 'ads' }, 'adm'), + jan = vlan_segment(32, '172.28.32.1/24', 'lan', { 'ads', 'adult' }, 'jan'), + int = vlan_segment(100, '172.28.100.1/24', 'lan', {}, 'int'), + wan = { kind = 'wan', firewall = { zone = 'wan' } }, + }, + interfaces = { + lan = { + kind = 'bridge', role = 'lan', segment = 'lan', members = { 'eth0' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.1.1/24' } }, + firewall = { zone = 'lan' }, + }, + wan = { + kind = 'ethernet', role = 'wan', segment = 'wan', endpoint = { ifname = 'eth1' }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, + dhcp = { enabled = false }, + firewall = { zone = 'wan' }, + }, + }, + dns = { + enabled = true, + domain = 'bigbox.home', + upstreams = { '1.1.1.1', '8.8.8.8' }, + cache = { size = 1000 }, + host_files = { + base_dir = '/data/devicecode/dns/hosts', + addnmount = true, + sources = { + ads = { file = 'ads.hosts' }, + adult = { file = 'adult.hosts' }, + }, + }, + records = { + config = { name = 'config.bigbox.home', address = '192.168.1.1' }, + }, + }, + dhcp = { defaults = { authoritative = true } }, + firewall = { + defaults = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT' }, + zones = { + lan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + wan = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT', masq = true, mtu_fix = true }, + }, + policies = { lan_to_wan = { src = 'lan', dest = 'wan' } }, + }, + routing = { routes = {} }, + wan = {}, + + vpn = {}, + diagnostics = {}, +} + +fibers.run(function() + local provider = assert(provider_loader.new({ + provider = 'openwrt', + debounce_s = 0.01, + platform = { segment_trunk = { ifname = trunk_ifname, protected = true } }, + run_cmd = function(argv) + print('dnsmasq-resilience config-only: skipped activation command ' .. table.concat(argv or {}, ' ')) + return true, nil + end, + shaper_run_cmd = function(_argv) return true, nil end, + }, {})) + + local valid = perform(provider:validate_op({ intent = intent })) + if not (valid and valid.ok == true) then fail('validate failed: ' .. tostring(valid and valid.err)) end + local result = perform(provider:apply_op({ intent = intent, opts = { generation = 1102024, apply_id = 'dnsmasq-multi-instance-resilience' } })) + if not (result and result.ok == true) then fail('apply failed: ' .. tostring(result and result.err)) end + local mgr = provider._uci_manager + wait_until(function() + local st = mgr and mgr.activation_status and mgr:activation_status() or nil + return st and (st.state == 'done' or st.state == 'idle') + end, 5, 'config-only activation runner should be idle') + provider:terminate('dnsmasq multi-instance resilience test configured') +end) + +print('openwrt dnsmasq multi-instance config generated: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'; ip link show dev '$GUEST_TRUNK_IFACE' >/dev/null" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_dnsmasq_multi_instance_resilience.lua" "$REMOTE/run_openwrt_dnsmasq_multi_instance_resilience.lua" +"$SSH" "cd '$REMOTE' && DEVICECODE_TEST_TRUNK_IFACE='$GUEST_TRUNK_IFACE' lua ./run_openwrt_dnsmasq_multi_instance_resilience.lua" +"$SSH" 'mkdir -p /data/devicecode/dns/hosts; : > /data/devicecode/dns/hosts/ads.hosts; : > /data/devicecode/dns/hosts/adult.hosts' + +"$SSH" '/etc/init.d/network reload' || true +"$WAIT_SSH" +wait_for_router "br-lan" 45 "ip -4 addr show dev br-lan | grep -q '192.168.1.1'" +wait_for_router "br-adm" 45 "ip -4 addr show dev br-adm | grep -q '172.28.8.1'" +wait_for_router "br-jan" 45 "ip -4 addr show dev br-jan | grep -q '172.28.32.1'" +wait_for_router "br-int" 45 "ip -4 addr show dev br-int | grep -q '172.28.100.1'" +"$SSH" '/etc/init.d/firewall restart' + +verify_instances_remote='set -eu +fail() { echo "$*" >&2; exit 1; } +insts="$(uci -q show dhcp | sed -n "s/^dhcp\.\([^=]*\)=dnsmasq/\1/p" | sort)" +count="$(printf "%s\n" "$insts" | sed "/^$/d" | wc -l)" +[ "$count" -ge 4 ] || fail "expected at least four dnsmasq instances, got $count: $insts" +listeners="$(ss -lunp 2>/dev/null || netstat -lunp 2>/dev/null || true)" +loopback_owner="" +for inst in $insts; do + cfg="/var/etc/dnsmasq.conf.$inst" + [ -f "$cfg" ] || fail "missing generated config $cfg" + ps w | grep -F "dnsmasq -C $cfg" | grep -qv grep || fail "dnsmasq process for $inst is not running" + if uci -q get dhcp.$inst.interface | grep -qw int; then + [ -z "$loopback_owner" ] || fail "multiple loopback owner candidates: $loopback_owner and $inst" + loopback_owner="$inst" + if uci -q get dhcp.$inst.notinterface | grep -qw lo; then fail "int dnsmasq $inst unexpectedly excludes loopback"; fi + if grep -q "^except-interface=lo$" "$cfg"; then fail "int dnsmasq $cfg unexpectedly contains except-interface=lo"; fi + else + uci -q get dhcp.$inst.notinterface | grep -qw lo || fail "dnsmasq $inst does not exclude loopback" + grep -q "^except-interface=lo$" "$cfg" || fail "$cfg does not contain except-interface=lo" + fi +done +[ -n "$loopback_owner" ] || fail "no int dnsmasq loopback owner found" +printf "%s\n" "$listeners" | grep -q "127.0.0.1:53" || fail "no DNS listener on 127.0.0.1:53 for loopback owner $loopback_owner: $listeners" +for spec in \ + "lan 192.168.1.1 br-lan" \ + "adm 172.28.8.1 br-adm" \ + "jan 172.28.32.1 br-jan" \ + "int 172.28.100.1 br-int" +do + set -- $spec + name="$1"; ip="$2"; dev="$3" + ip -4 addr show dev "$dev" | grep -q "$ip" || fail "$dev does not have $ip" + printf "%s\n" "$listeners" | grep -q "$ip:53" || fail "no DNS listener on $ip:53 for $name" + nslookup config.bigbox.home "$ip" | grep -q "192.168.1.1" || fail "$name DNS local record failed via $ip" +done +' + +restart_round=0 +while [ "$restart_round" -lt "$RESTART_ROUNDS" ]; do + restart_round=$((restart_round + 1)) + "$SSH" '/etc/init.d/dnsmasq restart' + wait_for_router "all dnsmasq instances after restart $restart_round" 45 "$verify_instances_remote" + # Give crashes caused by delayed bind conflicts a chance to surface. + sleep "${OPENWRT_DNSMASQ_RESILIENCE_SETTLE_S:-1}" + wait_for_router "all dnsmasq instances still alive after restart $restart_round" 20 "$verify_instances_remote" +done + +# Include a network reload cycle because netifd can briefly remove/recreate the +# bridge/vlan addresses that dnsmasq binds to. The instance set must recover +# deterministically afterwards. +"$SSH" '/etc/init.d/network reload' || true +"$WAIT_SSH" +wait_for_router "br-adm after network reload" 45 "ip -4 addr show dev br-adm | grep -q '172.28.8.1'" +wait_for_router "br-jan after network reload" 45 "ip -4 addr show dev br-jan | grep -q '172.28.32.1'" +wait_for_router "br-int after network reload" 45 "ip -4 addr show dev br-int | grep -q '172.28.100.1'" +"$SSH" '/etc/init.d/dnsmasq restart' +wait_for_router "all dnsmasq instances after network reload" 45 "$verify_instances_remote" +sleep "${OPENWRT_DNSMASQ_RESILIENCE_SETTLE_S:-1}" +wait_for_router "all dnsmasq instances stayed up after network reload" 20 "$verify_instances_remote" + +printf '%s\n' "openwrt dnsmasq multi-instance resilience: ok" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_int_bridge_client_dhcp_dns.sh b/tests/integration/openwrt_vm/tests/test_openwrt_int_bridge_client_dhcp_dns.sh new file mode 100755 index 00000000..0330c368 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_int_bridge_client_dhcp_dns.sh @@ -0,0 +1,366 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +WAIT_SSH="$VM_DIR/scripts/wait-ssh" +REMOTE="/tmp/devicecode-int-bridge-client-dhcp-dns" +WORK="$VM_DIR/work/int-bridge-client-dhcp-dns" + +# shellcheck disable=SC1091 +. "$VM_DIR/env.sh" + +as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "int bridge client DHCP/DNS test: need root privileges for: $*" >&2 + exit 1 + fi +} + +fail() { + echo "int bridge client DHCP/DNS test: $*" >&2 + dump_diag || true + exit 1 +} + +ns_exec() { + as_root ip netns exec "$OPENWRT_CLIENT_NS" "$@" +} + +find_udhcpc() { + if command -v udhcpc >/dev/null 2>&1; then + printf '%s\n' "$(command -v udhcpc)" + return 0 + fi + if command -v busybox >/dev/null 2>&1 && busybox udhcpc --help >/dev/null 2>&1; then + printf '%s\n' "$(command -v busybox) udhcpc" + return 0 + fi + return 1 +} + +host_has_ns() { + ip netns list 2>/dev/null | awk '{print $1}' | grep -qx "$OPENWRT_CLIENT_NS" +} + +dump_diag() { + echo "--- host bridge/client diagnostics ---" >&2 + ip -br link show dev "$OPENWRT_CLIENT_TAP" >&2 2>/dev/null || true + ip -br link show dev "$OPENWRT_CLIENT_HOST_IFACE" >&2 2>/dev/null || true + if host_has_ns; then + as_root ip netns exec "$OPENWRT_CLIENT_NS" ip -br addr >&2 2>/dev/null || true + as_root ip netns exec "$OPENWRT_CLIENT_NS" ip route >&2 2>/dev/null || true + fi + if ip link show dev "$OPENWRT_CLIENT_BRIDGE" >/dev/null 2>&1; then + ip -d link show dev "$OPENWRT_CLIENT_BRIDGE" >&2 2>/dev/null || true + bridge vlan show dev "$OPENWRT_CLIENT_TAP" >&2 2>/dev/null || true + bridge vlan show dev "$OPENWRT_CLIENT_HOST_IFACE" >&2 2>/dev/null || true + bridge link show master "$OPENWRT_CLIENT_BRIDGE" >&2 2>/dev/null || true + fi + echo "--- OpenWrt int diagnostics ---" >&2 + "$SSH" 'set +e + ip -br link show + ip -br addr show + bridge link show 2>/dev/null + echo "### link counters"; ip -s link show dev eth4 2>/dev/null; ip -s link show dev vl-int 2>/dev/null; ip -s link show dev br-int 2>/dev/null + echo "### bridge fdb br-int"; bridge fdb show br br-int 2>/dev/null || true + echo "### route"; ip route + echo "### ip_forward"; cat /proc/sys/net/ipv4/ip_forward 2>/dev/null || true + echo "### /etc/config/network"; sed -n "1,260p" /etc/config/network + echo "### /etc/config/dhcp"; sed -n "1,260p" /etc/config/dhcp + echo "### /etc/config/firewall"; sed -n "1,320p" /etc/config/firewall + echo "### fw4 check"; fw4 check 2>&1 || true + echo "### nft fw4 forward/srcnat"; nft list chain inet fw4 forward 2>&1 || true; nft list chain inet fw4 srcnat 2>&1 || true + echo "### dnsmasq processes"; ps w | grep "[d]nsmasq" || true + echo "### UDP listeners"; (ss -lunp 2>/dev/null || netstat -lunp 2>/dev/null || true) | grep -E ":(53|67)\b|dnsmasq" || true + echo "### dnsmasq generated"; for f in /var/etc/dnsmasq.conf* /tmp/etc/dnsmasq.conf*; do [ -f "$f" ] && { echo "### $f"; sed -n "1,220p" "$f"; }; done + echo "### captured DHCP on br-int"; cat /tmp/dcint-dhcp-br-int.txt 2>/dev/null || true + echo "### dnsmasq/netifd/firewall log"; logread -e dnsmasq -e netifd -e firewall -e fw4 | tail -n 220 + ' >&2 2>/dev/null || true + echo "--- end diagnostics ---" >&2 +} + +wait_for_host() { + label="$1" + timeout="$2" + shift 2 + deadline=$(( $(date +%s) + timeout )) + while [ "$(date +%s)" -le "$deadline" ]; do + if "$@" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + fail "timeout waiting for host $label" +} + +wait_for_router() { + label="$1" + timeout="$2" + shift 2 + cmd="$*" + deadline=$(( $(date +%s) + timeout )) + while [ "$(date +%s)" -le "$deadline" ]; do + if "$SSH" "$cmd" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + fail "timeout waiting for router $label" +} + +host_has_ns || fail "missing host network namespace $OPENWRT_CLIENT_NS; run scripts/setup-bridge-client-fabric first" +ip link show dev "$OPENWRT_CLIENT_TAP" >/dev/null 2>&1 || fail "missing tap $OPENWRT_CLIENT_TAP; run VM with OPENWRT_TAP_IFACES=$OPENWRT_CLIENT_TAP" +ip link show dev "$OPENWRT_CLIENT_BRIDGE" >/dev/null 2>&1 || fail "missing host client bridge $OPENWRT_CLIENT_BRIDGE" +bridge vlan show dev "$OPENWRT_CLIENT_TAP" | grep -q " $OPENWRT_CLIENT_VLAN" || fail "tap $OPENWRT_CLIENT_TAP is not trunking VLAN $OPENWRT_CLIENT_VLAN" +bridge vlan show dev "$OPENWRT_CLIENT_HOST_IFACE" | grep -q " $OPENWRT_CLIENT_VLAN" || fail "client host veth $OPENWRT_CLIENT_HOST_IFACE is not an access port for VLAN $OPENWRT_CLIENT_VLAN" + +GUEST_TRUNK_IFACE="eth$((OPENWRT_VM_WAN_IFACES + 1))" +mkdir -p "$WORK" +cat > "$WORK/run_openwrt_int_bridge_client_config.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local provider_loader = require 'services.hal.backends.network.provider' +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.05)) + end + if pred() then return true end + fail(label or 'condition was not satisfied before timeout') +end + +local trunk_ifname = os.getenv('DEVICECODE_TEST_TRUNK_IFACE') or 'eth4' +local vlan = tonumber(os.getenv('DEVICECODE_TEST_INT_VLAN') or '100') or 100 + +local intent = { + schema = 'devicecode.net.intent/1', + rev = 1002024, + generation = 1002024, + segments = { + lan = { + kind = 'lan', + addressing = { ipv4 = { mode = 'static', cidr = '192.168.1.1/24' } }, + dhcp = { enabled = true, start = 100, limit = 150, leasetime = '12h' }, + dns = { local_server = true, domain = 'vm.bigbox.test' }, + firewall = { zone = 'lan' }, + }, + int = { + kind = 'system', protected = true, + vlan = { id = vlan }, + addressing = { ipv4 = { mode = 'static', cidr = '172.28.100.1/24' } }, + dhcp = { enabled = true, start = 10, limit = 240, leasetime = '12h' }, + dns = { local_server = true, domain = 'bigbox.home' }, + firewall = { zone = 'lan' }, + }, + wan = { kind = 'wan', firewall = { zone = 'wan' } }, + }, + interfaces = { + lan = { + kind = 'bridge', role = 'lan', segment = 'lan', members = { 'eth0' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.1.1/24' } }, + firewall = { zone = 'lan' }, + }, + wan = { + kind = 'ethernet', role = 'wan', segment = 'wan', endpoint = { ifname = 'eth1' }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, + dhcp = { enabled = false }, + firewall = { zone = 'wan' }, + }, + }, + dns = { + enabled = true, + domain = 'bigbox.home', + upstreams = { '1.1.1.1', '8.8.8.8' }, + cache = { size = 1000 }, + records = { + config = { name = 'config.bigbox.home', address = '192.168.1.1' }, + }, + }, + dhcp = { defaults = { authoritative = true } }, + firewall = { + defaults = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT' }, + zones = { + lan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + wan = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT', masq = true, mtu_fix = true }, + }, + policies = { + lan_to_wan = { src = 'lan', dest = 'wan' }, + }, + }, + routing = { routes = {} }, + wan = {}, + + vpn = {}, + diagnostics = {}, +} + +fibers.run(function() + local provider = assert(provider_loader.new({ + provider = 'openwrt', + debounce_s = 0.01, + platform = { segment_trunk = { ifname = trunk_ifname, protected = true } }, + run_cmd = function(argv) + print('int-bridge-client config-only: skipped activation command ' .. table.concat(argv or {}, ' ')) + return true, nil + end, + shaper_run_cmd = function(_argv) return true, nil end, + }, {})) + + local valid = perform(provider:validate_op({ intent = intent })) + if not (valid and valid.ok == true) then fail('validate failed: ' .. tostring(valid and valid.err)) end + local result = perform(provider:apply_op({ intent = intent, opts = { generation = 1002024, apply_id = 'int-bridge-client-dhcp-dns' } })) + if not (result and result.ok == true) then fail('apply failed: ' .. tostring(result and result.err)) end + local mgr = provider._uci_manager + wait_until(function() + local st = mgr and mgr.activation_status and mgr:activation_status() or nil + return st and (st.state == 'done' or st.state == 'idle') + end, 5, 'config-only activation runner should be idle') + provider:terminate('int bridge client DHCP/DNS test configured') +end) + +print('openwrt int bridge client config generated: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'; ip link show dev '$GUEST_TRUNK_IFACE' >/dev/null" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_int_bridge_client_config.lua" "$REMOTE/run_openwrt_int_bridge_client_config.lua" +"$SSH" "cd '$REMOTE' && DEVICECODE_TEST_TRUNK_IFACE='$GUEST_TRUNK_IFACE' DEVICECODE_TEST_INT_VLAN='$OPENWRT_CLIENT_VLAN' lua ./run_openwrt_int_bridge_client_config.lua" + +"$SSH" '/etc/init.d/network reload' || true +"$WAIT_SSH" +"$SSH" '/etc/init.d/firewall restart' +wait_for_router "br-int" 45 "ip link show dev br-int" +wait_for_router "int address" 45 "ip -4 addr show dev br-int | grep -q '172.28.100.1'" +wait_for_router "vl-int" 45 "ip link show dev vl-int" +wait_for_router "wan default route" 45 "ip route | grep -q '^default .* dev eth1'" +wait_for_router "router public IP egress" 45 "ping -c 1 -W 3 8.8.8.8" +wait_for_router "router public DNS" 45 "nslookup '$OPENWRT_CLIENT_PUBLIC_DNS_NAME' 1.1.1.1 | grep -q '^Name:'" +"$SSH" '/etc/init.d/dnsmasq restart' +wait_for_router "dnsmasq" 45 "pgrep dnsmasq" +sleep 1 +wait_for_router "int DNS local record" 20 "nslookup config.bigbox.home 172.28.100.1 | grep -q '192.168.1.1'" +wait_for_router "int DNS public forwarding" 45 "nslookup '$OPENWRT_CLIENT_PUBLIC_DNS_NAME' 172.28.100.1 | grep -q '^Name:'" + +# Prove the VLAN trunk/access dataplane independently of dnsmasq before DHCP. +# If this fails, the problem is the VM/client fabric or VLAN delivery into eth4, +# not the OpenWrt DHCP/DNS generator. +as_root ip netns exec "$OPENWRT_CLIENT_NS" ip addr flush dev "$OPENWRT_CLIENT_IFACE" || true +as_root ip netns exec "$OPENWRT_CLIENT_NS" ip route flush dev "$OPENWRT_CLIENT_IFACE" || true +as_root ip netns exec "$OPENWRT_CLIENT_NS" ip addr add "${OPENWRT_CLIENT_CIDR_PREFIX}250/24" dev "$OPENWRT_CLIENT_IFACE" +check_static_tmp="$WORK/static-ping.out" +if ! as_root ip netns exec "$OPENWRT_CLIENT_NS" ping -c 2 -W 2 "$OPENWRT_CLIENT_ROUTER" >"$check_static_tmp" 2>&1; then + cat "$check_static_tmp" >&2 || true + fail "static int client could not reach $OPENWRT_CLIENT_ROUTER before DHCP" +fi +rm -f "$check_static_tmp" +as_root ip netns exec "$OPENWRT_CLIENT_NS" ip addr flush dev "$OPENWRT_CLIENT_IFACE" || true +as_root ip netns exec "$OPENWRT_CLIENT_NS" ip route flush dev "$OPENWRT_CLIENT_IFACE" || true + +UDHCPC_CMD="$(find_udhcpc || true)" +[ -n "$UDHCPC_CMD" ] || fail "host udhcpc is unavailable; install udhcpc or busybox with udhcpc applet" +LEASE_ENV="$WORK/bridge-client-lease.env" +LEASE_SCRIPT="$WORK/bridge-client-udhcpc.sh" +rm -f "$LEASE_ENV" +cat > "$LEASE_SCRIPT" < '$LEASE_ENV' + ip addr flush dev "\$interface" 2>/dev/null || true + ip addr add "\$ip/24" dev "\$interface" + ip link set "\$interface" up + if [ -n "\$router" ]; then + set -- \$router + ip route replace default via "\$1" dev "\$interface" 2>/dev/null || true + fi + ;; +esac +exit 0 +EOF +chmod +x "$LEASE_SCRIPT" + +as_root ip netns exec "$OPENWRT_CLIENT_NS" ip addr flush dev "$OPENWRT_CLIENT_IFACE" || true +as_root ip netns exec "$OPENWRT_CLIENT_NS" ip route flush dev "$OPENWRT_CLIENT_IFACE" || true +"$SSH" 'rm -f /tmp/dcint-dhcp-br-int.txt; if command -v tcpdump >/dev/null 2>&1; then (tcpdump -ni br-int -e -vv "port 67 or port 68" > /tmp/dcint-dhcp-br-int.txt 2>&1 & echo $! > /tmp/dcint-dhcp-br-int.pid); fi' || true +# shellcheck disable=SC2086 +as_root ip netns exec "$OPENWRT_CLIENT_NS" sh -c "$UDHCPC_CMD -i '$OPENWRT_CLIENT_IFACE' -q -n -t 10 -T 1 -s '$LEASE_SCRIPT'" || { "$SSH" 'if [ -f /tmp/dcint-dhcp-br-int.pid ]; then kill $(cat /tmp/dcint-dhcp-br-int.pid) 2>/dev/null || true; fi' || true; fail "host-side bridge client did not obtain an int lease"; } +"$SSH" 'if [ -f /tmp/dcint-dhcp-br-int.pid ]; then kill $(cat /tmp/dcint-dhcp-br-int.pid) 2>/dev/null || true; fi' || true +# Give libc-based tools inside the namespace the same resolver that DHCP handed +# to the client. nslookup below passes the server explicitly, but wget/curl use +# the namespace resolver file. +as_root mkdir -p "/etc/netns/$OPENWRT_CLIENT_NS" +printf 'nameserver %s\n' "$OPENWRT_CLIENT_ROUTER" | as_root tee "/etc/netns/$OPENWRT_CLIENT_NS/resolv.conf" >/dev/null +[ -f "$LEASE_ENV" ] || fail "host-side client lease environment was not captured" +# shellcheck disable=SC1090 +. "$LEASE_ENV" +case "$ip" in "$OPENWRT_CLIENT_CIDR_PREFIX"*) ;; *) fail "lease IP $ip is not in ${OPENWRT_CLIENT_CIDR_PREFIX}0/24" ;; esac +case " ${router:-} " in *" $OPENWRT_CLIENT_ROUTER "*) ;; *) fail "router option does not contain $OPENWRT_CLIENT_ROUTER: ${router:-}" ;; esac +case " ${dns:-} " in *" $OPENWRT_CLIENT_ROUTER "*) ;; *) fail "DNS option does not contain $OPENWRT_CLIENT_ROUTER: ${dns:-}" ;; esac + +check_ns() { + label="$1" + timeout="$2" + shift 2 + deadline=$(( $(date +%s) + timeout )) + while [ "$(date +%s)" -le "$deadline" ]; do + if ns_exec "$@" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + fail "$label failed" +} + +check_public_dns() { + label="$1" + tmp="$WORK/public-dns.out" + deadline=$(( $(date +%s) + 45 )) + while [ "$(date +%s)" -le "$deadline" ]; do + if ns_exec nslookup "$OPENWRT_CLIENT_PUBLIC_DNS_NAME" "$OPENWRT_CLIENT_ROUTER" >"$tmp" 2>&1 && grep -q '^Name:' "$tmp"; then + rm -f "$tmp" + return 0 + fi + sleep 1 + done + cat "$tmp" >&2 2>/dev/null || true + rm -f "$tmp" + fail "$label could not resolve $OPENWRT_CLIENT_PUBLIC_DNS_NAME via $OPENWRT_CLIENT_ROUTER" +} + +check_ns "client can ping int router" 20 ping -c 2 -W 2 "$OPENWRT_CLIENT_ROUTER" +if ! ns_exec ping -c 2 -W 2 8.8.8.8 >/dev/null 2>&1; then + echo "int bridge client DHCP/DNS test: warning: client ICMP to 8.8.8.8 failed; continuing with DNS/TCP checks" >&2 +fi +ns_exec nslookup config.bigbox.home "$OPENWRT_CLIENT_ROUTER" | grep -q '192.168.1.1' || fail "client could not resolve config.bigbox.home via int DNS" +check_public_dns "client public DNS" + +if command -v wget >/dev/null 2>&1; then + check_ns "client public HTTP fetch" 45 wget -q -T 10 -O /dev/null "$OPENWRT_CLIENT_PUBLIC_WEB_URL" +elif command -v curl >/dev/null 2>&1; then + check_ns "client public HTTP fetch" 45 curl -fsSL --max-time 15 -o /dev/null "$OPENWRT_CLIENT_PUBLIC_WEB_URL" +else + echo "int bridge client DHCP/DNS test: wget/curl unavailable; skipping public HTTP fetch" >&2 +fi + +printf '%s\n' "openwrt int bridge client DHCP/DNS/public-web: ok" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_jan_client_dhcp_dns.sh b/tests/integration/openwrt_vm/tests/test_openwrt_jan_client_dhcp_dns.sh new file mode 100755 index 00000000..f99fd4b7 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_jan_client_dhcp_dns.sh @@ -0,0 +1,299 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +WAIT_SSH="$VM_DIR/scripts/wait-ssh" +REMOTE="/tmp/devicecode-jan-client-dhcp-dns" +WORK="$VM_DIR/work/jan-client-dhcp-dns" + +mkdir -p "$WORK" +cat > "$WORK/run_openwrt_jan_client_config.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local provider_loader = require 'services.hal.backends.network.provider' +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.05)) + end + if pred() then return true end + fail(label or 'condition was not satisfied before timeout') +end + +local intent = { + schema = 'devicecode.net.intent/1', + rev = 32024, + generation = 32024, + segments = { + lan = { + kind = 'lan', + addressing = { ipv4 = { mode = 'static', cidr = '192.168.1.1/24' } }, + dhcp = { enabled = true, start = 100, limit = 150, leasetime = '12h' }, + dns = { local_server = true, domain = 'vm.bigbox.test' }, + firewall = { zone = 'lan' }, + }, + jan = { + kind = 'user', + vlan = { id = 32 }, + addressing = { ipv4 = { mode = 'static', cidr = '172.28.32.1/24' } }, + dhcp = { enabled = true, start = 10, limit = 240, leasetime = '12h' }, + dns = { local_server = true, domain = 'bigbox.home' }, + firewall = { zone = 'jan' }, + }, + wan = { kind = 'wan', firewall = { zone = 'wan' } }, + }, + interfaces = { + lan = { + kind = 'bridge', role = 'lan', segment = 'lan', members = { 'eth0' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.1.1/24' } }, + firewall = { zone = 'lan' }, + }, + wan = { + kind = 'ethernet', role = 'wan', segment = 'wan', endpoint = { ifname = 'eth1' }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, + dhcp = { enabled = false }, + firewall = { zone = 'wan' }, + }, + }, + dns = { + enabled = true, + domain = 'bigbox.home', + upstreams = { '1.1.1.1', '8.8.8.8' }, + cache = { size = 1000 }, + records = { + config = { name = 'config.bigbox.home', address = '192.168.1.1' }, + }, + }, + dhcp = { defaults = { authoritative = true } }, + firewall = { + defaults = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT' }, + zones = { + lan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + jan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + wan = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT', masq = true, mtu_fix = true }, + }, + policies = { + lan_to_wan = { src = 'lan', dest = 'wan' }, + jan_to_wan = { src = 'jan', dest = 'wan' }, + }, + }, + routing = { routes = {} }, + wan = {}, + + vpn = {}, + diagnostics = {}, +} + +fibers.run(function() + local provider = assert(provider_loader.new({ + provider = 'openwrt', + debounce_s = 0.01, + platform = { segment_trunk = { ifname = 'dcjantrunk0', protected = true } }, + -- This lane performs activation explicitly from the shell harness so SSH can + -- reconnect cleanly after network reload. Synchronous activation itself is + -- covered by the provider activation-contract VM test. + run_cmd = function(argv) + print('jan-client config-only: skipped activation command ' .. table.concat(argv or {}, ' ')) + return true, nil + end, + shaper_run_cmd = function(_argv) return true, nil end, + }, {})) + + local valid = perform(provider:validate_op({ intent = intent })) + if not (valid and valid.ok == true) then fail('validate failed: ' .. tostring(valid and valid.err)) end + local result = perform(provider:apply_op({ intent = intent, opts = { generation = 32024, apply_id = 'jan-client-dhcp-dns' } })) + if not (result and result.ok == true) then fail('apply failed: ' .. tostring(result and result.err)) end + local mgr = provider._uci_manager + wait_until(function() + local st = mgr and mgr.activation_status and mgr:activation_status() or nil + return st and (st.state == 'done' or st.state == 'idle') + end, 5, 'config-only activation runner should be idle') + provider:terminate('jan client DHCP/DNS test configured') +end) + +print('openwrt jan client config generated: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'; ip link del dcjantrunk0 2>/dev/null || true; ip link del dcjantrunk0p 2>/dev/null || true; ip link add dcjantrunk0 type veth peer name dcjantrunk0p; ip link set dcjantrunk0 up; ip link set dcjantrunk0p up" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_jan_client_config.lua" "$REMOTE/run_openwrt_jan_client_config.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_jan_client_config.lua" + +# Activate network in a shell-controlled phase so this test does not depend on +# keeping a long-running SSH command alive across network reload. dnsmasq is +# restarted inside the client exercise below, after the synthetic Jan client +# link exists; on some OpenWrt VM/kernel combinations dnsmasq will otherwise +# start before the bridge has carrier and will not offer on the synthetic link. +"$SSH" '/etc/init.d/network reload' || true +"$WAIT_SSH" +"$SSH" '/etc/init.d/firewall restart' + +"$SSH" 'set -eu + +fail() { echo "jan client DHCP/DNS test: $*" >&2; exit 1; } +cleanup() { + ip netns del dc-jan-client 2>/dev/null || true + ip link del dc-jan-host 2>/dev/null || true + rm -f /tmp/dc-jan-lease.env /tmp/dc-jan-udhcpc.sh /tmp/dc-jan-use-default-route +} +trap cleanup EXIT INT TERM + +wait_for() { + label="$1" + timeout="$2" + shift 2 + cmd="$*" + deadline=$(( $(date +%s) + timeout )) + while [ "$(date +%s)" -le "$deadline" ]; do + if sh -c "$cmd" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + fail "timeout waiting for $label" +} + +wait_for br-jan 45 "ip link show dev br-jan" +wait_for jan-address 45 "ip -4 addr show dev br-jan | grep -q 172.28.32.1" +wait_for vl-jan 45 "ip link show dev vl-jan" +wait_for wan-address 45 "ip -4 addr show dev eth1 | grep -q '\''inet '\''" +wait_for default-route 45 "ip route | grep -q '\''^default .* dev eth1'\''" + +dump_diag() { + echo "--- jan DHCP/DNS diagnostics ---" >&2 + ip -br link show >&2 2>/dev/null || true + ip -br addr show br-jan vl-jan dc-jan-host dc-jan-client0 >&2 2>/dev/null || true + bridge link show >&2 2>/dev/null || true + echo "--- /etc/config/dhcp ---" >&2 + sed -n "1,220p" /etc/config/dhcp >&2 2>/dev/null || true + echo "--- generated dnsmasq configs ---" >&2 + for f in /var/etc/dnsmasq.conf* /tmp/etc/dnsmasq.conf*; do + [ -f "$f" ] || continue + echo "### $f" >&2 + sed -n "1,220p" "$f" >&2 2>/dev/null || true + done + echo "--- dnsmasq processes ---" >&2 + ps w | grep "[d]nsmasq" >&2 2>/dev/null || true + echo "--- recent dnsmasq/netifd log ---" >&2 + logread -e dnsmasq -e netifd | tail -n 120 >&2 2>/dev/null || true + echo "--- end diagnostics ---" >&2 +} + + +HAVE_NETNS=0 +if ip netns add dc-jan-client >/dev/null 2>&1; then + HAVE_NETNS=1 + ip netns del dc-jan-client >/dev/null 2>&1 || true +fi + +client_exec() { + if [ "$HAVE_NETNS" = 1 ]; then + ip netns exec dc-jan-client "$@" + else + "$@" + fi +} + +ip link del dc-jan-host 2>/dev/null || true +ip link add dc-jan-host type veth peer name dc-jan-client0 +ip link set dc-jan-host master br-jan +ip link set dc-jan-host up + +if [ "$HAVE_NETNS" = 1 ]; then + ip netns add dc-jan-client + touch /tmp/dc-jan-use-default-route + ip link set dc-jan-client0 netns dc-jan-client + client_exec ip link set lo up + client_exec ip link set dc-jan-client0 up +else + echo "jan client DHCP/DNS test: ip netns unavailable; using same-namespace veth DHCP fallback" >&2 + ip link set dc-jan-client0 up +fi + +# Restart dnsmasq only after the synthetic client-facing bridge port is present. +# This keeps the test robust on VM kernels where an empty bridge does not have +# carrier when dnsmasq first enumerates DHCP-capable interfaces. +/etc/init.d/dnsmasq restart +wait_for dnsmasq 45 "pgrep dnsmasq" +sleep 1 + +cat > /tmp/dc-jan-udhcpc.sh <<"EOF" +#!/bin/sh +case "$1" in + bound|renew) + { + printf "ip='\''%s'\''\n" "$ip" + printf "router='\''%s'\''\n" "$router" + printf "dns='\''%s'\''\n" "$dns" + printf "domain='\''%s'\''\n" "$domain" + printf "subnet='\''%s'\''\n" "$subnet" + } > /tmp/dc-jan-lease.env + ip addr flush dev "$interface" 2>/dev/null || true + ip addr add "$ip/24" dev "$interface" + ip link set "$interface" up + if [ -f /tmp/dc-jan-use-default-route ] && [ -n "$router" ]; then + set -- $router + ip route replace default via "$1" dev "$interface" 2>/dev/null || true + fi + ;; +esac +exit 0 +EOF +chmod +x /tmp/dc-jan-udhcpc.sh + +client_exec udhcpc -i dc-jan-client0 -q -n -t 10 -T 1 -s /tmp/dc-jan-udhcpc.sh || { dump_diag; fail "udhcpc did not obtain a jan lease"; } +[ -f /tmp/dc-jan-lease.env ] || fail "lease environment was not captured" +. /tmp/dc-jan-lease.env +case "$ip" in 172.28.32.*) ;; *) fail "lease IP $ip is not in 172.28.32.0/24" ;; esac +case " ${router:-} " in *" 172.28.32.1 "*) ;; *) fail "router option does not contain 172.28.32.1: ${router:-}" ;; esac +case " ${dns:-} " in *" 172.28.32.1 "*) ;; *) fail "DNS option does not contain 172.28.32.1: ${dns:-}" ;; esac + +PUBLIC_DNS_NAME="${DEVICECODE_TEST_PUBLIC_DNS_NAME:-openwrt.org}" +PUBLIC_DNS_UPSTREAM="${DEVICECODE_TEST_PUBLIC_DNS_UPSTREAM:-1.1.1.1}" + +check_public_dns() { + label="$1"; shift + tmp="/tmp/dc-jan-public-dns.$$" + deadline=$(( $(date +%s) + 45 )) + while [ "$(date +%s)" -le "$deadline" ]; do + if "$@" >"$tmp" 2>&1 && grep -q "^Name:" "$tmp"; then + rm -f "$tmp" + return 0 + fi + sleep 1 + done + cat "$tmp" >&2 2>/dev/null || true + rm -f "$tmp" + fail "$label could not resolve $PUBLIC_DNS_NAME" +} + +nslookup config.bigbox.home 172.28.32.1 | grep -q "192.168.1.1" || fail "config.bigbox.home did not resolve via jan DNS" +check_public_dns "VM public DNS via $PUBLIC_DNS_UPSTREAM" nslookup "$PUBLIC_DNS_NAME" "$PUBLIC_DNS_UPSTREAM" + +if [ "$HAVE_NETNS" = 1 ]; then + client_exec nslookup config.bigbox.home 172.28.32.1 | grep -q "192.168.1.1" || fail "client could not resolve config.bigbox.home via jan DNS" + check_public_dns "jan client public DNS via 172.28.32.1" client_exec nslookup "$PUBLIC_DNS_NAME" 172.28.32.1 +else + # OpenWrt default VM kernels often omit network namespaces. DHCP above still + # proves a throwaway veth client can reach dnsmasq over br-jan; this fallback + # proves the jan dnsmasq instance itself can forward public DNS. + check_public_dns "jan DNS public forwarding via 172.28.32.1" nslookup "$PUBLIC_DNS_NAME" 172.28.32.1 +fi + +printf "%s\n" "openwrt jan client DHCP/DNS/public-DNS: ok" +' diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_jan_client_per_host_shaping.sh b/tests/integration/openwrt_vm/tests/test_openwrt_jan_client_per_host_shaping.sh new file mode 100755 index 00000000..4bee0ebd --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_jan_client_per_host_shaping.sh @@ -0,0 +1,362 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +STAGE_DEVICECODE="$VM_DIR/scripts/ensure-devicecode-staged" +WAIT_SSH="$VM_DIR/scripts/wait-ssh" +CLEANUP="$VM_DIR/scripts/cleanup-shaping-state" +REMOTE="/tmp/devicecode-jan-client-per-host-shaping" +WORK="$VM_DIR/work/jan-client-per-host-shaping" + +CONFIG_STATE="/tmp/devicecode-jan-client-per-host-shaping-config-backup" +CONFIG_BACKED_UP=0 + +backup_config() { + "$SSH" "CONFIG_STATE='$CONFIG_STATE' sh -s" <<'REMOTE_BACKUP' +set -eu +rm -rf "$CONFIG_STATE" +mkdir -p "$CONFIG_STATE" +: > "$CONFIG_STATE/present" +: > "$CONFIG_STATE/absent" +for name in network dhcp firewall mwan3; do + if [ -e "/etc/config/$name" ]; then + cp -a "/etc/config/$name" "$CONFIG_STATE/$name" + echo "$name" >> "$CONFIG_STATE/present" + else + echo "$name" >> "$CONFIG_STATE/absent" + fi +done +REMOTE_BACKUP + CONFIG_BACKED_UP=1 +} + +restore_config() { + [ "${CONFIG_BACKED_UP:-0}" = 1 ] || return 0 + echo "[jan-shaping] restoring saved OpenWrt config" >&2 + # Do not run network reload synchronously over the same SSH session: reloading + # the management network can leave the client waiting on a half-closed + # connection. Restore the files synchronously, then launch network/firewall + # activation in the background and wait for SSH to come back. + "$SSH" "CONFIG_STATE='$CONFIG_STATE' RESTORE_ACTIVATE='${DEVICECODE_RESTORE_ACTIVATE:-1}' sh -s" <<'REMOTE_RESTORE' +set +e +if [ -d "$CONFIG_STATE" ]; then + while read name; do + [ -n "$name" ] && [ -e "$CONFIG_STATE/$name" ] && cp -a "$CONFIG_STATE/$name" "/etc/config/$name" + done < "$CONFIG_STATE/present" + while read name; do + [ -n "$name" ] && rm -f "/etc/config/$name" + done < "$CONFIG_STATE/absent" + if [ "${RESTORE_ACTIVATE:-1}" = 1 ]; then + ( + sleep 1 + /etc/init.d/network reload >/tmp/devicecode-jan-shaping-restore-network.log 2>&1 + /etc/init.d/firewall restart >/tmp/devicecode-jan-shaping-restore-firewall.log 2>&1 + ) /tmp/devicecode-jan-shaping-restore-wrapper.log 2>&1 & + echo restore-launched + else + echo restore-files-only + fi +fi +REMOTE_RESTORE + if [ "${DEVICECODE_RESTORE_ACTIVATE:-1}" = 1 ]; then + sleep 1 + "$WAIT_SSH" >/dev/null 2>&1 || true + fi +} + +cleanup_all() { + set +e + echo "[jan-shaping] cleaning shaping test state" >&2 + "$CLEANUP" >/dev/null 2>&1 || true + restore_config || true +} + +mkdir -p "$WORK" +if [ "${DC_SHAPING_CLEANUP_DISABLED:-0}" != 1 ]; then + backup_config + trap cleanup_all EXIT INT TERM +fi +cat > "$WORK/run_openwrt_jan_shaping_config.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local exec = require 'fibers.io.exec' +local unpack = table.unpack or unpack +local provider_loader = require 'services.hal.backends.network.provider' +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.05)) + end + if pred() then return true end + fail(label or 'condition was not satisfied before timeout') +end + +local function jan_intent(shaping_enabled) + return { + schema = 'devicecode.net.intent/1', + rev = shaping_enabled and 32037 or 32036, + generation = shaping_enabled and 32037 or 32036, + segments = { + lan = { + kind = 'lan', + addressing = { ipv4 = { mode = 'static', cidr = '192.168.1.1/24' } }, + dhcp = { enabled = true, start = 100, limit = 150, leasetime = '12h' }, + dns = { local_server = true, domain = 'vm.bigbox.test' }, + firewall = { zone = 'lan' }, + }, + jan = { + kind = 'user', + vlan = { id = 32 }, + addressing = { ipv4 = { mode = 'static', cidr = '172.28.32.1/24' } }, + dhcp = { enabled = false }, + dns = { local_server = true, domain = 'bigbox.home' }, + firewall = { zone = 'jan' }, + shaping = shaping_enabled and { + download = { limit = '8mbit' }, + upload = { limit = '8mbit' }, + host_default = { + mode = 'budgeted_peak', + all_hosts = true, + download = { sustained_rate = '2mbit', peak_rate = '2mbit', burst_budget = '100k' }, + upload = { sustained_rate = '2mbit', peak_rate = '2mbit', burst_budget = '100k' }, + fq_codel = { flows = 1024, limit = 10240 }, + }, + } or {}, + }, + wan = { kind = 'wan', firewall = { zone = 'wan' } }, + }, + interfaces = { + lan = { + kind = 'bridge', role = 'lan', segment = 'lan', members = { 'eth0' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.1.1/24' } }, + firewall = { zone = 'lan' }, + }, + wan = { + kind = 'ethernet', role = 'wan', segment = 'wan', endpoint = { ifname = 'eth1' }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, + dhcp = { enabled = false }, + firewall = { zone = 'wan' }, + }, + }, + dns = { enabled = true, domain = 'bigbox.home', upstreams = { '1.1.1.1' }, cache = { size = 1000 } }, + dhcp = { defaults = { authoritative = true } }, + firewall = { + defaults = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT' }, + zones = { + lan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + jan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + wan = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT', masq = true, mtu_fix = true }, + }, + policies = { + lan_to_wan = { src = 'lan', dest = 'wan' }, + jan_to_wan = { src = 'jan', dest = 'wan' }, + }, + }, + routing = { routes = {} }, + wan = {}, + vpn = {}, diagnostics = {}, + } +end + +local mode = arg and arg[1] or 'config' + +local function real_run_cmd(argv) + local cmd = exec.command(unpack(argv or {})) + local out, st, code, sig, err = perform(cmd:combined_output_op()) + if st == 'exited' and code == 0 then return true, out or '', nil, code, st, sig end + return nil, out or '', err or out or 'command failed', code, st, sig +end + +local provider = assert(provider_loader.new({ + provider = 'openwrt', + debounce_s = 0.01, + shaping_target_timeout_s = 5, + platform = { segment_trunk = { ifname = 'dcjantrunk0', protected = true } }, + -- The shell harness owns network activation so SSH can reconnect cleanly. + -- Structural activation remains skipped, but the shaping lane must run real + -- ip/tc commands after br-jan exists; otherwise the data-plane assertions only + -- test the provider's dry-run result. + run_cmd = function(argv) + print('jan-shaping config lane: skipped activation command ' .. table.concat(argv or {}, ' ')) + return true, nil + end, + shaper_run_cmd = real_run_cmd, +}, {})) + +fibers.run(function() + local shaping_enabled = (mode == 'shape') + local result = perform(provider:apply_op({ + intent = jan_intent(shaping_enabled), + opts = { generation = shaping_enabled and 32037 or 32036, apply_id = 'jan-client-per-host-shaping-' .. mode }, + })) + if not (result and result.ok == true) then fail('apply failed: ' .. tostring(result and result.err)) end + if shaping_enabled then + if not (result.shaping and result.shaping.ok == true) then fail('shaping result missing') end + if not (result.shaping.links and result.shaping.links.jan and result.shaping.links.jan.iface == 'br-jan') then + fail('jan shaping should target br-jan, got ' .. tostring(result.shaping and result.shaping.links and result.shaping.links.jan and result.shaping.links.jan.iface)) + end + else + local mgr = provider._uci_manager + wait_until(function() + local st = mgr and mgr.activation_status and mgr:activation_status() or nil + return st and (st.state == 'done' or st.state == 'idle') + end, 5, 'config-only activation runner should be idle') + end + provider:terminate('jan per-host shaping test ' .. mode) +end) + +print('openwrt jan per-host shaping ' .. mode .. ': ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'; ip netns del dc-jan-shape-client 2>/dev/null || true; ip link del dc-jan-host 2>/dev/null || true; ip link del dcjantrunk0 2>/dev/null || true; ip link del dcjantrunk0p 2>/dev/null || true; ip link add dcjantrunk0 type veth peer name dcjantrunk0p; ip link set dcjantrunk0 up; ip link set dcjantrunk0p up" +CODE_REMOTE="$($STAGE_DEVICECODE)" +"$SCP_TO" "$WORK/run_openwrt_jan_shaping_config.lua" "$REMOTE/run_openwrt_jan_shaping_config.lua" +"$SSH" "cd '$CODE_REMOTE' && lua '$REMOTE/run_openwrt_jan_shaping_config.lua' config" + +"$SSH" '(/etc/init.d/network reload >/tmp/devicecode-jan-shaping-network.log 2>&1; /etc/init.d/firewall restart >/tmp/devicecode-jan-shaping-firewall.log 2>&1) /tmp/devicecode-jan-shaping-activation.log 2>&1 &' || true +"$WAIT_SSH" + +set +e +"$SSH" 'set -eu + +fail() { echo "jan client per-host shaping test: $*" >&2; exit 1; } +cleanup() { + ip netns del dc-jan-shape-client 2>/dev/null || true + ip link del dc-jan-host 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +wait_for() { + label="$1"; timeout="$2"; shift 2 + cmd="$*" + deadline=$(( $(date +%s) + timeout )) + while [ "$(date +%s)" -le "$deadline" ]; do + if sh -c "$cmd" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + fail "timeout waiting for $label" +} + +modprobe ifb 2>/dev/null || true +modprobe sch_htb 2>/dev/null || true +modprobe sch_ingress 2>/dev/null || true +modprobe sch_fq_codel 2>/dev/null || true +modprobe cls_u32 2>/dev/null || true +modprobe act_mirred 2>/dev/null || true + +wait_for br-jan 45 "ip link show dev br-jan" +wait_for jan-address 45 "ip -4 addr show dev br-jan | grep -q 172.28.32.1" +wait_for vl-jan 45 "ip link show dev vl-jan" + +if ! ip netns add dc-jan-shape-client >/dev/null 2>&1; then + echo "jan client per-host shaping test: SKIP: network namespaces unavailable; cannot generate unambiguous client data-plane traffic" >&2 + exit 77 +fi +ip netns del dc-jan-shape-client >/dev/null 2>&1 || true +' +rc="$?" +set -e +if [ "$rc" -ne 0 ]; then + if [ "$rc" -eq 77 ]; then exit 0; fi + exit "$rc" +fi + +# Apply shaping only after br-jan exists. This proves the provider-derived +# segment shaping target is the bridge data device, not the VLAN member. +"$SSH" "cd '$CODE_REMOTE' && lua '$REMOTE/run_openwrt_jan_shaping_config.lua' shape" + +"$SSH" 'set -eu + +fail() { echo "jan client per-host shaping test: $*" >&2; exit 1; } +cleanup() { + ip netns del dc-jan-shape-client 2>/dev/null || true + ip link del dc-jan-host 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +class_bytes() { + dev="$1" + classid="$2" + tc -s class show dev "$dev" | awk -v c="$classid" '\'' + $1 == "class" && $2 == "htb" && $3 == c { hit = 1; next } + hit && $1 == "Sent" { print $2; exit } + '\'' +} + +assert_class_exists() { + dev="$1"; classid="$2" + tc class show dev "$dev" | grep -q "class htb $classid" || fail "$dev missing class $classid" +} + +assert_counter_moves() { + label="$1"; dev="$2"; classid="$3"; shift 3 + before="$(class_bytes "$dev" "$classid")" + before="${before:-0}" + "$@" >/tmp/dc-jan-shaping-traffic.out 2>&1 || { + cat /tmp/dc-jan-shaping-traffic.out >&2 2>/dev/null || true + fail "$label traffic command failed" + } + after="$(class_bytes "$dev" "$classid")" + after="${after:-0}" + [ "$after" -gt "$before" ] || { + echo "--- tc class $dev ---" >&2 + tc -s class show dev "$dev" >&2 2>/dev/null || true + echo "--- tc filter br-jan parent 1: ---" >&2 + tc -s filter show dev br-jan parent 1: >&2 2>/dev/null || true + echo "--- tc filter br-jan ingress ---" >&2 + tc -s filter show dev br-jan parent ffff: >&2 2>/dev/null || true + fail "$label did not move $dev class $classid counter: before=$before after=$after" + } +} + +ip netns del dc-jan-shape-client 2>/dev/null || true +ip link del dc-jan-host 2>/dev/null || true +ip link add dc-jan-host type veth peer name dc-jan-client0 +ip link set dc-jan-host master br-jan +ip link set dc-jan-host up +ip netns add dc-jan-shape-client +ip link set dc-jan-client0 netns dc-jan-shape-client +ip netns exec dc-jan-shape-client ip link set lo up +ip netns exec dc-jan-shape-client ip link set dc-jan-client0 up +ip netns exec dc-jan-shape-client ip addr add 172.28.32.36/24 dev dc-jan-client0 +ip netns exec dc-jan-shape-client ip route replace default via 172.28.32.1 dev dc-jan-client0 + +# Confirm the shaping hierarchy is on the bridge-facing segment device. The old +# bug installed this tree on vl-jan, which would leave these assertions failing. +tc qdisc show dev br-jan | grep -q "qdisc htb 1:" || fail "br-jan does not have root HTB shaping qdisc" +tc qdisc show dev ifb_br_jan | grep -q "qdisc htb 1:" || fail "ifb_br_jan does not have root HTB shaping qdisc" +assert_class_exists br-jan 1:20 +assert_class_exists br-jan 20:1036 +assert_class_exists br-jan 1036:1 +assert_class_exists ifb_br_jan 1:20 +assert_class_exists ifb_br_jan 20:1036 +assert_class_exists ifb_br_jan 1036:1 +if tc qdisc show dev vl-jan 2>/dev/null | grep -q "qdisc htb 1:"; then + fail "stale shaping qdisc remains on vl-jan" +fi +if tc qdisc show dev ifb_vl_jan 2>/dev/null | grep -q "qdisc htb 1:"; then + fail "stale shaping qdisc remains on ifb_vl_jan" +fi + +assert_counter_moves "router-to-client" br-jan 20:1036 ping -I br-jan -c 20 -s 1200 172.28.32.36 +assert_counter_moves "client-to-router" ifb_br_jan 20:1036 ip netns exec dc-jan-shape-client ping -c 20 -s 1200 172.28.32.1 + +printf "%s\n" "openwrt jan client per-host shaping data-plane: ok" +' + diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_network_observer_event_ingress.sh b/tests/integration/openwrt_vm/tests/test_openwrt_network_observer_event_ingress.sh new file mode 100755 index 00000000..33bd52f7 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_network_observer_event_ingress.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-network-observer-event-ingress-test" +WORK="$VM_DIR/work/network-observer-event-ingress-test" + +mkdir -p "$WORK" + +cat > "$WORK/run_openwrt_network_observer_event_ingress.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', + './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', + './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', + './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', + './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local channel = require 'fibers.channel' +local provider_loader = require 'services.hal.backends.network.provider' +local hotplug_client = require 'services.hal.backends.network.providers.openwrt.hotplug_client' + +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function ok(v, msg) if not v then fail(msg or 'assertion failed') end return v end +local function eq(a, b, msg) if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function note(msg) io.stderr:write('[observer-ingress] ', tostring(msg), '\n') end + +local function wait_for_subject(rx, subject, timeout_s) + local deadline = fibers.now() + (timeout_s or 15.0) + while fibers.now() < deadline do + local remaining = deadline - fibers.now() + if remaining < 0 then remaining = 0 end + local which, msg = perform(fibers.named_choice({ + event = rx:get_op(), + timer = sleep.sleep_op(remaining), + })) + if which ~= 'event' then break end + if msg and msg.data and msg.data.subject == subject then return msg.data end + end + return nil, 'timed out waiting for ' .. tostring(subject) +end + +local function send_with_retry(record, socket_path) + local last + for _ = 1, 20 do + local sent, err = hotplug_client.send(record, { socket_path = socket_path }) + if sent then return true end + last = err + perform(sleep.sleep_op(0.05)) + end + return nil, last +end + +local function make_provider(ch, socket_path, opts) + opts = opts or {} + local provider, perr = provider_loader.new({ + provider = 'openwrt', + observer_socket_path = socket_path, + observer_debounce_s = opts.debounce_s or 0.05, + enable_live_snapshot = true, + enable_hotplug_socket = opts.enable_hotplug_socket ~= false, + enable_ubus_listener = opts.enable_ubus_listener == true, + initial_observation_snapshot = false, + }, { cap_emit_ch = ch }) + assert(provider, perr) + local watch = perform(provider:watch_op({})) + assert(watch and watch.ok == true, 'watch failed: ' .. tostring(watch and watch.err)) + if socket_path then eq(watch.socket_path, socket_path, 'watch socket path') end + return provider, watch +end + +fibers.run(function(scope) + local watchdog = assert(scope:child()) + local ok_watchdog, watchdog_err = watchdog:spawn(function() + perform(sleep.sleep_op(30.0)) + io.stderr:write('openwrt network observer event ingress: timed out\n') + os.exit(124) + end) + assert(ok_watchdog, watchdog_err) + + -- Keep the synthetic hotplug/MWAN socket path isolated from the ubus listener. + -- When MWAN3 is installed and active, ubus may emit background network events; + -- those events are valid, but they make this targeted socket-ingress assertion + -- timing-sensitive. A separate provider below tests the ubus ingress path. + note('starting provider watch for direct hotplug ingress') + local socket_path = '/tmp/devicecode-net-observe-test.sock' + os.remove(socket_path) + local ch = channel.new(64) + local provider = make_provider(ch, socket_path, { + enable_hotplug_socket = true, + enable_ubus_listener = false, + }) + + perform(sleep.sleep_op(0.25)) + note('injecting hotplug iface event') + + local sent, serr = provider:ingest_observation({ + source = 'hotplug', kind = 'hotplug', directory = 'iface', + env = { ACTION = 'ifup', INTERFACE = 'lan', DEVICE = 'br-lan', SUBSYSTEM = 'iface' }, + }) + assert(sent, 'hotplug ingest failed: ' .. tostring(serr)) + + local iface_ev = assert(wait_for_subject(ch, 'interface:lan', 15.0)) + note('received interface:lan') + eq(iface_ev.kind, 'interface_changed', 'hotplug interface event kind') + ok(type(iface_ev.observed) == 'table', 'hotplug event should include observed snapshot') + ok(type(iface_ev.observed.live) == 'table', 'hotplug event should include live snapshot') + + note('injecting mwan3 synthetic event') + + sent, serr = provider:ingest_observation({ + source = 'mwan3', kind = 'mwan3', directory = 'mwan3.user', + env = { ACTION = 'connected', INTERFACE = 'wan', DEVICE = 'eth1' }, + }) + assert(sent, 'mwan3 ingest failed: ' .. tostring(serr)) + + local mwan_ev = assert(wait_for_subject(ch, 'mwan:wan', 15.0)) + note('received mwan:wan') + eq(mwan_ev.kind, 'mwan_member_changed', 'mwan3 event kind') + ok(type(mwan_ev.observed.multiwan) == 'table', 'mwan3 event should include multiwan snapshot') + + note('terminating direct-ingress provider') + provider:terminate('direct ingress path complete') + perform(sleep.sleep_op(0.1)) + + note('starting provider watch for ubus') + local ubus_ch = channel.new(128) + local ubus_provider = make_provider(ubus_ch, '/tmp/devicecode-net-observe-test-ubus.sock', { + enable_hotplug_socket = false, + enable_ubus_listener = true, + debounce_s = 0.02, + }) + + perform(sleep.sleep_op(0.5)) + note('sending ubus network.interface event') + + local ubus_seen + for _ = 1, 30 do + os.execute("ubus send network.interface '{\"action\":\"ifupdate\",\"interface\":\"loopback\"}' >/dev/null 2>&1") + ubus_seen = wait_for_subject(ubus_ch, 'interface:loopback', 1.0) + if ubus_seen then break end + end + local ubus_ev = assert(ubus_seen, 'timed out waiting for interface:loopback from ubus listener') + note('received interface:loopback') + eq(ubus_ev.source, 'ubus', 'ubus listener event source') + eq(ubus_ev.kind, 'interface_changed', 'ubus interface event kind') + + note('terminating provider') + ubus_provider:terminate('test complete') + note('provider terminated') + watchdog:cancel('test complete') + perform(watchdog:join_op()) +end) + +print('openwrt network observer event ingress: ok') +os.exit(0) +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_network_observer_event_ingress.lua" "$REMOTE/run_openwrt_network_observer_event_ingress.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_network_observer_event_ingress.lua" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_apply.sh b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_apply.sh new file mode 100755 index 00000000..392240b6 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_apply.sh @@ -0,0 +1,307 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-network-provider-test" +WORK="$VM_DIR/work/network-provider-test" + +mkdir -p "$WORK" + +cat > "$WORK/run_openwrt_network_provider_apply.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', + './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', + './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', + './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', + './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local bus = require 'bus' +local trie = require 'trie' +local uci = require 'uci' +local provider_loader = require 'services.hal.backends.network.provider' +local names_mod = require 'services.hal.backends.network.providers.openwrt.names' + +assert(type(bus.new) == 'function', 'vendored bus did not load') +assert(type(trie.new_pubsub) == 'function', 'vendored trie did not load') + +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function eq(a, b, msg) + if a ~= b then + fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) + end +end + +local function assert_list(v, expected, label) + if type(v) ~= 'table' then fail((label or 'list') .. ' should be a table, got ' .. type(v)) end + eq(#v, #expected, (label or 'list') .. ' length') + for i = 1, #expected do eq(v[i], expected[i], (label or 'list') .. '[' .. i .. ']') end +end + +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.01)) + end + if pred() then return true end + fail(label or 'condition was not satisfied before timeout') +end + +local function mkdir_p(path) + local ok = os.execute("mkdir -p '" .. path .. "'") + if ok ~= true and ok ~= 0 then fail('mkdir failed for ' .. path) end +end + +local tmp = '/tmp/dc-network-provider-apply' +os.execute("rm -rf '" .. tmp .. "'") +local conf = tmp .. '/conf' +local save = tmp .. '/save' +mkdir_p(conf) +mkdir_p(save) +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do + local f = assert(io.open(conf .. '/' .. pkg, 'w')) + f:write('# devicecode OpenWrt network provider test\n') + f:close() +end + +local restarts = {} + +local intent = { + schema = 'devicecode.net.intent/1', + rev = 101, + generation = 3, + segments = { + lan = { + kind = 'lan', + vlan = { id = 10 }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.10.1/24' } }, + dhcp = { enabled = true, start = 20, limit = 50, leasetime = '6h' }, + dns = { local_server = true, domain = 'bigbox.home', host_files = { 'ads' } }, + firewall = { zone = 'lan' }, + }, + wan = { + kind = 'wan', + firewall = { zone = 'wan' }, + }, + }, + interfaces = { + lan = { + kind = 'bridge', + role = 'lan', + segment = 'lan', + members = { 'eth0', 'eth1' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.10.1/24' } }, + }, + wan = { + kind = 'ethernet', + role = 'wan', + segment = 'wan', + endpoint = { ifname = 'eth2' }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, + }, + }, + dns = { + enabled = true, + domain = 'bigbox.home', + upstreams = { '1.1.1.1', '8.8.8.8' }, + cache = { size = 1000 }, + host_files = { base_dir = '/tmp/devicecode-dns-hosts', sources = { ads = { file = 'ads.hosts' } } }, + records = { router = { name = 'config.bigbox.home', address = '192.168.10.1' }, bad = { name = 'bad.bigbox.home', address = '$UNIFI-ADDRESS' } }, + }, + dhcp = { defaults = { authoritative = true } }, + firewall = { + defaults = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT' }, + zones = { + lan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + wan = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT', masq = true, mtu_fix = true }, + }, + policies = { + lan_to_wan = { src = 'lan', dest = 'wan' }, + }, + }, + routing = { + routes = { + { interface = 'lan', target = '10.0.0.0/8', gateway = '192.168.10.254' }, + }, + }, + wan = { enabled = true, members = { wan = { interface = 'wan', mwan_metric = 1, weight = 1 } } }, + + vpn = {}, + diagnostics = {}, +} + +local name_ctx = assert(names_mod.allocate(intent)) + +fibers.run(function(_scope) + local provider, perr = provider_loader.new({ + provider = 'openwrt', + confdir = conf, + savedir = save, + debounce_s = 0.01, + run_cmd = function(argv) + restarts[#restarts + 1] = table.concat(argv, ' ') + return true, nil + end, + }, {}) + assert(provider, perr) + + local valid = perform(provider:validate_op({ intent = intent })) + assert(valid and valid.ok == true, 'validate failed: ' .. tostring(valid and valid.err)) + + local plan = perform(provider:plan_op({ intent = intent })) + assert(plan and plan.ok == true, 'plan failed: ' .. tostring(plan and plan.err)) + assert(plan.plan.packages.network.changes > 0, 'network plan should have changes') + assert(plan.plan.packages.dhcp.changes > 0, 'dhcp plan should have changes') + assert(plan.plan.packages.firewall.changes > 0, 'firewall plan should have changes') + + local result = perform(provider:apply_op({ intent = intent })) + assert(result and result.ok == true, 'apply failed: ' .. tostring(result and result.err)) + eq(result.backend, 'openwrt', 'backend') + eq(result.intent_rev, 101, 'intent_rev') + assert(result.activation == nil, 'provider activation should be synchronous for structural network apply') + eq(#restarts, 4, 'activation command count') + + provider:terminate('test complete') +end) + +eq(#restarts, 4, 'restart command count') +eq(restarts[1], '/etc/init.d/network reload', 'network reload') +eq(restarts[2], '/etc/init.d/dnsmasq restart', 'dnsmasq restart') +eq(restarts[3], '/etc/init.d/firewall restart', 'firewall restart') +eq(restarts[4], '/etc/init.d/mwan3 restart', 'mwan3 restart') + +local c = assert(uci.cursor(conf, save)) +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall' }) do + if type(c.load) == 'function' then pcall(function() c:load(pkg) end) end +end + +local function all(pkg) + if type(c.load) == 'function' then pcall(function() c:load(pkg) end) end + return c:get_all(pkg) or {} +end + +local function list_contains(v, item) + if type(v) == 'table' then + for i = 1, #v do if v[i] == item then return true end end + return false + end + return v == item +end + +local function section_by_name(pkg, section, stype) + local sec = all(pkg)[section] + if type(sec) ~= 'table' then fail('section not found in ' .. pkg .. ': ' .. tostring(section)) end + if stype ~= nil and sec['.type'] ~= stype then fail(pkg .. '.' .. tostring(section) .. ' expected type ' .. tostring(stype) .. ', got ' .. tostring(sec['.type'])) end + return section, sec +end + +local function assert_no_devicecode_metadata() + for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do + for name, sec in pairs(all(pkg)) do + if type(sec) == 'table' then + for _, opt in ipairs({ 'devicecode_managed', 'devicecode_owner', 'devicecode_semantic_id', 'devicecode_role' }) do + if sec[opt] ~= nil then fail('unexpected UCI metadata ' .. pkg .. '.' .. tostring(name) .. '.' .. opt) end + end + end + end + end +end + +local function find_firewall_zone(zone_name) + for name, sec in pairs(all('firewall')) do + if type(sec) == 'table' and sec['.type'] == 'zone' and sec.name == zone_name then + return name, sec + end + end + fail('firewall zone not found: ' .. tostring(zone_name)) +end + +local function find_firewall_forwarding(src, dest) + for name, sec in pairs(all('firewall')) do + if type(sec) == 'table' and sec['.type'] == 'forwarding' and sec.src == src and sec.dest == dest then + return name, sec + end + end + fail('firewall forwarding not found: ' .. tostring(src) .. ' -> ' .. tostring(dest)) +end + +local lan_bridge_sec, lan_bridge = section_by_name('network', name_ctx:section('dev_bridge', 'lan'), 'device') +eq(lan_bridge.name, 'br-lan', 'bridge name') +eq(lan_bridge.type, 'bridge', 'bridge device type') +assert_list(lan_bridge.ports, { 'eth0', 'eth1' }, 'bridge ports') + +local _lan_if_sec, lan_if = section_by_name('network', name_ctx:iface('lan'), 'interface') +eq(_lan_if_sec, 'lan', 'lan generated interface remains readable') +eq(lan_if.proto, 'static', 'lan proto') +eq(lan_if.device, 'br-lan', 'lan device') +eq(lan_if.ipaddr, '192.168.10.1', 'lan ipaddr') +eq(lan_if.netmask, '255.255.255.0', 'lan netmask') + +local _wan_if_sec, wan_if = section_by_name('network', name_ctx:iface('wan'), 'interface') +eq(_wan_if_sec, 'wan', 'wan generated interface remains readable') +eq(wan_if.proto, 'dhcp', 'wan proto') +eq(wan_if.device, 'eth2', 'wan device') +eq(wan_if.peerdns, '0', 'wan peerdns') +eq(wan_if.defaultroute, nil, 'wan defaultroute should use OpenWrt default') +eq(wan_if.metric, '11', 'wan auto route metric') + +local _route_sec, route = section_by_name('network', name_ctx:section('route', '1'), 'route') +eq(route.interface, 'lan', 'route interface') +eq(route.target, '10.0.0.0/8', 'route target') +eq(route.gateway, '192.168.10.254', 'route gateway') + +local dns_sec, dns = nil, nil +for name, sec in pairs(all('dhcp')) do + if type(sec) == 'table' and sec['.type'] == 'dnsmasq' and list_contains(sec.addnhosts, '/tmp/devicecode-dns-hosts/ads.hosts') then dns_sec, dns = name, sec; break end +end +if not dns then fail('ads dnsmasq instance not found') end +assert_list(dns.server, { '1.1.1.1', '8.8.8.8' }, 'dns upstreams') +eq(dns.cachesize, '1000', 'dns cache size') +local addnhosts = dns.addnhosts +if type(addnhosts) == 'table' then eq(addnhosts[1], '/tmp/devicecode-dns-hosts/ads.hosts', 'dns host file') else eq(addnhosts, '/tmp/devicecode-dns-hosts/ads.hosts', 'dns host file') end +local addresses = dns.address +local address_s = type(addresses) == 'table' and table.concat(addresses, ' ') or tostring(addresses) +if not address_s:find('/config.bigbox.home/192.168.10.1', 1, true) then fail('dns record missing: ' .. address_s) end +if address_s:find('$UNIFI-ADDRESS', 1, true) then fail('invalid dns record was emitted: ' .. address_s) end + +local _dhcp_sec, dhcp_lan = section_by_name('dhcp', name_ctx:section('dhcp', 'lan'), 'dhcp') +eq(dhcp_lan.interface, 'lan', 'dhcp interface') +eq(dhcp_lan.instance, dns_sec, 'dhcp instance') +eq(dhcp_lan.start, '20', 'dhcp start') +eq(dhcp_lan.limit, '50', 'dhcp limit') +eq(dhcp_lan.leasetime, '6h', 'dhcp leasetime') + +local _fw_defaults, fw_defaults = section_by_name('firewall', 'defaults', 'defaults') +eq(fw_defaults.input, 'REJECT', 'firewall defaults input') +local _zone_lan_sec, zone_lan = find_firewall_zone('lan') +eq(zone_lan.name, 'lan', 'lan zone name') +assert_list(zone_lan.network, { 'lan' }, 'lan zone networks') +local _zone_wan_sec, zone_wan = find_firewall_zone('wan') +eq(zone_wan.masq, '1', 'wan zone masq') +eq(zone_wan.mtu_fix, '1', 'wan zone mtu_fix') +local _fwd_sec, fwd = find_firewall_forwarding('lan', 'wan') +eq(fwd.src, 'lan', 'forwarding src') +eq(fwd.dest, 'wan', 'forwarding dest') + +assert_no_devicecode_metadata() +print('openwrt network provider minimal apply: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_network_provider_apply.lua" "$REMOTE/run_openwrt_network_provider_apply.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_network_provider_apply.lua" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_async_activation.sh b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_async_activation.sh new file mode 100755 index 00000000..f13ebeb5 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_async_activation.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-network-provider-async-activation-test" +WORK="$VM_DIR/work/network-provider-async-activation-test" + +mkdir -p "$WORK" + +cat > "$WORK/run_openwrt_network_provider_async_activation.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', + './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', + './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', + './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', + './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' +local uci = require 'uci' +local provider_loader = require 'services.hal.backends.network.provider' + +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function eq(a, b, msg) + if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end +end +local function assert_true(v, msg) if not v then fail(msg or 'assertion failed') end end + +local function mkdir_p(path) + local ok = os.execute("mkdir -p '" .. path .. "'") + if ok ~= true and ok ~= 0 then fail('mkdir failed for ' .. path) end +end + +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.01)) + end + if pred() then return true end + fail(label or 'condition was not satisfied before timeout') +end + +local tmp = '/tmp/dc-openwrt-provider-async-activation' +os.execute("rm -rf '" .. tmp .. "'") +local conf = tmp .. '/conf' +local save = tmp .. '/save' +mkdir_p(conf) +mkdir_p(save) +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do + local f = assert(io.open(conf .. '/' .. pkg, 'w')) + f:write('# devicecode provider synchronous activation integration test\n') + f:close() +end + +local intent = { + schema = 'devicecode.net.intent/1', + rev = 909, + generation = 44, + segments = { + lan = { + kind = 'lan', + addressing = { ipv4 = { mode = 'static', cidr = '192.168.10.1/24' } }, + dhcp = { enabled = true, start = 20, limit = 50, leasetime = '6h' }, + dns = { local_server = true, domain = 'bigbox.home' }, + firewall = { zone = 'lan' }, + }, + wan = { kind = 'wan', firewall = { zone = 'wan' } }, + }, + interfaces = { + lan = { + kind = 'bridge', role = 'lan', segment = 'lan', members = { 'eth0' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.10.1/24' } }, + }, + wan = { + kind = 'ethernet', role = 'wan', segment = 'wan', endpoint = { ifname = 'eth1' }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, + }, + }, + dns = { enabled = true, domain = 'bigbox.home', upstreams = { '1.1.1.1' }, cache = { size = 1000 } }, + dhcp = { defaults = { authoritative = true } }, + firewall = { + defaults = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT' }, + zones = { lan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, wan = { masq = true, mtu_fix = true } }, + policies = { lan_to_wan = { src = 'lan', dest = 'wan' } }, + }, + routing = { routes = {} }, + wan = { enabled = true, members = { wan = { interface = 'wan', mwan_metric = 1, weight = 1 } } }, + + vpn = {}, + diagnostics = {}, +} + +local restarts = {} +local run_started = false +local run_finished = false +local unblock_tx, unblock_rx = mailbox.new(1, { full = 'reject_newest' }) +local result = nil +local provider = nil + +fibers.run(function(scope) + provider = assert(provider_loader.new({ + provider = 'openwrt', + confdir = conf, + savedir = save, + debounce_s = 0.01, + run_cmd = function(argv) + restarts[#restarts + 1] = table.concat(argv, ' ') + run_started = true + if #restarts == 1 then + perform(unblock_rx:recv_op()) + end + run_finished = true + return true, nil + end, + }, {})) + + scope:spawn(function() + result = perform(provider:apply_op({ intent = intent, opts = { generation = 44, apply_id = 7 } })) + end) + + -- Provider structural apply now owns activation completion. The apply result + -- must not be reported while network/dnsmasq/firewall/mwan3 activation is + -- still blocked. + wait_until(function() return run_started == true end, 5, 'activation should start first command') + eq(restarts[1], '/etc/init.d/network reload', 'first activation command') + perform(sleep.sleep_op(0.05)) + assert_true(result == nil, 'provider apply must wait for activation command completion') + + local c = assert(uci.cursor(conf, save)) + if type(c.load) == 'function' then pcall(function() c:load('network') end) end + eq(c:get('network', 'lan'), 'interface', 'network.lan committed before activation completion') + eq(c:get('network', 'lan', 'proto'), 'static', 'network.lan proto committed before activation completion') + + assert(unblock_tx:send(true)) + wait_until(function() return result ~= nil end, 5, 'provider apply should complete after activation is released') + assert_true(result.ok == true, 'provider apply failed: ' .. tostring(result.err)) + assert_true(result.transaction and result.transaction.ok == true, 'transaction should succeed') + assert_true(result.activation == nil, 'synchronous provider activation should not return a scheduled activation token') + + eq(#restarts, 4, 'provider should run network, dnsmasq, firewall and mwan3 activation before reply') + eq(restarts[2], '/etc/init.d/dnsmasq restart', 'second activation command') + eq(restarts[3], '/etc/init.d/firewall restart', 'third activation command') + eq(restarts[4], '/etc/init.d/mwan3 restart', 'fourth activation command') + + provider:terminate('test complete') +end) + +print('openwrt network provider synchronous activation contract: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_network_provider_async_activation.lua" "$REMOTE/run_openwrt_network_provider_async_activation.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_network_provider_async_activation.lua" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_complete_rewrite.sh b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_complete_rewrite.sh new file mode 100755 index 00000000..a2acab1a --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_complete_rewrite.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-network-provider-complete-rewrite-test" +WORK="$VM_DIR/work/network-provider-complete-rewrite-test" + +mkdir -p "$WORK" +cat > "$WORK/run_openwrt_network_provider_complete_rewrite.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local uci = require 'uci' +local provider_loader = require 'services.hal.backends.network.provider' +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function eq(a, b, msg) if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function mkdir_p(path) local ok = os.execute("mkdir -p '" .. path .. "'"); if ok ~= true and ok ~= 0 then fail('mkdir failed for ' .. path) end end + +local tmp = '/tmp/dc-network-provider-complete-rewrite' +os.execute("rm -rf '" .. tmp .. "'") +local conf, save = tmp .. '/conf', tmp .. '/save' +mkdir_p(conf); mkdir_p(save) + +-- Deliberately create only two stale files. The provider/manager must create +-- missing packages and must completely replace stale package contents. +local f = assert(io.open(conf .. '/network', 'w')) +f:write([[config interface 'lan' + option proto 'dhcp' + option stale_option 'must_disappear' + +config interface 'oldwan' + option proto 'dhcp' +]]) +f:close() +f = assert(io.open(conf .. '/dhcp', 'w')) +f:write([[config dhcp 'lan' + option interface 'lan' + option instance 'old_dnsmasq' + option stale_option 'must_disappear' + +config dnsmasq 'old_dnsmasq' + option domainneeded '1' +]]) +f:close() + +local intent = { + schema = 'devicecode.net.intent/1', rev = 1, + segments = { + lan = { kind = 'lan', vlan = { id = 10 }, addressing = { ipv4 = { mode = 'static', cidr = '192.168.10.1/24' } }, dhcp = { enabled = true }, dns = { local_server = true }, firewall = { zone = 'lan' } }, + }, + interfaces = {}, + dns = { enabled = true, domain = 'bigbox.home', upstreams = { '1.1.1.1' } }, + dhcp = {}, firewall = { zones = { lan = {} } }, routing = {}, wan = {}, vpn = {}, diagnostics = {}, +} + +fibers.run(function() + local provider = assert(provider_loader.new({ provider = 'openwrt', confdir = conf, savedir = save, debounce_s = 0.01, platform = { segment_trunk = { ifname = 'eth0' } }, run_cmd = function() return true, nil end }, {})) + local result = perform(provider:apply_op({ intent = intent })) + assert(result and result.ok == true, 'apply failed: ' .. tostring(result and result.err)) + provider:terminate('test complete') +end) + +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do + local fh = io.open(conf .. '/' .. pkg, 'rb') + if not fh then fail('missing generated package file ' .. pkg) end + fh:close() +end + +local c = assert(uci.cursor(conf, save)) +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do if type(c.load) == 'function' then pcall(function() c:load(pkg) end) end end + +eq(c:get('network', 'oldwan'), nil, 'stale network section removed') +eq(c:get('network', 'lan', 'stale_option'), nil, 'stale network option removed from recreated section') +eq(c:get('dhcp', 'old_dnsmasq'), nil, 'stale dnsmasq removed') +local dhcp_sec = nil +for name, sec in pairs(c:get_all('dhcp') or {}) do if type(sec) == 'table' and sec['.type'] == 'dhcp' and sec.interface == 'lan' then dhcp_sec = sec end end +assert(dhcp_sec, 'lan dhcp section expected') +eq(dhcp_sec.stale_option, nil, 'stale dhcp option removed') +if dhcp_sec.instance == 'old_dnsmasq' then fail('dhcp instance should not point to stale dnsmasq') end + +eq(c:get('network', 'loopback'), 'interface', 'loopback generated') +eq(c:get('network', 'loopback', 'device'), 'lo', 'loopback device') + +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do + for name, sec in pairs(c:get_all(pkg) or {}) do + if type(sec) == 'table' then + for _, opt in ipairs({ 'devicecode_managed', 'devicecode_owner', 'devicecode_semantic_id', 'devicecode_role' }) do + if sec[opt] ~= nil then fail('unexpected UCI metadata ' .. pkg .. '.' .. tostring(name) .. '.' .. opt) end + end + end + end +end +print('openwrt complete rewrite and missing package creation: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_network_provider_complete_rewrite.lua" "$REMOTE/run_openwrt_network_provider_complete_rewrite.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_network_provider_complete_rewrite.lua" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_dnsmasq_groups.sh b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_dnsmasq_groups.sh new file mode 100755 index 00000000..07020696 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_dnsmasq_groups.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-network-provider-dnsmasq-groups-test" +WORK="$VM_DIR/work/network-provider-dnsmasq-groups-test" + +mkdir -p "$WORK" +cat > "$WORK/run_openwrt_network_provider_dnsmasq_groups.lua" <<'LUA' +package.path = table.concat({ './src/?.lua', './src/?/init.lua', './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', package.path }, ';') +local fibers = require 'fibers' +local uci = require 'uci' +local provider_loader = require 'services.hal.backends.network.provider' +local names_mod = require 'services.hal.backends.network.providers.openwrt.names' +local perform = fibers.perform +local function fail(msg) error(msg, 2) end +local function eq(a,b,msg) if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function mkdir_p(path) local ok=os.execute("mkdir -p '"..path.."'"); if ok ~= true and ok ~= 0 then fail('mkdir failed') end end +local function contains(list, value) if type(list)=='table' then for i=1,#list do if list[i]==value then return true end end; return false end return list==value end + +local tmp='/tmp/dc-network-provider-dnsmasq-groups'; os.execute("rm -rf '"..tmp.."'") +local conf, save=tmp..'/conf', tmp..'/save'; mkdir_p(conf); mkdir_p(save) +for _,pkg in ipairs({'network','dhcp','firewall','mwan3'}) do local f=assert(io.open(conf..'/'..pkg,'w')); f:close() end + +local function seg(kind, vid, host_files) + return { kind=kind or 'lan', vlan={id=vid}, addressing={ipv4={mode='static', cidr='192.168.'..tostring(vid)..'.1/24'}}, dhcp={enabled=true}, dns={local_server=true, host_files=host_files or {}, domain='bigbox.home'}, firewall={zone='lan'} } +end +local intent={ schema='devicecode.net.intent/1', rev=1, segments={ adm=seg('system',8,{'ads'}), ops=seg('system',9,{'ads'}), jan=seg('user',32,{'ads','adult'}), int=seg('system',100,{}) }, interfaces={}, dns={enabled=true, domain='bigbox.home', upstreams={'1.1.1.1'}, cache={size=1000}, host_files={base_dir='/data/devicecode/dns/hosts', addnmount=true, sources={ads={file='ads.hosts'}, adult={file='adult.hosts'}}}}, dhcp={}, firewall={zones={lan={}}}, routing={}, wan={}, vpn={}, diagnostics={} } +local name_ctx = assert(names_mod.allocate(intent)) + +fibers.run(function() + local provider=assert(provider_loader.new({provider='openwrt', confdir=conf, savedir=save, debounce_s=0.01, platform={segment_trunk={ifname='eth0'}}, run_cmd=function() return true,nil end}, {})) + local r=perform(provider:apply_op({intent=intent})); assert(r and r.ok==true, 'apply failed: '..tostring(r and r.err)); provider:terminate('done') +end) +local c=assert(uci.cursor(conf,save)); for _,pkg in ipairs({'dhcp'}) do if type(c.load)=='function' then pcall(function() c:load(pkg) end) end end +local dns_count=0; local ads_instance; local adult_instance; local standard_instance +for name,sec in pairs(c:get_all('dhcp') or {}) do + if type(sec)=='table' and sec['.type']=='dnsmasq' then + dns_count=dns_count+1 + assert(#name <= 15, 'dnsmasq instance name too long: '..name) + if contains(sec.addnhosts, '/data/devicecode/dns/hosts/ads.hosts') and not contains(sec.addnhosts, '/data/devicecode/dns/hosts/adult.hosts') then ads_instance=name end + if contains(sec.addnhosts, '/data/devicecode/dns/hosts/adult.hosts') then adult_instance=name end + if sec.addnhosts == nil then standard_instance=name end + end +end +eq(dns_count, 3, 'identical DNS policy should be grouped') +assert(ads_instance, 'ads instance expected'); assert(adult_instance, 'ads+adult instance expected'); assert(standard_instance, 'standard instance expected') +local all_dhcp = c:get_all('dhcp') or {} +assert(contains(all_dhcp[ads_instance].notinterface, 'lo'), 'ads instance should exclude loopback: '..ads_instance) +assert(contains(all_dhcp[adult_instance].notinterface, 'lo'), 'ads+adult instance should exclude loopback: '..adult_instance) +assert(not contains(all_dhcp[standard_instance].notinterface, 'lo'), 'standard/int instance should own loopback: '..standard_instance) +local function dhcp_for(seg) + local wanted = name_ctx:iface(seg) + for _,sec in pairs(c:get_all('dhcp') or {}) do if type(sec)=='table' and sec['.type']=='dhcp' and sec.interface==wanted then return sec end end + fail('dhcp section not found for '..seg) +end +eq(dhcp_for('adm').instance, ads_instance, 'adm shares ads instance') +eq(dhcp_for('ops').instance, ads_instance, 'ops shares ads instance') +eq(dhcp_for('jan').instance, adult_instance, 'jan uses adult instance') +eq(dhcp_for('int').instance, standard_instance, 'int uses standard instance') +print('openwrt dnsmasq grouping: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_network_provider_dnsmasq_groups.lua" "$REMOTE/run_openwrt_network_provider_dnsmasq_groups.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_network_provider_dnsmasq_groups.lua" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_fw4_schema.sh b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_fw4_schema.sh new file mode 100755 index 00000000..01a7b506 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_fw4_schema.sh @@ -0,0 +1,320 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-network-provider-fw4-schema-test" +WORK="$VM_DIR/work/network-provider-fw4-schema-test" + +mkdir -p "$WORK" + +if grep -R '"disable_ipv6"' "$ROOT_DIR/src/configs" >/dev/null 2>&1; then + echo 'product config must not model disable_ipv6 as a firewall default' >&2 + grep -R '"disable_ipv6"' "$ROOT_DIR/src/configs" >&2 || true + exit 1 +fi + + +cat > "$WORK/run_openwrt_network_provider_fw4_schema.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', + './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', + './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', + './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', + './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local uci = require 'uci' +local provider_loader = require 'services.hal.backends.network.provider' +local perform = fibers.perform + +local failures = {} +local function record(ok, msg) + if not ok then failures[#failures + 1] = msg end +end +local function eq(a, b, msg) + record(a == b, (msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) +end +local function absent(v, msg) + record(v == nil, (msg or 'value should be absent') .. ': got ' .. tostring(v)) +end +local function contains(list, needle) + if type(list) ~= 'table' then return list == needle end + for i = 1, #list do if list[i] == needle then return true end end + return false +end +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.01)) + end + if pred() then return true end + fail(label or 'condition was not satisfied before timeout') +end + +local function mkdir_p(path) + local ok = os.execute("mkdir -p '" .. path .. "'") + if ok ~= true and ok ~= 0 then error('mkdir failed for ' .. path) end +end +local function sorted_keys(t) + local out = {} + for k in pairs(t or {}) do out[#out + 1] = k end + table.sort(out) + return out +end + +local tmp = '/tmp/dc-network-provider-fw4-schema' +os.execute("rm -rf '" .. tmp .. "'") +local conf = tmp .. '/conf' +local save = tmp .. '/save' +mkdir_p(conf) +mkdir_p(save) +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do + local f = assert(io.open(conf .. '/' .. pkg, 'w')) + f:write('# devicecode OpenWrt fw4 schema test\n') + f:close() +end + +local restarts = {} + +-- Mirrors the generated firewall shape that caused live fw4 warnings: defaults, +-- zones, rules and forwardings are generated by the OpenWrt provider. +-- The provider must not write Devicecode-private metadata into firewall sections, +-- and the product-level intent must not contain unsupported fw4 defaults. +local intent = { + schema = 'devicecode.net.intent/1', + rev = 20260521, + generation = 1, + segments = { + lan = { + kind = 'lan', + addressing = { ipv4 = { mode = 'static', cidr = '192.168.10.1/24' } }, + dhcp = { enabled = true, start = 100, limit = 150, leasetime = '12h' }, + firewall = { zone = 'lan' }, + }, + lan_rst = { + kind = 'lan', + addressing = { ipv4 = { mode = 'static', cidr = '192.168.20.1/24' } }, + dhcp = { enabled = true, start = 100, limit = 150, leasetime = '12h' }, + firewall = { zone = 'lan_rst' }, + }, + wan = { + kind = 'wan', + firewall = { zone = 'wan' }, + }, + }, + interfaces = { + lan = { + kind = 'bridge', + role = 'lan', + segment = 'lan', + members = { 'eth0' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.10.1/24' } }, + }, + lan_rst = { + kind = 'bridge', + role = 'lan', + segment = 'lan_rst', + members = { 'eth1' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.20.1/24' } }, + }, + wan = { + kind = 'ethernet', + role = 'wan', + segment = 'wan', + endpoint = { ifname = 'eth2' }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, + }, + }, + dns = { + enabled = true, + domain = 'bigbox.test', + upstreams = { '1.1.1.1' }, + cache = { size = 512 }, + }, + dhcp = { defaults = { authoritative = true } }, + firewall = { + defaults = { + syn_flood = '1', + input = 'ACCEPT', + output = 'ACCEPT', + forward = 'REJECT', + }, + zones = { + lan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + lan_rst = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + wan = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT', masq = true, mtu_fix = true }, + }, + policies = { + lan_to_wan_1 = { from = 'lan', to = 'wan' }, + lan_rst_to_wan_1 = { from = 'lan_rst', to = 'wan' }, + }, + rules = { + Allow_DHCP_Renew = { + name = 'Allow-DHCP-Renew', family = 'ipv4', src = 'wan', proto = 'udp', dest_port = '68', target = 'ACCEPT', + }, + Allow_IGMP = { + name = 'Allow-IGMP', family = 'ipv4', src = 'wan', proto = 'igmp', target = 'ACCEPT', + }, + Allow_IPSec_ESP = { + name = 'Allow-IPSec-ESP', src = 'wan', proto = 'esp', target = 'ACCEPT', + }, + Allow_IPSec_ESP_RST = { + name = 'Allow-IPSec-ESP (RST)', src = 'lan_rst', dest = 'wan', proto = 'esp', target = 'ACCEPT', + }, + Allow_ISAKMP = { + name = 'Allow-ISAKMP', src = 'wan', proto = 'udp', dest_port = '500', target = 'ACCEPT', + }, + Allow_ISAKMP_RST = { + name = 'Allow-ISAKMP (RST)', src = 'lan_rst', dest = 'wan', proto = 'udp', dest_port = '500', target = 'ACCEPT', + }, + Allow_Ping = { + name = 'Allow-Ping', src = 'wan', proto = 'icmp', icmp_type = 'echo-request', family = 'ipv4', target = 'ACCEPT', + }, + Allow_DHCP_request_RST = { + name = 'Allow DHCP request (RST)', src = 'lan_rst', proto = 'udp', dest_port = '67', target = 'ACCEPT', + }, + Allow_DNS_queries_RST = { + name = 'Allow DNS queries (RST)', src = 'lan_rst', proto = { 'tcp', 'udp' }, dest_port = '53', target = 'ACCEPT', + }, + }, + }, + routing = {}, + wan = { enabled = true, members = { wan = { interface = 'wan', mwan_metric = 1, weight = 1 } } }, + + vpn = {}, + diagnostics = {}, +} + +fibers.run(function(_scope) + local provider, perr = provider_loader.new({ + provider = 'openwrt', + confdir = conf, + savedir = save, + debounce_s = 0.01, + run_cmd = function(argv) + restarts[#restarts + 1] = table.concat(argv, ' ') + return true, nil + end, + }, {}) + assert(provider, perr) + + local valid = perform(provider:validate_op({ intent = intent })) + assert(valid and valid.ok == true, 'validate failed: ' .. tostring(valid and valid.err)) + + local plan = perform(provider:plan_op({ intent = intent })) + assert(plan and plan.ok == true, 'plan failed: ' .. tostring(plan and plan.err)) + assert(plan.plan.packages.firewall.changes > 0, 'firewall plan should have changes') + + local result = perform(provider:apply_op({ intent = intent })) + assert(result and result.ok == true, 'apply failed: ' .. tostring(result and result.err)) + assert(result.activation == nil, 'provider activation should be synchronous for structural network apply') + eq(#restarts, 4, 'activation command count') + provider:terminate('test complete') +end) + +eq(restarts[#restarts], '/etc/init.d/mwan3 restart', 'last restart command') + +local c = assert(uci.cursor(conf, save)) +if type(c.load) == 'function' then pcall(function() c:load('firewall') end) end + +eq(c:get('firewall', 'defaults'), 'defaults', 'firewall.defaults type') +eq(c:get('firewall', 'zone_lan'), 'zone', 'firewall.zone_lan type') +eq(c:get('firewall', 'zone_lan_rst'), 'zone', 'firewall.zone_lan_rst type') +eq(c:get('firewall', 'zone_wan'), 'zone', 'firewall.zone_wan type') +eq(c:get('firewall', 'rule_Allow_DHCP_Renew'), 'rule', 'DHCP renew rule type') +eq(c:get('firewall', 'rule_Allow_DNS_queries_RST'), 'rule', 'RST DNS rule type') +record(contains(c:get('firewall', 'zone_lan', 'network'), 'lan'), 'zone_lan must contain lan network') +record(contains(c:get('firewall', 'zone_lan_rst', 'network'), 'lan_rst'), 'zone_lan_rst must contain lan_rst network') + +-- These assertions encode the schema boundary: Devicecode-private ownership +-- markers must not be written into OpenWrt's firewall4-owned schema. +local sections = {} +c:foreach('firewall', nil, function(s) + if type(s) == 'table' and type(s['.name']) == 'string' then + sections[#sections + 1] = s['.name'] + end +end) +table.sort(sections) +for _, section in ipairs(sections) do + absent(c:get('firewall', section, 'devicecode_managed'), 'firewall.' .. section .. '.devicecode_managed') + absent(c:get('firewall', section, 'devicecode_owner'), 'firewall.' .. section .. '.devicecode_owner') + absent(c:get('firewall', section, 'devicecode_semantic_id'), 'firewall.' .. section .. '.devicecode_semantic_id') + absent(c:get('firewall', section, 'devicecode_role'), 'firewall.' .. section .. '.devicecode_role') +end + +-- The higher-level product config no longer models disable_ipv6 as a firewall +-- default. This test therefore does not inject it; fw4 is the schema authority. +absent(c:get('firewall', 'defaults', 'disable_ipv6'), 'firewall.defaults.disable_ipv6') + +print('FIREWALL_CONF=' .. conf .. '/firewall') +print('OPENWRT_PROVIDER_FW4_SCHEMA_UCI_CHECKS=' .. (#failures == 0 and 'ok' or 'failed')) +if #failures > 0 then + for _, failure in ipairs(failures) do + io.stderr:write('SCHEMA_VIOLATION: ', failure, '\n') + end + os.exit(1) +end +print('openwrt network provider fw4 schema UCI checks: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_network_provider_fw4_schema.lua" "$REMOTE/run_openwrt_network_provider_fw4_schema.lua" + +set +e +LUA_OUTPUT="$("$SSH" "cd '$REMOTE' && lua ./run_openwrt_network_provider_fw4_schema.lua" 2>&1)" +LUA_RC=$? +set -e +printf '%s\n' "$LUA_OUTPUT" + +FW_FILE="$(printf '%s\n' "$LUA_OUTPUT" | sed -n 's/^FIREWALL_CONF=//p' | tail -n 1)" +if [ -z "$FW_FILE" ]; then + echo 'failed to locate generated firewall config path in Lua output' >&2 + exit 1 +fi + +set +e +FW4_OUTPUT="$("$SSH" "set -eu +fw_file='$FW_FILE' +out=/tmp/devicecode-fw4-schema-check.out +backup=/tmp/devicecode-fw4-schema-firewall.backup +cp /etc/config/firewall \"\$backup\" +restore() { cp \"\$backup\" /etc/config/firewall; rm -f \"\$backup\"; } +trap restore EXIT INT TERM +cp \"\$fw_file\" /etc/config/firewall +set +e +fw4 check >\"\$out\" 2>&1 +rc=\$? +set -e +cat \"\$out\" +if grep -E \"unknown option|not supported by fw4|specifies unknown option\" \"\$out\" >/dev/null 2>&1; then + echo 'fw4 schema warnings were emitted for generated firewall config' >&2 + exit 1 +fi +exit \"\$rc\"")" +FW4_RC=$? +set -e +printf '%s\n' "$FW4_OUTPUT" + +if [ "$LUA_RC" -ne 0 ]; then + echo "Lua-side generated UCI schema checks failed with rc=$LUA_RC" >&2 + exit "$LUA_RC" +fi +if [ "$FW4_RC" -ne 0 ]; then + echo "fw4 schema check failed with rc=$FW4_RC" >&2 + exit "$FW4_RC" +fi + +echo 'openwrt network provider fw4 schema: ok' diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_live_snapshot.sh b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_live_snapshot.sh new file mode 100755 index 00000000..fed06f50 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_live_snapshot.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-network-provider-live-snapshot-test" +WORK="$VM_DIR/work/network-provider-live-snapshot-test" + +mkdir -p "$WORK" + +cat > "$WORK/run_openwrt_network_provider_live_snapshot.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', + './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', + './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', + './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', + './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local provider_loader = require 'services.hal.backends.network.provider' + +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function ok(v, msg) if not v then fail(msg or 'assertion failed') end return v end +local function eq(a, b, msg) if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end + +fibers.run(function(_scope) + local provider, perr = provider_loader.new({ + provider = 'openwrt', + enable_live_snapshot = true, + enable_hotplug_socket = false, + enable_ubus_listener = false, + }, {}) + assert(provider, perr) + + local snapshot = perform(provider:snapshot_op({ live = true, interfaces = { 'loopback', 'lan' } })) + assert(snapshot and snapshot.ok == true, 'snapshot failed: ' .. tostring(snapshot and snapshot.err)) + eq(snapshot.backend, 'openwrt', 'snapshot backend') + + local observed = ok(snapshot.observed, 'observed snapshot expected') + eq(observed.schema, 'devicecode.net.observed/1', 'observed schema') + ok(type(observed.live) == 'table', 'live snapshot expected') + ok(type(observed.live.interfaces) == 'table', 'live interfaces expected') + ok(type(observed.live.routes) == 'table', 'live routes expected') + ok(type(observed.live.devices) == 'table', 'live devices expected') + ok(type(observed.multiwan) == 'table', 'multiwan observation expected') + + local have_iface = observed.live.interfaces.loopback ~= nil or observed.live.interfaces.lan ~= nil + ok(have_iface, 'expected live status for loopback or lan') + if observed.live.interfaces.loopback then + eq(observed.live.interfaces.loopback.id, 'loopback', 'loopback id') + ok(observed.live.interfaces.loopback.raw ~= nil, 'loopback raw status expected') + end + + if observed.multiwan.available then + ok(type(observed.multiwan.interfaces) == 'table', 'mwan interfaces expected') + ok(type(observed.multiwan.policies) == 'table', 'mwan policies expected') + else + ok(type(observed.multiwan.err) == 'string', 'mwan unavailable should carry an error') + end + + provider:terminate('test complete') +end) + +print('openwrt network provider live snapshot: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_network_provider_live_snapshot.lua" "$REMOTE/run_openwrt_network_provider_live_snapshot.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_network_provider_live_snapshot.lua" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_mwan_live_weights.sh b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_mwan_live_weights.sh new file mode 100755 index 00000000..39da50ea --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_mwan_live_weights.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-mwan-live-test" +WORK="$VM_DIR/work/mwan-live-test" + +mkdir -p "$WORK" +cat > "$WORK/run_openwrt_mwan_live_weights.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local exec = require 'fibers.io.exec' +local provider_loader = require 'services.hal.backends.network.provider' +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function contains(s, needle, msg) + if type(s) ~= 'string' or not s:find(needle, 1, true) then + fail((msg or ('missing text: ' .. tostring(needle))) .. '\n--- text ---\n' .. tostring(s)) + end +end +local function not_contains(s, needle, msg) + if type(s) == 'string' and s:find(needle, 1, true) then + fail(msg or ('unexpected text: ' .. tostring(needle))) + end +end + +local function assert_policy_probability(rules, iface, weight, remaining, expected, msg) + local comment = string.format('\"%s %d %d\"', iface, weight, remaining) + for line in tostring(rules or ''):gmatch('[^\r\n]+') do + if line:find('mwan3_policy_balanced', 1, true) and line:find(comment, 1, true) then + local prob = tonumber(line:match('%-%-probability%s+([%d%.]+)')) + if not prob then + fail((msg or 'missing probability') .. '\n--- line ---\n' .. line) + end + if math.abs(prob - expected) > 0.000001 then + fail((msg or 'unexpected probability') .. ': got ' .. tostring(prob) .. ' expected ' .. tostring(expected) .. '\n--- line ---\n' .. line) + end + return + end + end + fail((msg or 'missing policy rule') .. '\n--- rules ---\n' .. tostring(rules)) +end + +local function capture(...) + local cmd = exec.command(...) + local out, st, code, sig, err = perform(cmd:combined_output_op()) + if st == 'exited' and code == 0 then return out or '' end + fail('command failed: ' .. table.concat({ ... }, ' ') .. ' ' .. tostring(err or out or st)) +end + +fibers.run(function() + local before = capture('mwan3', 'status') + contains(before, 'balanced:', 'balanced policy before live update') + contains(before, 'wan (', 'wan present before live update') + contains(before, 'wanb (', 'wanb present before live update') + contains(before, 'wanc (', 'wanc present before live update') + + local provider = assert(provider_loader.new({ provider = 'openwrt', debounce_s = 0.01 }, {})) + local result = perform(provider:apply_live_weights_op({ + policy = 'balanced', + persist = false, + members = { + { interface = 'wan', metric = 1, weight = 50 }, + { interface = 'wanb', metric = 1, weight = 30 }, + { interface = 'wanc', metric = 1, weight = 20 }, + }, + })) + if result.ok ~= true then fail(result.err or 'live weights failed') end + + local status = capture('mwan3', 'status') + contains(status, 'balanced:', 'balanced policy after live update') + contains(status, 'wan (50%)', 'new flows should use wan 50% share') + contains(status, 'wanb (30%)', 'new flows should use wanb 30% share') + contains(status, 'wanc (20%)', 'new flows should use wanc 20% share') + + local rules = capture('iptables-save', '-t', 'mangle') + contains(rules, 'CONNMARK --restore-mark', 'existing connmarks should still be restored before policy') + contains(rules, 'CONNMARK --save-mark', 'connmarks should still be saved') + assert_policy_probability(rules, 'wan', 50, 100, 0.5, 'first conditional probability should be 50/100') + assert_policy_probability(rules, 'wanb', 30, 50, 0.6, 'second conditional probability should be 30/50') + contains(rules, '-A mwan3_policy_balanced -m mark --mark 0x0/0x3f00 -m comment --comment "wanc 20 20"', 'fall-through new-flow rule expected') + not_contains(rules, 'conntrack -F', 'live update must not flush conntrack') + + local restore = perform(provider:apply_live_weights_op({ + policy = 'balanced', + persist = false, + members = { + { interface = 'wan', metric = 1, weight = 1 }, + { interface = 'wanb', metric = 1, weight = 1 }, + { interface = 'wanc', metric = 1, weight = 1 }, + }, + })) + if restore.ok ~= true then fail(restore.err or 'restore to equal weights failed') end + provider:terminate('test complete') +end) + +print('openwrt network provider MWAN live weights: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_mwan_live_weights.lua" "$REMOTE/run_openwrt_mwan_live_weights.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_mwan_live_weights.lua" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_segment_trunk.sh b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_segment_trunk.sh new file mode 100755 index 00000000..000e6d3d --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_segment_trunk.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-network-provider-segment-trunk-test" +WORK="$VM_DIR/work/network-provider-segment-trunk-test" + +mkdir -p "$WORK" +cat > "$WORK/run_openwrt_network_provider_segment_trunk.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local uci = require 'uci' +local provider_loader = require 'services.hal.backends.network.provider' +local names_mod = require 'services.hal.backends.network.providers.openwrt.names' +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function eq(a, b, msg) + if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end +end +local function mkdir_p(path) + local ok = os.execute("mkdir -p '" .. path .. "'") + if ok ~= true and ok ~= 0 then fail('mkdir failed for ' .. path) end +end +local function assert_list_contains(list, value, label) + if type(list) ~= 'table' then fail((label or 'list') .. ' should be a table') end + for i = 1, #list do if list[i] == value then return true end end + fail((label or 'list') .. ' missing ' .. tostring(value)) +end + +local tmp = '/tmp/dc-network-provider-segment-trunk' +os.execute("rm -rf '" .. tmp .. "'") +local conf, save = tmp .. '/conf', tmp .. '/save' +mkdir_p(conf); mkdir_p(save) +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do + local f = assert(io.open(conf .. '/' .. pkg, 'w')) + f:write('# devicecode segment trunk test\n') + f:close() +end + +local restarts = {} +local shaper_cmds, shaper_batches = {}, {} +local provider = assert(provider_loader.new({ + provider = 'openwrt', + confdir = conf, + savedir = save, + debounce_s = 0.01, + platform = { segment_trunk = { ifname = 'eth0', protected = true } }, + run_cmd = function(argv) restarts[#restarts + 1] = table.concat(argv, ' '); return true, nil end, + shaper_run_cmd = function(argv) + local line = table.concat(argv, ' ') + shaper_cmds[#shaper_cmds + 1] = line + if argv[1] == 'tc' and argv[2] == '-batch' and type(argv[3]) == 'string' then + local f = io.open(argv[3], 'r') + if f then shaper_batches[#shaper_batches + 1] = f:read('*a') or ''; f:close() end + end + return true, nil + end, +}, {})) + +local intent = { + schema = 'devicecode.net.intent/1', + rev = 410, + generation = 4, + policies = { + vlan = { + reserved = { mgmt = 10, switch_control = 11, fabric = 12 }, + ranges = { service = { from = 100, to = 199 } }, + }, + }, + segments = { + mgmt = { + kind = 'system', protected = true, user_editable = false, + vlan = { id = 10, reserved = 'mgmt' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.8.1/24' } }, + firewall = { zone = 'mgmt' }, + dhcp = { enabled = false }, + }, + switch_control = { + kind = 'system', protected = true, user_editable = false, + vlan = { id = 11, reserved = 'switch_control' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.11.1/24' } }, + firewall = { zone = 'system' }, + dhcp = { enabled = false }, + }, + fabric = { + kind = 'system', protected = true, user_editable = false, + vlan = { id = 12, reserved = 'fabric' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.12.1/24' } }, + firewall = { zone = 'system' }, + dhcp = { enabled = false }, + }, + lan = { + kind = 'user', vlan = { id = 100 }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.100.1/24' } }, + dhcp = { enabled = true, start = 20, limit = 50, leasetime = '12h' }, + firewall = { zone = 'lan' }, + shaping = { download = { limit = '10mbit' }, upload = { limit = '10mbit' }, host_default = { mode = 'budgeted_peak', all_hosts = true, download = { sustained_rate = '2mbit', peak_rate = '4mbit', burst_budget = '100k' }, upload = { sustained_rate = '2mbit', peak_rate = '4mbit', burst_budget = '100k' } } }, + }, + guest = { + kind = 'guest', vlan = { id = 101 }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.101.1/24' } }, + dhcp = { enabled = true, start = 50, limit = 100, leasetime = '6h' }, + firewall = { zone = 'guest' }, + }, + iot = { + kind = 'user', l2 = { mode = 'direct' }, vlan = { id = 102 }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.102.1/24' } }, + dhcp = { enabled = false }, + firewall = { zone = 'lan' }, + shaping = { download = { limit = '10mbit' }, upload = { limit = '10mbit' }, host_default = { mode = 'budgeted_peak', all_hosts = true, download = { sustained_rate = '2mbit', peak_rate = '4mbit', burst_budget = '100k' }, upload = { sustained_rate = '2mbit', peak_rate = '4mbit', burst_budget = '100k' } } }, + }, + }, + interfaces = {}, + dns = { enabled = true, upstreams = { '1.1.1.1' } }, + dhcp = {}, + firewall = { + zones = { + mgmt = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + system = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + lan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + guest = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT' }, + }, + }, + routing = {}, wan = {}, vpn = {}, diagnostics = {}, +} + +local name_ctx = assert(names_mod.allocate(intent)) + +fibers.run(function() + local valid = perform(provider:validate_op({ intent = intent })) + assert(valid and valid.ok == true, 'validate failed: ' .. tostring(valid and valid.err)) + local plan = perform(provider:plan_op({ intent = intent })) + assert(plan and plan.ok == true, 'plan failed: ' .. tostring(plan and plan.err)) + assert(plan.plan.packages.network.sections >= 10, 'segment trunk should create network sections') + local result = perform(provider:apply_op({ intent = intent })) + assert(result and result.ok == true, 'apply failed: ' .. tostring(result and result.err)) + provider:terminate('test complete') +end) + +if #shaper_cmds == 0 then fail('shaper commands expected for shaped trunk segment') end +local shaper_text = table.concat(shaper_cmds, '\n') .. '\n' .. table.concat(shaper_batches, '\n') +if not shaper_text:find('br%-lan') then fail('bridged trunk segment shaping should target bridge data device br-lan, got: ' .. shaper_text) end +if shaper_text:find('dev vl%-lan') then fail('bridged trunk segment shaping must not target VLAN member device vl-lan') end +if shaper_text:find('eth0%.100') then fail('trunk segment shaping must not target legacy eth0.100') end +if not shaper_text:find('ifb_br_lan') then fail('bridged trunk segment ingress IFB should be derived from br-lan') end +if not shaper_text:find('vl%-iot') then fail('direct trunk segment shaping should target VLAN data device vl-iot, got: ' .. shaper_text) end +if not shaper_text:find('ifb_vl_iot') then fail('direct trunk segment ingress IFB should be derived from vl-iot') end + +local c = assert(uci.cursor(conf, save)) +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall' }) do if type(c.load) == 'function' then pcall(function() c:load(pkg) end) end end + +local function all(pkg) + if type(c.load) == 'function' then pcall(function() c:load(pkg) end) end + return c:get_all(pkg) or {} +end +local function section_by_name(pkg, section, stype) + local sec = all(pkg)[section] + if type(sec) ~= 'table' then fail('section not found: ' .. pkg .. '.' .. tostring(section)) end + if stype ~= nil and sec['.type'] ~= stype then fail(pkg .. '.' .. tostring(section) .. ' expected type ' .. tostring(stype) .. ', got ' .. tostring(sec['.type'])) end + return section, sec +end + +local function find_firewall_zone(zone_name) + for name, sec in pairs(all('firewall')) do + if type(sec) == 'table' and sec['.type'] == 'zone' and sec.name == zone_name then return name, sec end + end + fail('firewall zone not found: ' .. tostring(zone_name)) +end +local function contains(list, value) + if type(list) == 'table' then for i = 1, #list do if list[i] == value then return true end end end + return list == value +end + +local expected = { + mgmt = { vid = '10', cidr_ip = '192.168.8.1', netmask = '255.255.255.0' }, + switch_control = { vid = '11', cidr_ip = '192.168.11.1', netmask = '255.255.255.0' }, + fabric = { vid = '12', cidr_ip = '192.168.12.1', netmask = '255.255.255.0' }, + lan = { vid = '100', cidr_ip = '192.168.100.1', netmask = '255.255.255.0' }, + guest = { vid = '101', cidr_ip = '192.168.101.1', netmask = '255.255.255.0' }, +} + +local iface_by_seg = {} +for seg, e in pairs(expected) do + local _vlan_sec, vlan = section_by_name('network', name_ctx:section('dev_vlan', seg), 'device') + eq(vlan.type, '8021q', seg .. ' vlan device type') + eq(vlan.ifname, 'eth0', seg .. ' vlan ifname') + eq(vlan.vid, e.vid, seg .. ' vid') + assert(#vlan.name <= 14, seg .. ' vlan name length') + local _br_sec, br = section_by_name('network', name_ctx:section('dev_bridge', seg), 'device') + eq(br.type, 'bridge', seg .. ' bridge device type') + assert_list_contains(br.ports, vlan.name, seg .. ' bridge ports') + assert(#br.name <= 14, seg .. ' bridge name length') + local ifsec, iface = section_by_name('network', name_ctx:iface(seg), 'interface') + iface_by_seg[seg] = ifsec + eq(iface.device, br.name, seg .. ' interface bridge device') + eq(iface.ipaddr, e.cidr_ip, seg .. ' ipaddr') + eq(iface.netmask, e.netmask, seg .. ' netmask') + assert(#ifsec <= 8, seg .. ' logical interface length') +end + +local _iot_vlan_sec, iot_vlan = section_by_name('network', name_ctx:section('dev_vlan', 'iot'), 'device') +eq(iot_vlan.type, '8021q', 'iot direct vlan device type') +eq(iot_vlan.ifname, 'eth0', 'iot direct vlan ifname') +eq(iot_vlan.vid, '102', 'iot direct vid') +local _iot_ifsec, iot_iface = section_by_name('network', name_ctx:iface('iot'), 'interface') +eq(iot_iface.device, iot_vlan.name, 'direct segment interface should use VLAN device') +eq(iot_iface.ipaddr, '192.168.102.1', 'iot direct ipaddr') +eq(iot_iface.netmask, '255.255.255.0', 'iot direct netmask') +if all('network')[name_ctx:section('dev_bridge', 'iot')] ~= nil then fail('direct segment should not create bridge device') end + +local _dhcp_lan_sec, dhcp_lan = section_by_name('dhcp', name_ctx:section('dhcp', 'lan'), 'dhcp') +local _dhcp_guest_sec, dhcp_guest = section_by_name('dhcp', name_ctx:section('dhcp', 'guest'), 'dhcp') +local _dhcp_mgmt_sec, dhcp_mgmt = section_by_name('dhcp', name_ctx:section('dhcp', 'mgmt'), 'dhcp') +eq(dhcp_lan.interface, iface_by_seg.lan, 'lan dhcp interface') +eq(dhcp_guest.interface, iface_by_seg.guest, 'guest dhcp interface') +eq(dhcp_mgmt.ignore, '1', 'mgmt dhcp ignored') + +local _zone_lan_sec, zone_lan = find_firewall_zone('lan') +local _zone_guest_sec, zone_guest = find_firewall_zone('guest') +local _zone_system_sec, zone_system = find_firewall_zone('system') +assert_list_contains(zone_lan.network, iface_by_seg.lan, 'lan zone network') +assert_list_contains(zone_guest.network, iface_by_seg.guest, 'guest zone network') +assert_list_contains(zone_system.network, iface_by_seg.switch_control, 'system zone network') +assert_list_contains(zone_system.network, iface_by_seg.fabric, 'system zone network') + +print('openwrt network provider segment trunk: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_network_provider_segment_trunk.lua" "$REMOTE/run_openwrt_network_provider_segment_trunk.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_network_provider_segment_trunk.lua" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_snapshot.sh b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_snapshot.sh new file mode 100755 index 00000000..a35dd32b --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_snapshot.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-network-provider-snapshot-test" +WORK="$VM_DIR/work/network-provider-snapshot-test" + +mkdir -p "$WORK" + +cat > "$WORK/run_openwrt_network_provider_snapshot.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', + './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', + './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', + './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', + './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local bus = require 'bus' +local trie = require 'trie' +local provider_loader = require 'services.hal.backends.network.provider' + +assert(type(bus.new) == 'function', 'vendored bus did not load') +assert(type(trie.new_pubsub) == 'function', 'vendored trie did not load') + +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function eq(a, b, msg) + if a ~= b then + fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) + end +end + +local function assert_list(v, expected, label) + if type(v) ~= 'table' then fail((label or 'list') .. ' should be a table, got ' .. type(v)) end + eq(#v, #expected, (label or 'list') .. ' length') + for i = 1, #expected do eq(v[i], expected[i], (label or 'list') .. '[' .. i .. ']') end +end + +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.01)) + end + if pred() then return true end + fail(label or 'condition was not satisfied before timeout') +end + +local function mkdir_p(path) + local ok = os.execute("mkdir -p '" .. path .. "'") + if ok ~= true and ok ~= 0 then fail('mkdir failed for ' .. path) end +end + +local tmp = '/tmp/dc-network-provider-snapshot' +os.execute("rm -rf '" .. tmp .. "'") +local conf = tmp .. '/conf' +local save = tmp .. '/save' +mkdir_p(conf) +mkdir_p(save) +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do + local f = assert(io.open(conf .. '/' .. pkg, 'w')) + f:write('# devicecode OpenWrt network provider snapshot test\n') + f:close() +end + +local restarts = {} + +local intent = { + schema = 'devicecode.net.intent/1', + rev = 202, + generation = 4, + segments = { + lan = { + kind = 'lan', + vlan = { id = 10 }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.10.1/24' } }, + dhcp = { enabled = true, start = 20, limit = 50, leasetime = '6h' }, + firewall = { zone = 'lan' }, + }, + wan = { + kind = 'wan', + firewall = { zone = 'wan' }, + }, + }, + interfaces = { + lan = { + kind = 'bridge', + role = 'lan', + segment = 'lan', + members = { 'eth0', 'eth1' }, + addressing = { ipv4 = { mode = 'static', cidr = '192.168.10.1/24' } }, + }, + wan = { + kind = 'ethernet', + role = 'wan', + segment = 'wan', + endpoint = { ifname = 'eth2' }, + addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, + }, + }, + dns = { + enabled = true, + upstreams = { '1.1.1.1', '8.8.8.8' }, + }, + dhcp = {}, + firewall = { + defaults = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT' }, + zones = { + lan = { input = 'ACCEPT', output = 'ACCEPT', forward = 'REJECT' }, + wan = { input = 'REJECT', output = 'ACCEPT', forward = 'REJECT', masq = true, mtu_fix = true }, + }, + policies = { + lan_to_wan = { src = 'lan', dest = 'wan' }, + }, + }, + routing = { + routes = { + { interface = 'lan', target = '10.0.0.0/8', gateway = '192.168.10.254' }, + }, + }, + wan = { enabled = true, members = { wan = { interface = 'wan', mwan_metric = 1, weight = 1 } } }, + + vpn = {}, + diagnostics = {}, +} + +fibers.run(function(_scope) + local provider, perr = provider_loader.new({ + provider = 'openwrt', + confdir = conf, + savedir = save, + debounce_s = 0.01, + run_cmd = function(argv) + restarts[#restarts + 1] = table.concat(argv, ' ') + return true, nil + end, + }, {}) + assert(provider, perr) + + local result = perform(provider:apply_op({ intent = intent })) + assert(result and result.ok == true, 'apply failed: ' .. tostring(result and result.err)) + assert(result.activation == nil, 'provider activation should be synchronous for structural network apply') + + local snapshot = perform(provider:snapshot_op({ live = false })) + assert(snapshot and snapshot.ok == true, 'snapshot failed: ' .. tostring(snapshot and snapshot.err)) + eq(snapshot.backend, 'openwrt', 'snapshot backend') + assert(type(snapshot.packages) == 'table', 'snapshot should retain raw packages for diagnostics') + assert(snapshot.observed.live == nil, 'non-live snapshot should not query ubus') + + local observed = snapshot.observed + assert(type(observed) == 'table', 'snapshot.observed should be a table') + eq(observed.schema, 'devicecode.net.observed/1', 'observed schema') + + eq(observed.interfaces.lan.kind, 'bridge', 'lan kind') + eq(observed.interfaces.lan.segment, 'lan', 'lan segment') + assert_list(observed.interfaces.lan.members, { 'eth0', 'eth1' }, 'lan bridge members') + eq(observed.interfaces.lan.addressing.ipv4.mode, 'static', 'lan ipv4 mode') + eq(observed.interfaces.lan.addressing.ipv4.cidr, '192.168.10.1/24', 'lan cidr') + + eq(observed.interfaces.wan.kind, 'ethernet', 'wan kind') + eq(observed.interfaces.wan.segment, 'wan', 'wan segment') + eq(observed.interfaces.wan.endpoint.ifname, 'eth2', 'wan endpoint') + eq(observed.interfaces.wan.addressing.ipv4.mode, 'dhcp', 'wan ipv4 mode') + eq(observed.interfaces.wan.addressing.ipv4.peerdns, false, 'wan peerdns') + eq(observed.interfaces.wan.addressing.ipv4.metric, 11, 'wan auto route metric') + + eq(observed.segments.lan.firewall.zone, 'lan', 'lan segment zone') + eq(observed.segments.lan.dhcp.enabled, true, 'lan segment dhcp enabled') + eq(observed.segments.lan.dhcp.start, 20, 'lan dhcp start') + eq(observed.segments.lan.dhcp.limit, 50, 'lan dhcp limit') + eq(observed.segments.lan.dhcp.leasetime, '6h', 'lan dhcp leasetime') + assert_list(observed.segments.lan.interfaces, { 'lan' }, 'lan segment interfaces') + + assert_list(observed.dns.upstreams, { '1.1.1.1', '8.8.8.8' }, 'dns upstreams') + + eq(observed.firewall.defaults.input, 'REJECT', 'firewall default input') + eq(observed.firewall.defaults.output, 'ACCEPT', 'firewall default output') + eq(observed.firewall.defaults.forward, 'REJECT', 'firewall default forward') + assert_list(observed.firewall.zones.lan.networks, { 'lan' }, 'lan zone networks') + assert_list(observed.firewall.zones.wan.networks, { 'wan' }, 'wan zone networks') + eq(observed.firewall.zones.wan.masq, true, 'wan zone masq') + eq(observed.firewall.zones.wan.mtu_fix, true, 'wan zone mtu_fix') + eq(observed.firewall.policies.fwd_lan_to_wan_1.src, 'lan', 'forwarding src') + eq(observed.firewall.policies.fwd_lan_to_wan_1.dest, 'wan', 'forwarding dest') + + eq(#observed.routing.routes, 1, 'route count') + eq(observed.routing.routes[1].interface, 'lan', 'route interface') + eq(observed.routing.routes[1].target, '10.0.0.0/8', 'route target') + eq(observed.routing.routes[1].gateway, '192.168.10.254', 'route gateway') + + wait_until(function() return #restarts == 4 end, 1, 'activation commands should run') + provider:terminate('test complete') +end) + +eq(#restarts, 4, 'restart command count') + +print('openwrt network provider snapshot: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_network_provider_snapshot.lua" "$REMOTE/run_openwrt_network_provider_snapshot.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_network_provider_snapshot.lua" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_vlan_mwan_shaping.sh b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_vlan_mwan_shaping.sh new file mode 100755 index 00000000..c5a24784 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_network_provider_vlan_mwan_shaping.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-network-provider-advanced-test" +WORK="$VM_DIR/work/network-provider-advanced-test" + +mkdir -p "$WORK" +cat > "$WORK/run_openwrt_network_provider_advanced.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local uci = require 'uci' +local provider_loader = require 'services.hal.backends.network.provider' +local perform = fibers.perform + +local function fail(msg) error(msg, 2) end +local function eq(a,b,msg) if a ~= b then fail((msg or 'values differ') .. ': expected '..tostring(b)..', got '..tostring(a)) end end +local function wait_until(pred, timeout_s, label) + local deadline = fibers.now() + (timeout_s or 1) + while fibers.now() < deadline do + if pred() then return true end + perform(sleep.sleep_op(0.01)) + end + if pred() then return true end + fail(label or 'condition was not satisfied before timeout') +end + +local function mkdir_p(path) local ok = os.execute("mkdir -p '"..path.."'"); if ok ~= true and ok ~= 0 then fail('mkdir failed') end end + +local tmp = '/tmp/dc-network-provider-advanced' +os.execute("rm -rf '"..tmp.."'") +local conf, save = tmp..'/conf', tmp..'/save' +mkdir_p(conf); mkdir_p(save) +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do local f=assert(io.open(conf..'/'..pkg,'w')); f:write('# test\n'); f:close() end + +local shaper_cmds, restarts, live_restores = {}, {}, {} +local provider = assert(provider_loader.new({ + provider = 'openwrt', confdir = conf, savedir = save, debounce_s = 0.01, platform = { segment_trunk = { ifname = 'eth0' } }, + run_cmd = function(argv) restarts[#restarts+1] = table.concat(argv, ' '); return true, nil end, + shaper_run_cmd = function(argv) shaper_cmds[#shaper_cmds+1] = table.concat(argv, ' '); return true, nil end, + speedtest_run_cmd = function(argv) return true, '55', nil end, + mwan_run_cmd_capture = function(argv) + if table.concat(argv, ' ') ~= 'iptables-save -t mangle' then return nil, '', 'unexpected capture command' end + return true, [[ +*mangle +:mwan3_iface_in_wan_a - [0:0] +:mwan3_iface_in_wan_b - [0:0] +:mwan3_policy_balanced - [0:0] +-A mwan3_iface_in_wan_a -i wwan0 -m mark --mark 0x0/0x3f00 -m comment --comment wan_a -j MARK --set-xmark 0x100/0x3f00 +-A mwan3_iface_in_wan_b -i wwan1 -m mark --mark 0x0/0x3f00 -m comment --comment wan_b -j MARK --set-xmark 0x300/0x3f00 +-A mwan3_policy_balanced -m mark --mark 0x0/0x3f00 -m statistic --mode random --probability 0.50000000000 -m comment --comment "wan_b 1 2" -j MARK --set-xmark 0x300/0x3f00 +-A mwan3_policy_balanced -m mark --mark 0x0/0x3f00 -m comment --comment "wan_a 1 1" -j MARK --set-xmark 0x100/0x3f00 +COMMIT +]], nil + end, + mwan_run_restore = function(content) live_restores[#live_restores+1] = content; return true, nil end, +}, {})) + +local intent = { + schema = 'devicecode.net.intent/1', rev = 303, + segments = { + lan = { kind='lan', vlan={id=10}, addressing={ipv4={mode='static', cidr='192.168.10.1/24'}}, dhcp={enabled=true}, firewall={zone='lan'} }, + wan = { kind='wan', firewall={zone='wan'} }, + }, + interfaces = { + lan = { kind='bridge', role='lan', segment='lan', members={'eth0'}, addressing={ipv4={mode='static', cidr='192.168.10.1/24'}} }, + wan_a = { kind='cellular', role='wan', segment='wan', endpoint={ifname='wwan0'}, addressing={ipv4={mode='dhcp'}} }, + wan_b = { kind='cellular', role='wan', segment='wan', endpoint={ifname='wwan1'}, addressing={ipv4={mode='dhcp'}} }, + }, + firewall = { + defaults={ input='ACCEPT', output='ACCEPT', forward='REJECT', synflood_protect=true }, + zones={lan={}, wan={masq=true, mtu_fix=true}}, + policies={lan_to_wan={from='lan', to='wan'}}, + rules={ allow_dns={ name='Allow DNS', src='lan', proto='tcp udp', dest_port='53', target='ACCEPT' } }, + }, + routing={ routes={ default_lab={ target='192.168.99.1', interface='lan', gateway='192.168.10.254' } } }, + dns={ + domain='bigbox.home', + upstreams={'1.1.1.1','8.8.8.8'}, + cache={size=1000}, + host_files={ base_dir='/tmp/devicecode-dns-hosts', sources={ ads={file='ads.hosts'} } }, + records={ router={ name='config.bigbox.home', address='192.168.10.1' } }, + }, + dhcp={ defaults={ lease_time='12h', authoritative=true }, reservations={ unifi={ name='unifi', mac='00:11:22:33:44:55', ip='192.168.10.2' } } }, + vpn={}, diagnostics={}, + wan = { enabled=true, policy='weighted_failover', load_balancing={speedtests=true, policy='balanced'}, members={ gsm_a={interface='wan_a', mwan_metric=1, weight=1}, gsm_b={interface='wan_b', mwan_metric=1, weight=1} } }, +} +intent.segments.lan.dns = { local_server=true, domain='bigbox.home', host_files={'ads'} } +intent.segments.lan.shaping = { download={limit='10mbit'}, upload={limit='10mbit'}, host_default={ mode='budgeted_peak', all_hosts=true, download={sustained_rate='2mbit', peak_rate='4mbit', burst_budget='100k'}, upload={sustained_rate='2mbit', peak_rate='4mbit', burst_budget='100k'} } } + +fibers.run(function() + local plan = perform(provider:plan_op({ intent = intent })) + assert(plan.ok == true, plan.err) + eq(plan.plan.domains.vlan.status, 'implemented', 'vlan domain') + eq(plan.plan.domains.multiwan.status, 'implemented', 'mwan domain') + local result = perform(provider:apply_op({ intent = intent })) + assert(result.ok == true, result.err) + assert(result.activation == nil, 'provider activation should be synchronous for structural network apply') + eq(#restarts, 4, 'activation command count') + local speed = perform(provider:speedtest_op({ interface='wan_a', device='wwan0' })) + eq(speed.peak_mbps, 55, 'speedtest fake result') + local live = perform(provider:apply_live_weights_op({ policy='balanced', members={{id='gsm_a', interface='wan_a', metric=1, weight=80},{id='gsm_b', interface='wan_b', metric=1, weight=20}}, persist=true })) + assert(live.ok == true, live.err) + provider:terminate('test complete') +end) + +if #shaper_cmds == 0 then fail('shaper commands expected') end +if #live_restores ~= 1 then fail('one live weight restore expected') end +if not live_restores[1]:find('%-%-probability 0.80000000000') then fail('live first-member probability expected') end +if not live_restores[1]:find('%*mangle', 1, false) then fail('iptables-restore mangle payload expected') end +local joined = table.concat(restarts, '\n') +if not joined:find('mwan3 restart', 1, true) then fail('structural apply should restart mwan3') end + +local c = assert(uci.cursor(conf, save)) +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do if type(c.load) == 'function' then pcall(function() c:load(pkg) end) end end +if c:get('network', 'dev_lan_10') then + eq(c:get('network', 'dev_lan_10'), 'device', 'vlan device type') + eq(c:get('network', 'dev_lan_10', 'vid'), '10', 'vlan vid') +end +eq(c:get('network', 'route_default_lab'), 'route', 'map-shaped route') +local dns_sec, dns = nil, nil +for name, sec in pairs(c:get_all('dhcp') or {}) do + if type(sec) == 'table' and sec['.type'] == 'dnsmasq' then + local ah = sec.addnhosts + local has_ads = false + if type(ah) == 'table' then for i = 1, #ah do if ah[i] == '/tmp/devicecode-dns-hosts/ads.hosts' then has_ads = true end end else has_ads = (ah == '/tmp/devicecode-dns-hosts/ads.hosts') end + if has_ads then dns_sec, dns = name, sec; break end + end +end +if not dns then fail('per-segment dnsmasq for ads not found') end +eq(dns.cachesize, '1000', 'dns cache size') +local addnhosts = dns.addnhosts +if type(addnhosts) == 'table' then eq(addnhosts[1], '/tmp/devicecode-dns-hosts/ads.hosts', 'segment host file') else eq(addnhosts, '/tmp/devicecode-dns-hosts/ads.hosts', 'segment host file') end +local addresses = dns.address +local address_s = type(addresses) == 'table' and table.concat(addresses, ' ') or tostring(addresses) +if not address_s:find('/config.bigbox.home/192.168.10.1', 1, true) then fail('dns address record not applied: '..address_s) end +eq(c:get('dhcp', 'host_unifi'), 'host', 'dhcp reservation') +eq(c:get('firewall', 'rule_allow_dns'), 'rule', 'firewall rule') +eq(c:get('firewall', 'defaults', 'synflood_protect'), '1', 'firewall synflood default') +eq(c:get('mwan3', 'balanced'), 'policy', 'mwan balanced policy') +eq(c:get('mwan3', 'default_rule_v4'), 'rule', 'mwan default rule') +print('openwrt network provider VLAN/MWAN/shaping: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_network_provider_advanced.lua" "$REMOTE/run_openwrt_network_provider_advanced.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_network_provider_advanced.lua" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_segment_shaping_modes.sh b/tests/integration/openwrt_vm/tests/test_openwrt_segment_shaping_modes.sh new file mode 100755 index 00000000..a6eadccd --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_segment_shaping_modes.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +STAGE_DEVICECODE="$VM_DIR/scripts/ensure-devicecode-staged" +CLEANUP="$VM_DIR/scripts/cleanup-shaping-state" +REMOTE="/tmp/devicecode-segment-shaping-modes" +WORK="$VM_DIR/work/segment-shaping-modes" + +mkdir -p "$WORK" +"$CLEANUP" >/dev/null 2>&1 || true +trap 'set +e; "$CLEANUP" >/dev/null 2>&1 || true' EXIT INT TERM + +cat > "$WORK/run_segment_shaping_modes.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local exec = require 'fibers.io.exec' +local tc_u32 = require 'services.hal.backends.network.providers.openwrt.tc_u32_shaper' +local unpack = table.unpack or unpack +local perform = fibers.perform + +local function run(argv) + local cmd = exec.command(unpack(argv)) + local out, st, code = perform(cmd:combined_output_op()) + if st == 'exited' and code == 0 then return true, out or '', nil, code end + return nil, out or '', out or 'command failed', code +end + +local function apply_link(iface, mode, aggregate) + local egress = { + enabled = true, + mode = mode, + match = 'dst', + host_rate = '2mbit', host_ceil = '8mbit', host_burst = '100k', host_cburst = '100k', + fq_codel = { flows = 128, limit = 1024, memory_limit = '1Mb' }, + hosts = { ['172.29.32.36'] = { rate = '1mbit', ceil = '3mbit', burst = '50k', cburst = '50k' } }, + } + if aggregate then + egress.segment_aggregate = true + egress.pool_rate = '8mbit' + egress.pool_ceil = '8mbit' + end + local result = tc_u32.apply({ + links = { + test = { + iface = iface, + subnet = '172.29.32.1/24', + egress = egress, + ingress = { enabled = false }, + }, + }, + }, { run_cmd = run }) + assert(result and result.ok == true, tostring(result and result.err)) +end + +fibers.run(function() + apply_link('dcs-bflat', 'budgeted_peak', false) + apply_link('dcs-bagg', 'budgeted_peak', true) + apply_link('dcshape-borrow', 'borrow_to_ceil', true) +end) + +print('openwrt segment shaping modes apply: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'; ip link add dcs-bflat type veth peer name dcs-bflatp; ip link set dcs-bflat up; ip link set dcs-bflatp up; ip link add dcs-bagg type veth peer name dcs-baggp; ip link set dcs-bagg up; ip link set dcs-baggp up; ip link add dcshape-borrow type veth peer name dcshape-borrowp; ip link set dcshape-borrow up; ip link set dcshape-borrowp up" +CODE_REMOTE="$($STAGE_DEVICECODE)" +"$SCP_TO" "$WORK/run_segment_shaping_modes.lua" "$REMOTE/run.lua" +"$SSH" "cd '$CODE_REMOTE' && lua '$REMOTE/run.lua'" + +"$SSH" 'set -eu +fail() { echo "segment shaping modes test: $*" >&2; exit 1; } + +# budgeted_peak without aggregate: root -> host budget -> host peak -> fq_codel +tc class show dev dcs-bflat | grep -q "class htb 1:1036" || fail "flat budgeted_peak budget class should be 1:1036" +tc qdisc show dev dcs-bflat | grep -q "parent 1:1036" || fail "flat budgeted_peak should attach a peak HTB qdisc to 1:1036" +tc class show dev dcs-bflat | grep -q "class htb 1036:1" || fail "flat budgeted_peak peak class should be 1036:1" +tc qdisc show dev dcs-bflat | grep -q "parent 1036:1" || fail "flat budgeted_peak fq_codel should attach to 1036:1" +if tc class show dev dcs-bflat | grep -q "class htb 1:20"; then + fail "flat budgeted_peak should not create a segment aggregate class" +fi +if tc class show dev dcs-bflat | grep -q "class htb 20:1036"; then + fail "flat budgeted_peak should not create aggregate budget class 20:1036" +fi + +# budgeted_peak with aggregate: root -> segment aggregate -> host budget -> host peak -> fq_codel +tc class show dev dcs-bagg | grep -q "class htb 1:20" || fail "aggregate budgeted_peak should create segment aggregate 1:20" +tc qdisc show dev dcs-bagg | grep -Eq "qdisc htb 20: .*parent 1:20" || fail "aggregate budgeted_peak should attach inner HTB to 1:20" +tc class show dev dcs-bagg | grep -q "class htb 20:1036" || fail "aggregate budgeted_peak budget class should be 20:1036" +tc qdisc show dev dcs-bagg | grep -q "parent 20:1036" || fail "aggregate budgeted_peak should attach a peak HTB qdisc to 20:1036" +tc class show dev dcs-bagg | grep -q "class htb 1036:1" || fail "aggregate budgeted_peak peak class should be 1036:1" +tc qdisc show dev dcs-bagg | grep -q "parent 1036:1" || fail "aggregate budgeted_peak fq_codel should attach to 1036:1" + +# borrow_to_ceil keeps the existing inner-pool host leaf semantics. +tc class show dev dcshape-borrow | grep -q "class htb 20:1036" || fail "borrow_to_ceil host class should be 20:1036" +if tc class show dev dcshape-borrow | grep -q "class htb 1036:1"; then + fail "borrow_to_ceil should not create a budgeted peak class 1036:1" +fi +tc qdisc show dev dcshape-borrow | grep -q "parent 20:1036" || fail "borrow_to_ceil fq_codel should attach to 20:1036" + +printf "%s\n" "openwrt segment shaping modes topology: ok" +' + diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_shaping_idempotent_apply.sh b/tests/integration/openwrt_vm/tests/test_openwrt_shaping_idempotent_apply.sh new file mode 100755 index 00000000..b1722cd7 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_shaping_idempotent_apply.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +STAGE_DEVICECODE="$VM_DIR/scripts/ensure-devicecode-staged" +CLEANUP="$VM_DIR/scripts/cleanup-shaping-state" +REMOTE="/tmp/devicecode-shaping-idempotent-apply" +WORK="$VM_DIR/work/shaping-idempotent-apply" + +mkdir -p "$WORK" +"$CLEANUP" >/dev/null 2>&1 || true +trap 'set +e; "$CLEANUP" >/dev/null 2>&1 || true' EXIT INT TERM + +cat > "$WORK/run_shaping_idempotent_apply.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local exec = require 'fibers.io.exec' +local tc_u32 = require 'services.hal.backends.network.providers.openwrt.tc_u32_shaper' +local unpack = table.unpack or unpack +local perform = fibers.perform + +local function run(argv) + local cmd = exec.command(unpack(argv)) + local out, st, code = perform(cmd:combined_output_op()) + if st == 'exited' and code == 0 then return true, out or '', nil, code end + return nil, out or '', out or 'command failed', code +end + +local function link(iface, host_rate, ifb_name) + return { + iface = iface, + subnet = '172.30.32.1/24', + egress = { + enabled = true, mode = 'budgeted_peak', match = 'dst', + pool_rate = '8mbit', pool_ceil = '8mbit', + host_rate = host_rate or '2mbit', host_ceil = '6mbit', host_burst = '100k', host_cburst = '100k', + fq_codel = { flows = 128, limit = 1024, memory_limit = '1Mb' }, + hosts = { ['172.30.32.36'] = { rate = host_rate or '2mbit', ceil = '6mbit', burst = '100k', cburst = '100k' } }, + }, + ingress = { + enabled = true, mode = 'budgeted_peak', match = 'src', ifb = ifb_name or 'ifb_dcs_idem', + pool_rate = '8mbit', pool_ceil = '8mbit', + host_rate = host_rate or '2mbit', host_ceil = '6mbit', host_burst = '100k', host_cburst = '100k', + fq_codel = { flows = 128, limit = 1024, memory_limit = '1Mb' }, + hosts = { ['172.30.32.36'] = { rate = host_rate or '2mbit', ceil = '6mbit', burst = '100k', cburst = '100k' } }, + }, + } +end + +local function apply(iface, host_rate, ifb_name) + local result = tc_u32.apply({ links = { test = link(iface, host_rate, ifb_name) } }, { run_cmd = run }) + assert(result and result.ok == true, tostring(result and result.err)) +end + +fibers.run(function() + -- Exercise explicit cleanup of a former target so stale old attachments do not + -- remain after a target migration. + apply('dcshape-old', '2mbit', 'ifb_dcshape_old') + local ok, err = tc_u32.clear('dcshape-old', { ifb = 'ifb_dcshape_old', delete_ifb = true, run_cmd = run }) + assert(ok == true, tostring(err)) + + -- Repeated applies on the active target should be safe and should preserve the + -- live data plane. The third apply changes a host class rate so the class + -- reconciliation path is also exercised. + apply('dcshape-idem', '2mbit') + apply('dcshape-idem', '2mbit') + apply('dcshape-idem', '3mbit') +end) + +print('openwrt shaping idempotent apply: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'; ip netns del dcshape-idem-peer 2>/dev/null || true; ip link add dcshape-idem type veth peer name dcshape-idemp; ip link set dcshape-idem up; ip addr add 172.30.32.1/24 dev dcshape-idem; ip netns add dcshape-idem-peer; ip link set dcshape-idemp netns dcshape-idem-peer; ip netns exec dcshape-idem-peer ip link set lo up; ip netns exec dcshape-idem-peer ip link set dcshape-idemp up; ip netns exec dcshape-idem-peer ip addr add 172.30.32.36/24 dev dcshape-idemp; ip netns exec dcshape-idem-peer ip route replace default via 172.30.32.1 dev dcshape-idemp; ip link add dcshape-old type veth peer name dcshape-oldp; ip link set dcshape-old up; ip link set dcshape-oldp up" +CODE_REMOTE="$($STAGE_DEVICECODE)" +"$SCP_TO" "$WORK/run_shaping_idempotent_apply.lua" "$REMOTE/run.lua" +"$SSH" "cd '$CODE_REMOTE' && lua '$REMOTE/run.lua'" + +"$SSH" 'set -eu +fail() { echo "shaping idempotent apply test: $*" >&2; exit 1; } +class_bytes() { + dev="$1"; classid="$2" + tc -s class show dev "$dev" | awk -v c="$classid" '\'' + $1 == "class" && $2 == "htb" && $3 == c { hit = 1; next } + hit && $1 == "Sent" { print $2; exit } + '\'' +} +assert_counter_moves() { + label="$1"; dev="$2"; classid="$3"; shift 3 + before="$(class_bytes "$dev" "$classid")"; before="${before:-0}" + "$@" >/tmp/dc-shaping-idem-traffic.out 2>&1 || { cat /tmp/dc-shaping-idem-traffic.out >&2; fail "$label traffic command failed"; } + after="$(class_bytes "$dev" "$classid")"; after="${after:-0}" + [ "$after" -gt "$before" ] || fail "$label did not move $dev $classid: before=$before after=$after" +} + +[ "$(tc qdisc show dev dcshape-idem | grep -c "qdisc htb 1:")" -eq 1 ] || fail "expected one root HTB on dcshape-idem" +[ "$(tc qdisc show dev dcshape-idem | grep -c "qdisc ingress ffff:")" -eq 1 ] || fail "expected one ingress qdisc on dcshape-idem" +[ "$(tc qdisc show dev ifb_dcs_idem | grep -c "qdisc htb 1:")" -eq 1 ] || fail "expected one root HTB on ifb_dcs_idem" +tc class show dev dcshape-idem | grep -q "class htb 1:20" || fail "missing egress segment aggregate 1:20" +tc class show dev dcshape-idem | grep -q "class htb 20:1036" || fail "missing egress host budget class 20:1036" +tc class show dev dcshape-idem | grep -q "class htb 1036:1" || fail "missing egress host peak class 1036:1" +tc class show dev ifb_dcs_idem | grep -q "class htb 1:20" || fail "missing ingress segment aggregate 1:20" +tc class show dev ifb_dcs_idem | grep -q "class htb 20:1036" || fail "missing ingress host budget class 20:1036" +tc class show dev ifb_dcs_idem | grep -q "class htb 1036:1" || fail "missing ingress host peak class 1036:1" + +if tc qdisc show dev dcshape-old 2>/dev/null | grep -q "qdisc htb 1:"; then + fail "stale root HTB remains on old target dcshape-old" +fi +if ip link show dev ifb_dcshape_old >/dev/null 2>&1; then + fail "stale old IFB remains after clear" +fi + +assert_counter_moves "egress" dcshape-idem 20:1036 ping -I dcshape-idem -c 10 -s 800 172.30.32.36 +assert_counter_moves "ingress" ifb_dcs_idem 20:1036 sh -c "ip netns exec dcshape-idem-peer ping -c 10 -s 800 172.30.32.1 >/dev/null 2>&1 || true" + +printf "%s\n" "openwrt shaping idempotent apply topology/data-plane: ok" +' + diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_uart_stty_coreutils.sh b/tests/integration/openwrt_vm/tests/test_openwrt_uart_stty_coreutils.sh new file mode 100755 index 00000000..7e2be12e --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_uart_stty_coreutils.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-uart-stty-coreutils-test" +WORK="$VM_DIR/work/uart-stty-coreutils-test" + +mkdir -p "$WORK" +cat > "$WORK/run_openwrt_uart_stty_coreutils.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local exec = require 'fibers.io.exec' +local safe = require 'coxpcall' + +local ok_posix, posix = pcall(require, 'posix') +assert(ok_posix and type(posix) == 'table', 'luaposix is required') +assert(type(posix.openpty) == 'function', 'luaposix openpty is required') + +local perform = fibers.perform + +local function fail(msg) + error(msg, 2) +end + +local function exec_capture(args) + local cmd = exec.command(args) + local out, st, code, sig, err = perform(cmd:combined_output_op()) + out = out or '' + if st == 'exited' and code == 0 then + return out + end + fail('command failed: ' .. table.concat(args, ' ') .. + '\nstatus=' .. tostring(st) .. ' code=' .. tostring(code) .. ' signal=' .. tostring(sig) .. + '\nerr=' .. tostring(err) .. '\n--- output ---\n' .. out) +end + +local function exec_ok(args) + exec_capture(args) +end + +local function close_fd(fd) + if type(fd) == 'number' and type(posix.close) == 'function' then + pcall(posix.close, fd) + elseif type(fd) == 'userdata' and type(fd.close) == 'function' then + pcall(function() fd:close() end) + end +end + +local function pathish(v) + return type(v) == 'string' and v:sub(1, 1) == '/' +end + +local function fdnum(v) + if type(v) == 'number' then return v end + if type(v) == 'userdata' then + for _, name in ipairs({ 'fileno', 'getfd' }) do + local fn = v[name] + if type(fn) == 'function' then + local ok, n = pcall(fn, v) + if ok and type(n) == 'number' then return n end + end + end + end + return nil +end + +local function ttyname(fd) + fd = fdnum(fd) + if type(fd) ~= 'number' then return nil end + local fn = posix.ttyname + if type(fn) ~= 'function' then + local ok_unistd, unistd = pcall(require, 'posix.unistd') + if ok_unistd and type(unistd) == 'table' then + fn = unistd.ttyname + end + end + if type(fn) ~= 'function' then return nil end + local ok, name = pcall(fn, fd) + if ok and pathish(name) then return name end + return nil +end + +local function describe_returns(...) + local parts = {} + for i = 1, select('#', ...) do + local v = select(i, ...) + parts[#parts + 1] = tostring(i) .. '=' .. type(v) .. ':' .. tostring(v) + end + return table.concat(parts, ', ') +end + +local raw_a, raw_b, raw_c, raw_d = posix.openpty() +assert(raw_a, 'openpty failed: ' .. tostring(raw_b or raw_d)) + +-- luaposix variants differ. Prefer an explicit path return; otherwise ask the +-- OS for the slave fd name. Do not pass tostring(file) through a shell. +local master = raw_a +local slave = raw_b +local slave_name = pathish(raw_c) and raw_c or pathish(raw_b) and raw_b or ttyname(raw_b) or ttyname(raw_c) +if not pathish(slave_name) then + close_fd(raw_a) + close_fd(raw_b) + close_fd(raw_c) + fail('openpty slave path unavailable: ' .. describe_returns(raw_a, raw_b, raw_c, raw_d)) +end + +local strict = { + 'stty', '-F', slave_name, '115200', + 'cs8', '-cstopb', '-parenb', '-crtscts', + '-ixon', '-ixoff', '-icrnl', + '-icanon', '-echo', '-isig', '-iexten', + '-opost', '-onlcr', + 'min', '1', 'time', '0', + 'clocal', 'cread', +} + +local dirty = { + 'stty', '-F', slave_name, '9600', + 'icrnl', 'ixon', 'opost', 'onlcr', + 'isig', 'icanon', 'iexten', 'echo', + 'min', '0', 'time', '5', +} + +local function assert_contains(raw, token) + assert(raw:find(token, 1, true), 'expected stty output to contain ' .. token .. ':\n' .. raw) +end + +local function assert_min_time(raw) + assert(raw:find('min%s*=%s*1'), 'expected min = 1:\n' .. raw) + assert(raw:find('time%s*=%s*0'), 'expected time = 0:\n' .. raw) +end + +local ok, msg +fibers.run(function() + ok, msg = safe.pcall(function() + exec_capture({ 'stty', '--version' }) + exec_ok(dirty) + exec_ok(strict) + local raw = exec_capture({ 'stty', '-F', slave_name, '-a' }) + + assert_contains(raw, 'speed 115200') + for _, token in ipairs({ + 'cs8', 'cread', 'clocal', + '-cstopb', '-parenb', '-crtscts', + '-ixon', '-ixoff', '-icrnl', + '-icanon', '-echo', '-isig', '-iexten', + '-opost', '-onlcr', + }) do + assert_contains(raw, token) + end + assert_min_time(raw) + end) +end) + +close_fd(master) +close_fd(slave) +close_fd(raw_c) +if not ok then error(msg, 0) end +print('openwrt coreutils-stty PTY raw UART setup: ok') +LUA + +"$SSH" 'command -v lua >/dev/null' +"$SSH" 'command -v stty >/dev/null' +"$SSH" 'opkg list-installed | grep -q "^coreutils-stty -" || { echo "coreutils-stty is not installed" >&2; exit 1; }' +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$WORK/run_openwrt_uart_stty_coreutils.lua" "$REMOTE/run_openwrt_uart_stty_coreutils.lua" +"$SSH" "cd '$REMOTE' && lua ./run_openwrt_uart_stty_coreutils.lua" + +echo "openwrt UART stty coreutils: ok" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_vm_generated_configs_expected.sh b/tests/integration/openwrt_vm/tests/test_openwrt_vm_generated_configs_expected.sh new file mode 100755 index 00000000..5b9d3382 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_vm_generated_configs_expected.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +REMOTE="/tmp/devicecode-vm-generated-config-expected-test" + +DEVICECODE_VM_MWAN_CONFIG_FORCE=1 "$SCRIPT_DIR/setup_devicecode_vm_mwan_generated_config.sh" + +mkdir -p "$VM_DIR/work" +cat > "$VM_DIR/work/run_devicecode_vm_generated_config_expected.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + './fixtures/?.lua', + package.path, +}, ';') + +local uci = require 'uci' +local names_mod = require 'services.hal.backends.network.providers.openwrt.names' +local vm_intent = require 'devicecode_vm_mwan_intent' + +local function fail(msg) error(msg, 2) end +local function eq(a, b, msg) if a ~= b then fail((msg or 'values differ') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function assert_true(v, msg) if not v then fail(msg or 'assertion failed') end end +local function list_values(v) + if type(v) == 'table' then return v end + if v == nil then return {} end + return { v } +end +local function sorted(v) + local out = {} + for i = 1, #(v or {}) do out[i] = tostring(v[i]) end + table.sort(out) + return out +end +local function eq_list(v, expected, msg) + local a, b = sorted(list_values(v)), sorted(expected) + if #a ~= #b then fail((msg or 'list length') .. ': expected ' .. table.concat(b, ',') .. ', got ' .. table.concat(a, ',')) end + for i = 1, #b do if a[i] ~= b[i] then fail((msg or 'list') .. ': expected ' .. table.concat(b, ',') .. ', got ' .. table.concat(a, ',')) end end +end +local function section(c, pkg, name, typ) + local sec = c:get_all(pkg, name) + if type(sec) ~= 'table' then fail('missing section ' .. pkg .. '.' .. tostring(name)) end + if typ then eq(sec['.type'], typ, pkg .. '.' .. name .. ' type') end + return sec +end +local function find_section(c, pkg, typ, pred, label) + local all = c:get_all(pkg) or {} + for name, sec in pairs(all) do + if type(sec) == 'table' and sec['.type'] == typ and pred(name, sec) then return name, sec end + end + fail('missing ' .. (label or (pkg .. ' ' .. typ))) +end +local function assert_no_metadata(c) + for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do + for name, sec in pairs(c:get_all(pkg) or {}) do + if type(sec) == 'table' then + for _, opt in ipairs({ 'devicecode_managed', 'devicecode_owner', 'devicecode_semantic_id', 'devicecode_role' }) do + if sec[opt] ~= nil then fail('unexpected metadata ' .. pkg .. '.' .. tostring(name) .. '.' .. opt) end + end + end + end + end +end +local function assert_no_legacy_modem_names(c) + for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do + for name, sec in pairs(c:get_all(pkg) or {}) do + local blob = tostring(name) + if type(sec) == 'table' then + for k, v in pairs(sec) do + blob = blob .. ' ' .. tostring(k) + if type(v) == 'table' then blob = blob .. ' ' .. table.concat(v, ' ') else blob = blob .. ' ' .. tostring(v) end + end + end + if blob:find('mdm0', 1, true) or blob:find('mdm1', 1, true) then fail('legacy modem name leaked in ' .. pkg .. '.' .. tostring(name)) end + end + end +end + +local intent = vm_intent.intent() +local name_ctx = assert(names_mod.allocate(intent)) +local c = assert(uci.cursor('/etc/config')) +for _, pkg in ipairs({ 'network', 'dhcp', 'firewall', 'mwan3' }) do if type(c.load) == 'function' then pcall(function() c:load(pkg) end) end end + +local loopback = section(c, 'network', 'loopback', 'interface') +eq(loopback.device, 'lo', 'loopback device') +eq(loopback.proto, 'static', 'loopback proto') +section(c, 'network', 'globals', 'globals') + +local br = section(c, 'network', name_ctx:section('dev_bridge', 'lan'), 'device') +eq(br.name, 'br-lan', 'LAN bridge name') +eq(br.type, 'bridge', 'LAN bridge type') +eq_list(br.ports, { 'eth0' }, 'LAN bridge ports') +local lan = section(c, 'network', name_ctx:iface('lan'), 'interface') +eq(lan.proto, 'static', 'LAN proto') +eq(lan.device, 'br-lan', 'LAN device') +eq(lan.ipaddr, '192.168.1.1', 'LAN ipaddr') +eq(lan.netmask, '255.255.255.0', 'LAN netmask') + +for _, wan in ipairs(vm_intent.wans) do + local sec = section(c, 'network', name_ctx:iface(wan.id), 'interface') + eq(sec.proto, 'dhcp', wan.id .. ' proto') + eq(sec.device, wan.device, wan.id .. ' device') + eq(sec.peerdns, '0', wan.id .. ' peerdns') + eq(sec.metric, tostring(wan.route_metric), wan.id .. ' generated route metric') + eq(sec.defaultroute, nil, wan.id .. ' defaultroute should use OpenWrt default') + + local dh = section(c, 'dhcp', name_ctx:section('dhcp', wan.id), 'dhcp') + eq(dh.interface, name_ctx:iface(wan.id), wan.id .. ' DHCP interface') + eq(dh.ignore, '1', wan.id .. ' DHCP ignore') + + local mw_if = section(c, 'mwan3', name_ctx:mwan_iface(wan.id), 'interface') + eq(mw_if.enabled, '1', wan.id .. ' mwan enabled') + eq(mw_if.family, 'ipv4', wan.id .. ' mwan family') + eq_list(mw_if.track_ip, { wan.gateway }, wan.id .. ' track_ip') + eq(mw_if.initial_state, 'online', wan.id .. ' mwan initial_state') + eq(mw_if.reliability, '1', wan.id .. ' reliability') + + local member_name, member = find_section(c, 'mwan3', 'member', function(_, s) return s.interface == name_ctx:mwan_iface(wan.id) end, wan.id .. ' mwan member') + eq(member.metric, '1', wan.id .. ' mwan policy metric') + eq(member.weight, tostring(wan.weight), wan.id .. ' mwan weight') +end + +local dh_lan = section(c, 'dhcp', name_ctx:section('dhcp', 'lan'), 'dhcp') +eq(dh_lan.interface, 'lan', 'LAN DHCP interface') +eq(dh_lan.start, '100', 'LAN DHCP start') +eq(dh_lan.limit, '150', 'LAN DHCP limit') +find_section(c, 'dhcp', 'dnsmasq', function(_, s) + return s.domain == 'vm.bigbox.test' and tostring(s.cachesize) == '1000' +end, 'vm dnsmasq') + +local _, zone_lan = find_section(c, 'firewall', 'zone', function(_, s) return s.name == 'lan' end, 'LAN firewall zone') +eq_list(zone_lan.network, { 'lan' }, 'LAN firewall networks') +local _, zone_wan = find_section(c, 'firewall', 'zone', function(_, s) return s.name == 'wan' end, 'WAN firewall zone') +eq(zone_wan.masq, '1', 'WAN zone masq') +eq(zone_wan.mtu_fix, '1', 'WAN zone mtu_fix') +eq_list(zone_wan.network, { 'wan', 'wanb', 'wanc' }, 'WAN firewall networks') +find_section(c, 'firewall', 'forwarding', function(_, s) return s.src == 'lan' and s.dest == 'wan' end, 'LAN to WAN forwarding') + +local policy = section(c, 'mwan3', name_ctx:mwan_policy('balanced'), 'policy') +eq(policy.last_resort, 'unreachable', 'balanced last_resort') +local expected_members = {} +for _, wan in ipairs(vm_intent.wans) do + local member_name = (find_section(c, 'mwan3', 'member', function(_, s) return s.interface == name_ctx:mwan_iface(wan.id) end, wan.id .. ' member')) + expected_members[#expected_members + 1] = member_name +end +eq_list(policy.use_member, expected_members, 'balanced policy members') +local https = section(c, 'mwan3', name_ctx:mwan_rule('https'), 'rule') +eq(https.proto, 'tcp', 'https sticky proto') +eq(https.dest_port, '443', 'https sticky dest_port') +eq(https.family, 'ipv4', 'https sticky family') +eq(https.sticky, '1', 'https sticky flag') +eq(https.use_policy, name_ctx:mwan_policy('balanced'), 'https sticky policy') +local rule = section(c, 'mwan3', name_ctx:mwan_rule('default_rule_v4'), 'rule') +eq(rule.dest_ip, '0.0.0.0/0', 'default rule dest') +eq(rule.family, 'ipv4', 'default rule family') +eq(rule.use_policy, name_ctx:mwan_policy('balanced'), 'default rule policy') + +assert_no_metadata(c) +assert_no_legacy_modem_names(c) +print('openwrt VM generated /etc/config files match expected Devicecode shape: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE/fixtures'" +"$SCP_TO" "$ROOT_DIR/src" "$REMOTE/src" +"$SCP_TO" "$ROOT_DIR/vendor" "$REMOTE/vendor" +"$SCP_TO" "$VM_DIR/fixtures/devicecode_vm_mwan_intent.lua" "$REMOTE/fixtures/devicecode_vm_mwan_intent.lua" +"$SCP_TO" "$VM_DIR/work/run_devicecode_vm_generated_config_expected.lua" "$REMOTE/run_devicecode_vm_generated_config_expected.lua" +"$SSH" "cd '$REMOTE' && lua ./run_devicecode_vm_generated_config_expected.lua" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_vm_mwan_connected.sh b/tests/integration/openwrt_vm/tests/test_openwrt_vm_mwan_connected.sh new file mode 100755 index 00000000..db21f7cb --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_vm_mwan_connected.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +SSH="$VM_DIR/scripts/ssh" + +# First generate the real /etc/config/* files without running disruptive +# activation commands inside the Devicecode apply SSH session. Then activate +# those files with explicit service commands below; a network reload may briefly +# drop the management SSH connection, so each activation phase is bounded and +# followed by wait-ssh. +DEVICECODE_VM_MWAN_CONFIG_ACTIVATE=0 DEVICECODE_VM_MWAN_CONFIG_FORCE=1 "$SCRIPT_DIR/setup_devicecode_vm_mwan_generated_config.sh" + +run_activation_step() { + label="$1"; shift + echo "[openwrt-vm] activation: $label" + "$SSH" "$@" || true + "$VM_DIR/scripts/wait-ssh" +} + +run_activation_step 'network reload' "/etc/init.d/network reload >/tmp/devicecode-vm-network-reload.log 2>&1 || { cat /tmp/devicecode-vm-network-reload.log; exit 1; }" +run_activation_step 'firewall restart' "/etc/init.d/firewall restart >/tmp/devicecode-vm-firewall-restart.log 2>&1 || { cat /tmp/devicecode-vm-firewall-restart.log; exit 1; }" +run_activation_step 'mwan3 enable' "/etc/init.d/mwan3 enable >/tmp/devicecode-vm-mwan3-enable.log 2>&1 || true" +run_activation_step 'mwan3 restart' "/etc/init.d/mwan3 restart >/tmp/devicecode-vm-mwan3-restart.log 2>&1 || { cat /tmp/devicecode-vm-mwan3-restart.log; exit 1; }" + +"$SSH" 'sh -s' <<'REMOTE' +set -eu +for dev in eth1 eth2 eth3; do + ip link show "$dev" >/dev/null || { echo "missing $dev" >&2; exit 1; } +done + +wait_for() { + label="$1"; shift + deadline=$(( $(date +%s) + 45 )) + while :; do + if sh -c "$*" >/dev/null 2>&1; then return 0; fi + if [ "$(date +%s)" -gt "$deadline" ]; then + echo "timed out waiting for $label" >&2 + sh -c "$*" || true + exit 1 + fi + sleep 1 + done +} + +wait_for 'wan DHCP address' "ip -4 addr show dev eth1 | grep -q 'inet 172\.31\.1\.'" +wait_for 'wanb DHCP address' "ip -4 addr show dev eth2 | grep -q 'inet 172\.31\.2\.'" +wait_for 'wanc DHCP address' "ip -4 addr show dev eth3 | grep -q 'inet 172\.31\.3\.'" + +ip -4 route show default | tee /tmp/devicecode-vm-default-routes.log +grep -q 'default .* dev eth1 .* metric 11' /tmp/devicecode-vm-default-routes.log +grep -q 'default .* dev eth2 .* metric 12' /tmp/devicecode-vm-default-routes.log +grep -q 'default .* dev eth3 .* metric 13' /tmp/devicecode-vm-default-routes.log + +ping -c 1 -W 2 -I eth1 172.31.1.2 >/dev/null +ping -c 1 -W 2 -I eth2 172.31.2.2 >/dev/null +ping -c 1 -W 2 -I eth3 172.31.3.2 >/dev/null + +wait_for 'mwan3 balanced policy and https sticky rule' "mwan3 status > /tmp/devicecode-vm-mwan-status.log 2>&1 && grep -q 'balanced:' /tmp/devicecode-vm-mwan-status.log && grep -q 'wan (' /tmp/devicecode-vm-mwan-status.log && grep -q 'wanb (' /tmp/devicecode-vm-mwan-status.log && grep -q 'wanc (' /tmp/devicecode-vm-mwan-status.log && grep -Eq 'S[[:space:]]+https' /tmp/devicecode-vm-mwan-status.log" +wait_for 'mwan3 interfaces online' "mwan3 status > /tmp/devicecode-vm-mwan-status.log 2>&1 && grep -Eq 'interface wan is online|interface wan .* online' /tmp/devicecode-vm-mwan-status.log && grep -Eq 'interface wanb is online|interface wanb .* online' /tmp/devicecode-vm-mwan-status.log && grep -Eq 'interface wanc is online|interface wanc .* online' /tmp/devicecode-vm-mwan-status.log" +cat /tmp/devicecode-vm-mwan-status.log + +iptables-save -t mangle >/tmp/devicecode-vm-mangle.rules +grep -q '^:mwan3_policy_balanced ' /tmp/devicecode-vm-mangle.rules +grep -q '^:mwan3_rule_https ' /tmp/devicecode-vm-mangle.rules +grep -q 'mwan3_policy_balanced' /tmp/devicecode-vm-mangle.rules +grep -q -- '--dports 443' /tmp/devicecode-vm-mangle.rules +grep -q '^:mwan3_iface_in_wan ' /tmp/devicecode-vm-mangle.rules +grep -q '^:mwan3_iface_in_wanb ' /tmp/devicecode-vm-mangle.rules +grep -q '^:mwan3_iface_in_wanc ' /tmp/devicecode-vm-mangle.rules +REMOTE + +echo 'openwrt VM Devicecode generated config has active MWAN connectivity: ok' diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_wan_mark_download_capability.sh b/tests/integration/openwrt_vm/tests/test_openwrt_wan_mark_download_capability.sh new file mode 100755 index 00000000..31d6955d --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_wan_mark_download_capability.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +STAGE_DEVICECODE="$VM_DIR/scripts/ensure-devicecode-staged" +CLEANUP="$VM_DIR/scripts/cleanup-shaping-state" +REMOTE="/tmp/devicecode-wan-mark-download-capability" +WORK="$VM_DIR/work/wan-mark-download-capability" + +mkdir -p "$WORK" +"$CLEANUP" >/dev/null 2>&1 || true +trap 'set +e; "$CLEANUP" >/dev/null 2>&1 || true' EXIT INT TERM + +cat > "$WORK/run_wan_mark_download_capability.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local exec = require 'fibers.io.exec' +local tc_mark = require 'services.hal.backends.network.providers.openwrt.tc_mark_shaper' +local unpack = table.unpack or unpack +local perform = fibers.perform + +local function run(argv) + local cmd = exec.command(unpack(argv)) + local out, st, code = perform(cmd:combined_output_op()) + if st == 'exited' and code == 0 then return true, out or '', nil, code end + return nil, out or '', out or 'command failed', code +end + +local result +fibers.run(function() + result = tc_mark.apply({ + marks = { mask = '0x00f00000', control = '0x00100000', client = '0x00200000' }, + links = { + cap = { + kind = 'wan_mark', iface = 'dcwan-cap', + egress = { enabled = false }, + ingress = { + enabled = true, ifb = 'ifb_dcwan_cap', + root = { rate = '1gbit', ceil = '1gbit' }, + control = { rate = '1gbit', ceil = '1gbit' }, + client = { rate = '8mbit', ceil = '8mbit' }, + fq_codel = { flows = 128, limit = 1024, memory_limit = '1Mb' }, + }, + }, + }, + }, { run_cmd = run }) +end) + +if result and result.ok == true then + print('WAN_DOWNLOAD_CAPABILITY=supported') +else + print('WAN_DOWNLOAD_CAPABILITY=unsupported') + print('WAN_DOWNLOAD_ERROR=' .. tostring(result and result.err or 'unknown')) +end +LUA + +"$SSH" 'set -eu +modprobe ifb 2>/dev/null || true +modprobe sch_htb 2>/dev/null || true +modprobe sch_ingress 2>/dev/null || true +modprobe sch_fq_codel 2>/dev/null || true +modprobe cls_u32 2>/dev/null || true +modprobe cls_fw 2>/dev/null || true +modprobe act_mirred 2>/dev/null || true +modprobe act_ctinfo 2>/dev/null || true +ip link del dcwan-cap 2>/dev/null || true +ip link del ifb_dcwan_cap 2>/dev/null || true +ip link add dcwan-cap type veth peer name dcwan-capp +ip link set dcwan-cap up +ip link set dcwan-capp up +' + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +CODE_REMOTE="$($STAGE_DEVICECODE)" +"$SCP_TO" "$WORK/run_wan_mark_download_capability.lua" "$REMOTE/run.lua" +out="$($SSH "cd '$CODE_REMOTE' && lua '$REMOTE/run.lua'")" +printf '%s\n' "$out" + +case "$out" in + *WAN_DOWNLOAD_CAPABILITY=supported*) + "$SSH" 'set -eu + fail() { echo "wan mark download capability test: $*" >&2; exit 1; } + tc qdisc show dev dcwan-cap | grep -q "qdisc ingress ffff:" || fail "supported path should install ingress qdisc" + tc filter show dev dcwan-cap parent ffff: | grep -q "ctinfo" || fail "supported path should restore connmark with ctinfo" + tc qdisc show dev ifb_dcwan_cap | grep -q "qdisc htb 1:" || fail "supported path should install IFB root HTB" + tc class show dev ifb_dcwan_cap | grep -q "class htb 1:10" || fail "supported path should install control class" + tc class show dev ifb_dcwan_cap | grep -q "class htb 1:20" || fail "supported path should install client class" + printf "%s\n" "openwrt wan mark download capability: supported" + ' + ;; + *WAN_DOWNLOAD_CAPABILITY=unsupported*) + "$SSH" 'set -eu + fail() { echo "wan mark download capability test: $*" >&2; exit 1; } + if tc qdisc show dev ifb_dcwan_cap 2>/dev/null | grep -q "qdisc htb 1:"; then + fail "unsupported path should not install misleading IFB shaping root" + fi + printf "%s\n" "openwrt wan mark download capability: unsupported safely reported" + ' + ;; + *) + echo "wan mark download capability test: could not determine capability" >&2 + exit 1 + ;; +esac + diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_wan_mark_shaping_contract.sh b/tests/integration/openwrt_vm/tests/test_openwrt_wan_mark_shaping_contract.sh new file mode 100755 index 00000000..5c5552b2 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_wan_mark_shaping_contract.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SCP_TO="$VM_DIR/scripts/scp-to" +STAGE_DEVICECODE="$VM_DIR/scripts/ensure-devicecode-staged" +SSH="$VM_DIR/scripts/ssh" +WORK="$VM_DIR/work/wan-mark-shaping-contract" +REMOTE="/tmp/devicecode-wan-mark-shaping-contract" + +mkdir -p "$WORK" +cat > "$WORK/run_openwrt_wan_mark_shaping_contract.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local provider_loader = require 'services.hal.backends.network.provider' + +local function fail(msg) error(msg, 2) end +local function contains(s, needle, msg) + if type(s) ~= 'string' or not s:find(needle, 1, true) then fail(msg or ('missing ' .. tostring(needle))) end +end + +local cmds = {} +local restores = {} +local provider = assert(provider_loader.new({ + provider = 'openwrt', + allow_fake_uci = true, + debounce_s = 0.01, + run_cmd = function(_argv) return true, '', nil end, + shaper_run_cmd = function(argv) + cmds[#cmds + 1] = table.concat(argv, ' ') + return true, '', nil + end, + shaper_run_restore = function(payload) + restores[#restores + 1] = payload + return true, nil, '' + end, +}, {})) + +local intent = { + schema = 'devicecode.net.intent/1', rev = 1, + segments = { wan = { kind = 'wan', firewall = { zone = 'wan' } } }, + interfaces = { + wan = { kind = 'ethernet', role = 'wan', segment = 'wan', endpoint = { ifname = 'eth1' }, addressing = { ipv4 = { mode = 'dhcp' } } }, + }, + firewall = { zones = { wan = { masq = true } }, policies = {}, rules = {} }, + routing = {}, dns = {}, dhcp = {}, vpn = {}, diagnostics = {}, + wan = { + enabled = true, + members = { + wired = { + interface = 'wan', mwan_metric = 1, weight = 1, + shaping = { download = { limit = '80mbit' }, upload = { limit = '20mbit' } }, + }, + }, + }, +} + +fibers.run(function() + local result = fibers.perform(provider:apply_op({ intent = intent })) + assert(result and result.ok == true, 'apply failed: ' .. tostring(result and result.err)) + provider:terminate('test complete') +end) + +if #restores ~= 1 then fail('expected one iptables-restore payload, got ' .. tostring(#restores)) end +local restore = restores[1] +contains(restore, ':DEVICECODE_SHAPING_OUTPUT', 'output chain expected') +contains(restore, ':DEVICECODE_SHAPING_FORWARD', 'forward chain expected') +contains(restore, '-A DEVICECODE_SHAPING_OUTPUT -o eth1', 'router-originated traffic should be marked on WAN') +contains(restore, 'devicecode-shaping router exempt', 'router exemption comment expected') +contains(restore, '-A DEVICECODE_SHAPING_FORWARD -o eth1', 'forwarded client traffic should be marked on WAN') +contains(restore, 'CONNMARK --save-mark --mask 0x00f00000', 'marks should be saved to conntrack') + +local all = table.concat(cmds, '\n') +contains(all, 'tc qdisc add dev eth1 root handle 1: htb default 20', 'WAN upload root qdisc expected') +contains(all, 'tc class replace dev eth1 parent 1:1 classid 1:10 htb rate 1gbit ceil 1gbit', 'router/control class should be high-rate') +contains(all, 'tc class replace dev eth1 parent 1:1 classid 1:20 htb rate 20mbit ceil 20mbit', 'client upload class should be limited') +contains(all, 'tc filter add dev eth1 parent ffff: protocol ip prio 1 u32 match u32 0 0 action ctinfo cpmark 0x00f00000 action mirred egress redirect dev ifb_eth1', 'download path should restore connmark before IFB redirect') +contains(all, 'tc class replace dev ifb_eth1 parent 1:1 classid 1:20 htb rate 80mbit ceil 80mbit', 'client download class should be limited') + +print('openwrt wan mark shaping contract: ok') +LUA + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +CODE_REMOTE="$($STAGE_DEVICECODE)" +"$SCP_TO" "$WORK/run_openwrt_wan_mark_shaping_contract.lua" "$REMOTE/run.lua" +"$SSH" "cd '$CODE_REMOTE' && lua '$REMOTE/run.lua'" diff --git a/tests/integration/openwrt_vm/tests/test_openwrt_wan_mark_upload_exemption.sh b/tests/integration/openwrt_vm/tests/test_openwrt_wan_mark_upload_exemption.sh new file mode 100755 index 00000000..1e0f4ee6 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_openwrt_wan_mark_upload_exemption.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +ROOT_DIR="$(CDPATH= cd -- "$VM_DIR/../../.." && pwd)" +SSH="$VM_DIR/scripts/ssh" +SCP_TO="$VM_DIR/scripts/scp-to" +STAGE_DEVICECODE="$VM_DIR/scripts/ensure-devicecode-staged" +CLEANUP="$VM_DIR/scripts/cleanup-shaping-state" +REMOTE="/tmp/devicecode-wan-mark-upload-exemption" +WORK="$VM_DIR/work/wan-mark-upload-exemption" + +mkdir -p "$WORK" +"$CLEANUP" >/dev/null 2>&1 || true +IP_FORWARD_ORIG="$($SSH 'cat /proc/sys/net/ipv4/ip_forward' 2>/dev/null || true)" +cleanup_all() { + set +e + if [ -n "${IP_FORWARD_ORIG:-}" ]; then + "$SSH" "sysctl -w net.ipv4.ip_forward=$IP_FORWARD_ORIG >/dev/null 2>&1" >/dev/null 2>&1 || true + fi + "$CLEANUP" >/dev/null 2>&1 || true +} +trap cleanup_all EXIT INT TERM + +cat > "$WORK/run_wan_mark_upload_exemption.lua" <<'LUA' +package.path = table.concat({ + './src/?.lua', './src/?/init.lua', + './vendor/lua-fibers/src/?.lua', './vendor/lua-fibers/src/?/init.lua', + './vendor/lua-bus/src/?.lua', './vendor/lua-bus/src/?/init.lua', + './vendor/lua-trie/src/?.lua', './vendor/lua-trie/src/?/init.lua', + package.path, +}, ';') + +local fibers = require 'fibers' +local exec = require 'fibers.io.exec' +local shaper = require 'services.hal.backends.network.providers.openwrt.shaper' +local unpack = table.unpack or unpack +local perform = fibers.perform + +local function run(argv) + local cmd = exec.command(unpack(argv)) + local out, st, code = perform(cmd:combined_output_op()) + if st == 'exited' and code == 0 then return true, out or '', nil, code end + return nil, out or '', out or 'command failed', code +end + +local req = { + marks = { mask = '0x00f00000', control = '0x00100000', client = '0x00200000' }, + links = { + test_wan = { + kind = 'wan_mark', + iface = 'dcwan0', + egress = { + enabled = true, + root = { rate = '1gbit', ceil = '1gbit' }, + control = { rate = '1gbit', ceil = '1gbit' }, + client = { rate = '2mbit', ceil = '2mbit' }, + fq_codel = { flows = 128, limit = 1024, memory_limit = '1Mb' }, + }, + ingress = { enabled = false }, + }, + }, +} + +fibers.run(function() + local result = shaper.apply(req, { run_cmd = run }) + assert(result and result.ok == true, tostring(result and result.err)) +end) + +print('openwrt wan mark upload exemption apply: ok') +LUA + +"$SSH" 'set -eu +modprobe sch_htb 2>/dev/null || true +modprobe sch_fq_codel 2>/dev/null || true +modprobe cls_fw 2>/dev/null || true +modprobe xt_mark 2>/dev/null || true +modprobe xt_CONNMARK 2>/dev/null || true +modprobe xt_comment 2>/dev/null || true +ip link del dcwan0 2>/dev/null || true +ip link del dcclient-host 2>/dev/null || true +ip netns del dcwan-upstream 2>/dev/null || true +ip netns del dcwan-client 2>/dev/null || true +ip netns add dcwan-upstream +ip netns add dcwan-client +ip link add dcwan0 type veth peer name dcwan-up0 +ip link set dcwan-up0 netns dcwan-upstream +ip addr add 10.203.0.1/24 dev dcwan0 +ip link set dcwan0 up +ip netns exec dcwan-upstream ip addr add 10.203.0.2/24 dev dcwan-up0 +ip netns exec dcwan-upstream ip link set lo up +ip netns exec dcwan-upstream ip link set dcwan-up0 up +ip netns exec dcwan-upstream ip route add 10.204.0.0/24 via 10.203.0.1 dev dcwan-up0 + +ip link add dcclient-host type veth peer name dcclient0 +ip link set dcclient0 netns dcwan-client +ip addr add 10.204.0.1/24 dev dcclient-host +ip link set dcclient-host up +ip netns exec dcwan-client ip addr add 10.204.0.2/24 dev dcclient0 +ip netns exec dcwan-client ip link set lo up +ip netns exec dcwan-client ip link set dcclient0 up +ip netns exec dcwan-client ip route add default via 10.204.0.1 dev dcclient0 +sysctl -w net.ipv4.ip_forward=1 >/dev/null +iptables -I FORWARD -i dcclient-host -o dcwan0 -j ACCEPT +iptables -I FORWARD -i dcwan0 -o dcclient-host -j ACCEPT +' + +"$SSH" "rm -rf '$REMOTE'; mkdir -p '$REMOTE'" +CODE_REMOTE="$($STAGE_DEVICECODE)" +"$SCP_TO" "$WORK/run_wan_mark_upload_exemption.lua" "$REMOTE/run.lua" +"$SSH" "cd '$CODE_REMOTE' && lua '$REMOTE/run.lua'" + +"$SSH" 'set -eu +fail() { echo "wan mark upload exemption test: $*" >&2; exit 1; } +class_bytes() { + dev="$1"; classid="$2" + tc -s class show dev "$dev" | awk -v c="$classid" '\'' + $1 == "class" && $2 == "htb" && $3 == c { hit = 1; next } + hit && $1 == "Sent" { print $2; exit } + '\'' +} +assert_moves() { + label="$1"; classid="$2"; shift 2 + before="$(class_bytes dcwan0 "$classid")"; before="${before:-0}" + "$@" >/tmp/dcwan-upload-traffic.out 2>&1 || { cat /tmp/dcwan-upload-traffic.out >&2; fail "$label traffic failed"; } + after="$(class_bytes dcwan0 "$classid")"; after="${after:-0}" + [ "$after" -gt "$before" ] || { + tc -s class show dev dcwan0 >&2 2>/dev/null || true + iptables -t mangle -S >&2 2>/dev/null || true + fail "$label did not move class $classid: before=$before after=$after" + } +} + +tc qdisc show dev dcwan0 | grep -q "qdisc htb 1:" || fail "missing WAN root HTB" +tc class show dev dcwan0 | grep -q "class htb 1:10" || fail "missing router/control class" +tc class show dev dcwan0 | grep -q "class htb 1:20" || fail "missing client class" +tc filter show dev dcwan0 parent 1: | grep -Eq "(classid|flowid) 1:10" || { tc filter show dev dcwan0 parent 1: >&2 2>/dev/null || true; fail "missing control mark filter"; } +tc filter show dev dcwan0 parent 1: | grep -Eq "(classid|flowid) 1:20" || { tc filter show dev dcwan0 parent 1: >&2 2>/dev/null || true; fail "missing client mark filter"; } + +assert_moves "router-originated upload" 1:10 sh -c "ping -I dcwan0 -c 10 -s 800 10.203.0.2 >/dev/null 2>&1 || true" + +# With MWAN active in the integration VM, a synthetic test interface that is not +# an MWAN member can be policy-routed or dropped before it ever reaches dcwan0. +# Keep the forwarded-client contract covered by the mangle-chain assertions +# above, and exercise the client TC class by applying the same client mark to +# router-originated test packets on dcwan0. Append this rule after +# DEVICECODE_SHAPING_OUTPUT; otherwise the router-exempt rule in that chain will +# overwrite the test client mark and the packet will still classify as control. +iptables -t mangle -A OUTPUT -o dcwan0 -m comment --comment devicecode-test-client-mark -j MARK --set-xmark 0x00200000/0x00f00000 +assert_moves "client-marked upload" 1:20 sh -c "ping -I dcwan0 -c 10 -s 800 10.203.0.2 >/dev/null 2>&1 || true" +iptables -t mangle -D OUTPUT -o dcwan0 -m comment --comment devicecode-test-client-mark -j MARK --set-xmark 0x00200000/0x00f00000 2>/dev/null || true + +printf "%s\n" "openwrt wan mark upload exemption data-plane: ok" +' + diff --git a/tests/integration/openwrt_vm/tests/test_tc_veth_baseline.sh b/tests/integration/openwrt_vm/tests/test_tc_veth_baseline.sh new file mode 100755 index 00000000..cce336e1 --- /dev/null +++ b/tests/integration/openwrt_vm/tests/test_tc_veth_baseline.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +VM_DIR="$(dirname "$SCRIPT_DIR")" +SSH="$VM_DIR/scripts/ssh" + +run() { + "$SSH" "$@" +} + +run ' + set -eu + + cleanup() { + ip link del dc-veth-a 2>/dev/null || true + ip link del dc-ifb0 2>/dev/null || true + } + trap cleanup EXIT + cleanup + + echo "[openwrt-vm] checking kernel modules" + modprobe veth 2>/dev/null || true + modprobe ifb 2>/dev/null || true + modprobe sch_htb 2>/dev/null || true + modprobe sch_ingress 2>/dev/null || true + modprobe sch_fq_codel 2>/dev/null || true + modprobe cls_u32 2>/dev/null || true + modprobe act_mirred 2>/dev/null || true + + echo "[openwrt-vm] creating veth" + ip link add dc-veth-a type veth peer name dc-veth-b + ip link set dc-veth-a up + ip link set dc-veth-b up + + echo "[openwrt-vm] adding root htb" + tc qdisc add dev dc-veth-a root handle 1: htb default 10 + + echo "[openwrt-vm] adding htb class" + tc class add dev dc-veth-a parent 1: classid 1:10 htb rate 10mbit ceil 10mbit + + echo "[openwrt-vm] adding fq_codel" + tc qdisc add dev dc-veth-a parent 1:10 fq_codel + + echo "[openwrt-vm] adding ifb" + ip link add dc-ifb0 type ifb + ip link set dc-ifb0 up + + echo "[openwrt-vm] adding ingress" + tc qdisc add dev dc-veth-b ingress + + echo "[openwrt-vm] tc state" + tc qdisc show dev dc-veth-a + tc class show dev dc-veth-a + tc qdisc show dev dc-veth-b + + echo "openwrt tc/veth baseline: ok" +' diff --git a/tests/integration/openwrt_vm/work/.gitignore b/tests/integration/openwrt_vm/work/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/tests/integration/openwrt_vm/work/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/run.lua b/tests/run.lua new file mode 100644 index 00000000..c6ee9e24 --- /dev/null +++ b/tests/run.lua @@ -0,0 +1,259 @@ +-- tests/run.lua + +package.path = "../src/?.lua;" .. package.path +package.path = '../?.lua;../?/init.lua;./?.lua;./?/init.lua;' .. package.path + +local function add_path(prefix) + package.path = prefix .. '?.lua;' .. prefix .. '?/init.lua;' .. package.path +end + +local env = os.getenv('DEVICECODE_ENV') or 'dev' +if env == 'prod' then + add_path('./lib/') +else + add_path('../vendor/lua-fibers/src/') + add_path('../vendor/lua-bus/src/') + add_path('../vendor/lua-trie/src/') + add_path('./') +end + +local safe = require 'coxpcall' +local stdlib = require 'posix.stdlib' + +assert(stdlib.setenv('CONFIG_TARGET', 'services')) + +local files = { + 'unit.config.codec_spec', + 'unit.config.state_spec', + 'unit.main.service_spec', + 'unit.config.service_spec', + 'unit.system_spec', + 'unit.devicecode.service_base_spec', + 'unit.monitor.test_service', + 'unit.hal.cap_sdk_spec', + 'unit.hal.hal_compat_spec', + 'unit.hal.openwrt_uci_manager_spec', + 'unit.hal.openwrt_uci_singleton_compat_spec', + 'unit.hal.service_raw_host_spec', + 'unit.hal.control_store_provider_spec', + 'unit.hal.control_store_manager_spec', + "unit.hal.signature_verify_manager_spec", + "unit.hal.signature_verify_provider_spec", + "unit.hal.signature_verify_openssl_spec", + "unit.hal.artifact_store_driver_spec", + "unit.hal.artifact_store_provider_spec", + "unit.hal.artifact_store_manager_spec", + "unit.hal.uart_driver_spec", + "unit.hal.uart_manager_spec", + "unit.hal.network_manager_spec", + "unit.hal.modem_linux_mm_spec", + "unit.hal.modem_qmi_spec", + "unit.hal.openwrt_network_observer_spec", + "unit.hal.openwrt_network_provider_advanced_spec", + "unit.hal.common_uci_compat_spec", + "unit.hal.openwrt_names_spec", + 'unit.fabric.test_model', + 'unit.fabric.test_config', + 'unit.fabric.test_dependencies', + 'unit.fabric.test_session', + 'unit.fabric.test_link', + 'unit.fabric.test_io', + 'unit.fabric.test_protocol', + 'unit.fabric.test_topics', + 'unit.fabric.test_service', + 'unit.fabric.test_transfer', + 'unit.fabric.test_transfer_sender', + 'unit.fabric.test_bridge', + 'unit.fabric.test_state', + 'unit.fabric.test_fabric', + 'unit.device.test_action_manager', + 'unit.device.test_catalogue', + 'unit.device.test_dependencies', + 'unit.device.test_model', + 'unit.device.test_phase2', + 'unit.device.test_publisher', + 'unit.device.test_service', + 'unit.device.test_static', + 'integration.devhost.device_public_seams_spec', + 'unit.update.test_active_runtime', + 'unit.update.test_architecture', + 'unit.update.test_bundled_probe', + 'unit.update.test_config', + 'unit.update.test_events', + 'unit.update.test_generation_refactor', + 'unit.update.test_ingest_artifacts', + 'unit.update.test_job_repository', + 'unit.update.test_job_runtime', + 'unit.update.test_job_store_memory', + 'unit.update.test_job_store_control_store', + 'unit.update.test_manager_requests', + 'unit.update.test_model', + 'unit.update.test_observe_reconcile', + 'unit.update.test_publisher', + 'unit.update.test_service', + 'unit.update.test_service_phase2', + 'unit.devicecode.blob_source_spec', + 'unit.devicecode.signal_bridge_spec', + 'unit.devicecode.exec_direct_usage_spec', + 'unit.devicecode.support_contracts_spec', + 'unit.shared.table_spec', + 'unit.shared.topic_spec', + 'unit.shared.validate_spec', + 'unit.shared.hash_xxhash32_spec', + 'integration.devhost.main_failure_spec', + 'integration.devhost.config_recovery_spec', + 'integration.devhost.monitor_logging_spec', + 'integration.devhost.startup_dependency_order_spec', + 'integration.devhost.dependency_uniformity_spec', + "integration.devhost.hal_uart_spec", + "integration.devhost.fabric_hal_uart_smoke_spec", + "integration.devhost.fabric_public_service_path_spec", + 'integration.devhost.update_public_seams_spec', + 'integration.devhost.mcu_update_full_path_spec', + 'integration.devhost.mcu_update_http_uart_spec', + 'integration.devhost.mcu_update_http_uart_fault_spec', + 'unit.support.test_capdeps', + 'unit.support.test_capability_dependencies', + 'unit.support.test_dependency_failure', + 'unit.support.test_dependency_slot', + 'unit.support.test_scoped_work', + 'unit.support.test_queue', + 'unit.support.test_priority_event', + 'unit.support.test_request_owner', + 'unit.support.test_resource', + 'unit.support.test_bus_cleanup', + 'unit.support.test_config_watch', + 'unit.support.test_config_watch_architecture', + 'unit.support.test_service_events', + 'unit.support.test_retained_publish', + 'unit.http.transport.test_cqueues_driver', + 'unit.http.transport.test_lua_http', + 'unit.http.transport.test_legacy_http1_close', + 'unit.http.transport.test_websocket', + 'unit.http.transport.test_terminate', + 'unit.http.test_headers', + 'unit.http.test_body', + 'unit.http.test_policy', + 'unit.http.test_config', + 'unit.http.test_client', + 'unit.http.test_websocket_client', + 'unit.http.test_service', + 'unit.http.test_service_architecture', + 'unit.http.test_structure', + 'integration.devhost.test_http_transport_real', + 'integration.devhost.test_http_service_real', + "unit.ui.test_architecture", + "unit.ui.test_config", + "unit.ui.test_http_listener", + "unit.ui.test_http_request", + "unit.ui.test_http_response_stream", + "unit.ui.test_read_model", + "unit.ui.test_service", + "unit.ui.test_sessions", + "unit.ui.test_supervision", + "unit.ui.test_update_upload", + "unit.ui.test_user_operation", + "unit.ui.test_local_ui", + "integration.devhost.local_ui_http_spec", + 'unit.metrics.processing_spec', + 'unit.metrics.config_spec', + 'unit.metrics.senml_spec', + 'unit.metrics.http_spec', + 'integration.devhost.metrics_spec', + "unit.gsm.test_apn_model", + "unit.gsm.test_apn_store_control_store", + "unit.gsm.test_apn", + "unit.gsm.test_state", + "unit.net.test_architecture", + "unit.net.test_config", + "unit.net.test_backhaul_model", + "unit.net.test_intent_realiser", + "unit.net.test_hal_client", + "unit.net.test_service_behaviour", + "unit.net.test_wan_runtime", + "unit.wired.test_config", + "unit.wired.test_dependencies", + "unit.wired.test_publisher", + "unit.wired.test_trunk_validation", + "unit.wired.test_architecture", + "integration.devhost.wired_assembly_projection_spec", + "integration.devhost.rtl8380m_switch_spec", + "unit.device.test_wired_provider_curation", + "unit.hal.modem_process_flags_spec", + "unit.hal.wired_provider_spec", +} + +local function monotonic_now() + if type(os.clock) == 'function' then return os.clock() end + return 0 +end + +local function sorted_keys(t) + local keys = {} + for k in pairs(t) do keys[#keys + 1] = k end + table.sort(keys, function(a, b) return tostring(a) < tostring(b) end) + return keys +end + +local function should_run(modname, testname, filter_text) + if not filter_text or filter_text == '' then return true end + local needle = string.lower(filter_text) + local hay = string.lower(('%s :: %s'):format(tostring(modname), tostring(testname))) + return string.find(hay, needle, 1, true) ~= nil +end + +local function format_duration_s(dt) + return ('%.3fs'):format(tonumber(dt) or 0) +end + +local TEST_FILTER = os.getenv('TEST_FILTER') or '' + +local total, failed, skipped = 0, 0, 0 +local failure_rows = {} + +for i = 1, #files do + local modname = files[i] + local mod = require(modname) + local keys = sorted_keys(mod) + + for j = 1, #keys do + local name = keys[j] + local fn = mod[name] + if type(fn) == 'function' then + if should_run(modname, name, TEST_FILTER) then + total = total + 1 + io.write(('[TEST] %s :: %s ... '):format(modname, name)) + local t0 = monotonic_now() + local ok, err = safe.xpcall(fn, function(e) + return debug.traceback(tostring(e), 2) + end) + local dt = monotonic_now() - t0 + if ok then + io.write('ok ' .. format_duration_s(dt) .. '\n') + else + failed = failed + 1 + failure_rows[#failure_rows + 1] = { + name = ('%s :: %s'):format(modname, name), + err = tostring(err), + dt = dt, + } + io.write('FAIL ' .. format_duration_s(dt) .. '\n') + io.write(tostring(err) .. '\n') + end + else + skipped = skipped + 1 + end + end + end +end + +io.write(('\n%d tests, %d failed, %d skipped\n'):format(total, failed, skipped)) +if #failure_rows > 0 then + io.write('\nFailure summary:\n') + for i = 1, #failure_rows do + local row = failure_rows[i] + local first = tostring(row.err):match('([^\n]+)') or tostring(row.err) + io.write((' - %s [%s]\n %s\n'):format(row.name, format_duration_s(row.dt), first)) + end +end +os.exit(failed == 0 and 0 or 1) diff --git a/tests/support/bus_probe.lua b/tests/support/bus_probe.lua new file mode 100644 index 00000000..abfaead4 --- /dev/null +++ b/tests/support/bus_probe.lua @@ -0,0 +1,238 @@ +-- tests/support/bus_probe.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' + +local M = {} + +local function fail(msg) + error(msg, 0) +end + +function M.wait_until(pred, opts) + opts = opts or {} + local timeout = opts.timeout or 1.0 + local interval = opts.interval or 0.005 + local deadline = fibers.now() + timeout + + while fibers.now() < deadline do + if pred() then + return true + end + sleep.sleep(interval) + end + + return pred() +end + + +function M.wait_changed_until(label, changed_op_fn, version_fn, pred, opts) + opts = opts or {} + local timeout = opts.timeout or 1.0 + local v = pred() + if v then return v end + + local seen = version_fn() + while true do + local which = fibers.perform(op.named_choice({ + changed = changed_op_fn(seen):wrap(function(version) + seen = version or seen + end), + timeout = sleep.sleep_op(timeout), + })) + if which == 'timeout' then + fail('timed out waiting for ' .. tostring(label)) + end + v = pred() + if v then return v end + end +end + +function M.wait_holder(label, holder, pred, opts) + return M.wait_changed_until(label, + function(seen) return holder:changed_op(seen) end, + function() return holder:version() end, + pred, + opts) +end + + +local function topic_label(topic) + local parts = {} + for i = 1, #(topic or {}) do parts[#parts + 1] = tostring(topic[i]) end + return table.concat(parts, '/') +end + +function M.wait_versioned_until(label, version_fn, changed_op_fn, pred, opts) + opts = opts or {} + local timeout = opts.timeout or 1.0 + + local v = pred() + if v then return v end + + local seen = version_fn() + while true do + local which, version, reason = fibers.perform(op.named_choice({ + changed = changed_op_fn(seen), + timeout = sleep.sleep_op(timeout), + })) + + if which == 'timeout' then + fail('timed out waiting for ' .. tostring(label)) + end + if version == nil then + fail(tostring(label) .. ' closed: ' .. tostring(reason or 'closed')) + end + + seen = version + v = pred() + if v then return v end + end +end + +local function retained_view_for(conn, topic, opts) + local view = opts.view + if view then return view, false end + + if type(conn.retained_view) ~= 'function' then + fail('retained_view support is required for retained-state assertions') + end + + return conn:retained_view(opts.view_topic or topic, opts.view_opts), true +end + +function M.wait_retained_message(conn, topic, opts) + opts = opts or {} + + local view, own_view = retained_view_for(conn, topic, opts) + local msg = M.wait_versioned_until( + 'retained topic ' .. topic_label(topic), + function() return view:version() end, + function(seen) return view:changed_op(seen) end, + function() return view:get(topic) end, + opts + ) + + if own_view then view:close() end + return msg +end + +function M.wait_retained_payload(conn, topic, opts) + local msg = M.wait_retained_message(conn, topic, opts) + return msg.payload, msg +end + +function M.wait_retained_absent(conn, topic, opts) + opts = opts or {} + + local view, own_view = retained_view_for(conn, topic, opts) + + local function absent() + if view:get(topic) == nil then return true end + return nil + end + + if opts.since == nil then + local v = absent() + if v then + if own_view then view:close() end + return v + end + end + + local timeout = opts.timeout or 1.0 + local seen = opts.since or view:version() + while true do + local which, version, reason = fibers.perform(op.named_choice({ + changed = view:changed_op(seen), + timeout = sleep.sleep_op(timeout), + })) + + if which == 'timeout' then + fail('timed out waiting for retained removal ' .. topic_label(topic)) + end + if version == nil then + fail('retained view closed: ' .. tostring(reason or 'closed')) + end + + seen = version + local v = absent() + if v then + if own_view then view:close() end + return v + end + end +end + +function M.wait_message(conn, topic, opts) + opts = opts or {} + local timeout = opts.timeout or 1.0 + local sub = opts.sub or conn:subscribe(topic, { + queue_len = opts.queue_len or 8, + full = opts.full or 'drop_oldest', + }) + local own_sub = (opts.sub == nil) + + local which, a, b = fibers.perform(op.named_choice({ + msg = sub:recv_op(), + timeout = sleep.sleep_op(timeout):wrap(function() + return true + end), + })) + + if own_sub then + sub:unsubscribe() + end + + if which == 'timeout' then + fail(('timed out waiting for topic %s'):format(tostring(topic[1] or '?'))) + end + + local msg, err = a, b + if not msg then + fail('subscription ended: ' .. tostring(err)) + end + return msg +end + +function M.wait_payload(conn, topic, opts) + local msg = M.wait_message(conn, topic, opts) + return msg.payload, msg +end + +function M.collect_messages(conn, topic, count, opts) + opts = opts or {} + local timeout = opts.timeout or 1.0 + local sub = conn:subscribe(topic, { + queue_len = opts.queue_len or math.max(count, 8), + full = opts.full or 'drop_oldest', + }) + + local out = {} + while #out < count do + local which, a, b = fibers.perform(op.named_choice({ + msg = sub:recv_op(), + timeout = sleep.sleep_op(timeout):wrap(function() + return true + end), + })) + + if which == 'timeout' then + sub:unsubscribe() + fail(('timed out collecting %d messages; got %d'):format(count, #out)) + end + + local msg, err = a, b + if not msg then + sub:unsubscribe() + fail('subscription ended: ' .. tostring(err)) + end + out[#out + 1] = msg + end + + sub:unsubscribe() + return out +end + +return M diff --git a/tests/support/dcmcu_fixture.lua b/tests/support/dcmcu_fixture.lua new file mode 100644 index 00000000..7dd2dec0 --- /dev/null +++ b/tests/support/dcmcu_fixture.lua @@ -0,0 +1,45 @@ +local cjson_ok, cjson = pcall(require, 'cjson.safe') +if not cjson_ok then cjson = require 'cjson' end + +local M = {} + +local function u16le(n) + return string.char(n % 256, math.floor(n / 256) % 256) +end + +local function u32le(n) + return string.char(n % 256, math.floor(n / 256) % 256, math.floor(n / 65536) % 256, math.floor(n / 16777216) % 256) +end + +function M.make(image_id, payload, opts) + opts = opts or {} + payload = payload or string.rep('payload-', 64) + local manifest = assert(cjson.encode({ + schema = 1, + component = 'mcu', + target = { + product_family = opts.product_family or 'bigbox', + hardware_profile = opts.hardware_profile or 'bb-v1-cm5-2', + mcu_board_family = opts.mcu_board_family or 'rp2354a', + }, + build = { + version = opts.version or '15.0', + build_id = opts.build_id or 'test-build', + image_id = image_id or 'mcu-image-new', + }, + payload = { + format = 'raw-bin', + length = #payload, + sha256 = opts.payload_sha256 or string.rep('0', 64), + }, + signing = { + key_id = opts.key_id or 'test-key', + sig_alg = 'ed25519', + }, + })) + local header_len = 32 + local header = 'DCMCUIMG' .. u16le(1) .. u16le(header_len) .. u32le(#manifest) .. string.rep('\0', header_len - 16) + return header .. manifest .. payload +end + +return M diff --git a/tests/support/device_diag.lua b/tests/support/device_diag.lua new file mode 100644 index 00000000..165e9f22 --- /dev/null +++ b/tests/support/device_diag.lua @@ -0,0 +1,34 @@ +local cjson = require 'cjson.safe' + +local M = {} + +local function enc(v) + local ok, s = pcall(cjson.encode, v) + if ok and s then return s end + return tostring(v) +end + +local function add(out, label, fn) + if not fn then return end + local ok, v = pcall(fn) + if ok then + out[#out + 1] = label .. '=' .. enc(v) + else + out[#out + 1] = label .. '=' + end +end + +function M.render(opts) + opts = opts or {} + local out = { '-- device diag --' } + add(out, 'service', opts.service_fn) + add(out, 'summary', opts.summary_fn) + add(out, 'components', opts.components_fn) + add(out, 'sources', opts.sources_fn) + add(out, 'component_cm5', opts.cm5_fn) + add(out, 'component_mcu', opts.mcu_fn) + add(out, 'extra', opts.extra_fn) + return table.concat(out, '\n') +end + +return M diff --git a/tests/support/diag_plugins/config.lua b/tests/support/diag_plugins/config.lua new file mode 100644 index 00000000..46ef888d --- /dev/null +++ b/tests/support/diag_plugins/config.lua @@ -0,0 +1,10 @@ +return { + name = 'config', + topic_groups = { + { label = 'cfg', topic = { 'cfg', '#' } }, + { label = 'ccmd', topic = { 'cmd', 'config', '#' } }, + { label = 'csvc', topic = { 'svc', 'config', '#' } }, + + + }, +} diff --git a/tests/support/diag_plugins/device.lua b/tests/support/diag_plugins/device.lua new file mode 100644 index 00000000..d5fd6fec --- /dev/null +++ b/tests/support/diag_plugins/device.lua @@ -0,0 +1,25 @@ +local device_diag = require 'tests.support.device_diag' + +return { + name = 'device', + topic_groups = { + { label = 'device', topic = { 'state', 'device', '#' } }, + { label = 'dcmd', topic = { 'cmd', 'device', '#' } }, + }, + section = function(helper, opts) + opts = opts or {} + local conn = opts.conn + + return { + render = function() + return device_diag.render(helper.shallow_merge({ + service_fn = opts.service_fn or opts.device_service_fn or (conn and helper.retained_fn(conn, { 'svc', 'device', 'status' }) or nil), + summary_fn = opts.summary_fn or opts.device_summary_fn or (conn and helper.retained_fn(conn, { 'state', 'device' }) or nil), + cm5_fn = opts.cm5_fn or (conn and helper.retained_fn(conn, { 'state', 'device', 'component', opts.cm5_component or 'cm5' }) or nil), + mcu_fn = opts.mcu_fn or (conn and helper.retained_fn(conn, { 'state', 'device', 'component', opts.mcu_component or 'mcu' }) or nil), + extra_fn = opts.extra_fn or opts.device_extra_fn, + }, opts)) + end, + } + end, +} diff --git a/tests/support/diag_plugins/obs.lua b/tests/support/diag_plugins/obs.lua new file mode 100644 index 00000000..63a7d2dd --- /dev/null +++ b/tests/support/diag_plugins/obs.lua @@ -0,0 +1,9 @@ +return { + name = 'obs', + topic_groups = { + + { label = 'obs', topic = { 'obs', '#' } }, + { label = 'svc', topic = { 'svc', '#' } }, + + }, +} diff --git a/tests/support/diag_plugins/rpc.lua b/tests/support/diag_plugins/rpc.lua new file mode 100644 index 00000000..c4eab078 --- /dev/null +++ b/tests/support/diag_plugins/rpc.lua @@ -0,0 +1,8 @@ +return { + name = 'rpc', + topic_groups = { + + + { label = 'rpc', topic = { 'rpc', '#' } }, + }, +} diff --git a/tests/support/diag_registry.lua b/tests/support/diag_registry.lua new file mode 100644 index 00000000..8e9ae295 --- /dev/null +++ b/tests/support/diag_registry.lua @@ -0,0 +1,10 @@ +return { + plugins = { + config = require 'tests.support.diag_plugins.config', + device = require 'tests.support.diag_plugins.device', + obs = require 'tests.support.diag_plugins.obs', + rpc = require 'tests.support.diag_plugins.rpc', + }, + profiles = { + }, +} diff --git a/tests/support/fake_control_store.lua b/tests/support/fake_control_store.lua new file mode 100644 index 00000000..6b4a5116 --- /dev/null +++ b/tests/support/fake_control_store.lua @@ -0,0 +1,147 @@ +-- tests/support/fake_control_store.lua +-- +-- Tiny in-memory stand-in for HAL's curated control-store capability. It binds +-- cap/control-store//rpc/{get,put,delete,list} on the bus and records calls +-- so devhost tests can prove UI -> GSM -> control-store paths are used without +-- requiring a real HAL backend or filesystem. + +local fibers = require 'fibers' + +local M = {} +local Store = {} +Store.__index = Store + +local METHODS = { 'get', 'put', 'delete', 'list' } + +local function t(...) return { ... } end + +local function cap_status_topic(id) + return t('cap', 'control-store', id, 'status') +end + +local function cap_meta_topic(id) + return t('cap', 'control-store', id, 'meta') +end + +local function cap_state_topic(id) + return t('cap', 'control-store', id, 'state') +end + +local function cap_rpc_topic(id, method) + return t('cap', 'control-store', id, 'rpc', method) +end + +local function copy(v) + if type(v) ~= 'table' then return v end + local out = {} + for k, sv in pairs(v) do out[k] = copy(sv) end + return out +end + +local function sorted_keys(values, prefix) + local out = {} + prefix = prefix or '' + for key in pairs(values or {}) do + if prefix == '' or tostring(key):sub(1, #prefix) == prefix then + out[#out + 1] = key + end + end + table.sort(out) + return out +end + +function Store:new(opts) + opts = opts or {} + return setmetatable({ + id = opts.id or 'gsm', + values = copy(opts.values or {}), + calls = {}, + endpoints = {}, + }, Store) +end + +function Store:_reply(method, payload) + payload = payload or {} + local key = payload.key + self.calls[#self.calls + 1] = { method = method, payload = copy(payload) } + + if method == 'get' then + if type(key) ~= 'string' or key == '' then return { ok = false, reason = 'invalid key' } end + local body = self.values[key] + if body == nil then return { ok = false, reason = 'not found' } end + return { ok = true, reason = body } + elseif method == 'put' then + if type(key) ~= 'string' or key == '' then return { ok = false, reason = 'invalid key' } end + if type(payload.data) ~= 'string' then return { ok = false, reason = 'data must be a string' } end + self.values[key] = payload.data + return { ok = true, reason = true } + elseif method == 'delete' then + if type(key) ~= 'string' or key == '' then return { ok = false, reason = 'invalid key' } end + self.values[key] = nil + return { ok = true, reason = true } + elseif method == 'list' then + return { ok = true, reason = sorted_keys(self.values, payload.prefix) } + end + return { ok = false, reason = 'unsupported method: ' .. tostring(method) } +end + +function Store:start(conn, opts) + opts = opts or {} + local id = opts.id or self.id or 'gsm' + local scope = opts.scope + self.id = id + + conn:retain(cap_state_topic(id), 'added') + conn:retain(cap_status_topic(id), { + schema = 'devicecode.cap.status/1', + state = 'available', + available = true, + methods = METHODS, + }) + conn:retain(cap_meta_topic(id), { + schema = 'devicecode.cap.meta/1', + class = 'control-store', + id = id, + offerings = { get = true, put = true, delete = true, list = true }, + methods = METHODS, + fake = true, + }) + + for _, method in ipairs(METHODS) do + local method_name = method + local ep = assert(conn:bind(cap_rpc_topic(id, method_name), { queue_len = 32 })) + self.endpoints[method_name] = ep + + local function serve() + while true do + local req = fibers.perform(ep:recv_op()) + if req == nil then return end + req:reply(self:_reply(method_name, req.payload or {})) + end + end + + if scope and type(scope.spawn) == 'function' then + local ok, err = scope:spawn(serve) + if ok ~= true then error(err or 'fake control-store spawn failed', 0) end + else + fibers.spawn(serve) + end + end + return true +end + +function Store:get(key) + return self.values[key] +end + +function Store:put(key, value) + self.values[key] = value + return true +end + +function M.new(opts) + return Store:new(opts) +end + +M.Store = Store +return M diff --git a/tests/support/fake_hal.lua b/tests/support/fake_hal.lua new file mode 100644 index 00000000..8b947f05 --- /dev/null +++ b/tests/support/fake_hal.lua @@ -0,0 +1,174 @@ +-- tests/support/fake_hal.lua + +local fibers = require 'fibers' + +local M = {} + +local function cap_status_topic(class, id) + return { 'cap', class, id, 'status' } +end + +local function cap_state_topic(class, id) + return { 'cap', class, id, 'state' } +end + +local function cap_meta_topic(class, id) + return { 'cap', class, id, 'meta' } +end + +local function cap_rpc_topic(class, id, method) + return { 'cap', class, id, 'rpc', method } +end + +local function strip_json_suffix(filename) + if type(filename) ~= 'string' then return nil end + return (filename:gsub('%.json$', '')) +end + +local function legacy_state_req_from_filename(filename, data) + return { + ns = 'config', + key = strip_json_suffix(filename), + data = data, + } +end + +local function map_cap_call(class, id, method, req) + if class == 'fs' and id == 'config' and method == 'read' then + return 'read_state', legacy_state_req_from_filename(req and req.filename) + end + if class == 'fs' and id == 'state' and method == 'write' then + return 'write_state', legacy_state_req_from_filename(req and req.filename, req and req.data) + end + return method, req +end + +local function map_cap_reply(method, reply) + if type(reply) ~= 'table' then return reply end + if method == 'read_state' then + local found = rawget(reply, 'found') + local data = rawget(reply, 'data') + local err_text = rawget(reply, 'err') + if reply.ok == true and found == true then + return { ok = true, reason = data } + end + if reply.ok == true and found == false then + return { ok = false, reason = err_text or 'not found', code = reply.code } + end + end + local err_text = rawget(reply, 'err') + if reply.ok ~= nil and reply.reason == nil and err_text ~= nil then + return { ok = reply.ok, reason = err_text, code = reply.code } + end + return reply +end + +local function method_offerings(method) + if method == 'read_state' then return 'fs', 'config', { read = true } end + if method == 'write_state' then return 'fs', 'state', { write = true } end + return nil, nil, nil +end + +---@class FakeHal +---@field calls table[] +---@field scripted table|table[] +---@field backend string +---@field caps table +local FakeHal = {} +FakeHal.__index = FakeHal + +function FakeHal:new(opts) + opts = opts or {} + return setmetatable({ + calls = {}, + scripted = opts.scripted or {}, + backend = opts.backend or 'fakehal', + caps = opts.caps or {}, + }, FakeHal) +end + +function FakeHal:_next_reply(method, req, request) + local entry = self.scripted[method] + if type(entry) == 'function' then return entry(req, request) end + if type(entry) == 'table' and #entry > 0 then + local v = table.remove(entry, 1) + if type(v) == 'function' then return v(req, request) end + return v + end + return { ok = false, err = 'no scripted reply for ' .. tostring(method) } +end + +function FakeHal:_record_call(method, req, request) + self.calls[#self.calls + 1] = { + method = method, + req = req, + request = request, + } +end + +function FakeHal:_start_capability_rpc(conn, class, id, offering, legacy_method) + conn:retain(cap_state_topic(class, id), 'added') + conn:retain(cap_status_topic(class, id), { + state = 'available', + available = true, + }) + conn:retain(cap_meta_topic(class, id), { offerings = { [offering] = true } }) + + local ep = conn:bind(cap_rpc_topic(class, id, offering), { queue_len = 16 }) + + fibers.spawn(function() + while true do + local req, err = ep:recv() + if not req then return end + + local raw_req = req.payload or {} + local method, mapped_req = map_cap_call(class, id, offering, raw_req) + method = legacy_method or method + + self:_record_call(method, mapped_req, req) + + local reply = self:_next_reply(method, mapped_req, req) + reply = map_cap_reply(method, reply) + if type(reply) ~= 'table' then + req:fail(reply) + elseif reply.ok == true then + req:reply(reply) + else + req:reply(reply) + end + end + end) +end + +function FakeHal:_start_capabilities(conn) + local started = {} + for method, enabled in pairs(self.caps) do + if enabled then + local class, id, offerings = method_offerings(method) + if class and id and offerings then + local key = class .. '\0' .. tostring(id) + if not started[key] then started[key] = true end + for offering in pairs(offerings) do + self:_start_capability_rpc(conn, class, id, offering, method) + end + end + end + end +end + +function FakeHal:start(conn, opts) + opts = opts or {} + local name = opts.name or 'hal' + conn:retain({ 'svc', name, 'announce' }, { + role = 'hal', + backend = self.backend, + caps = self.caps, + }) + self:_start_capabilities(conn) +end + +function M.new(opts) + return FakeHal:new(opts) +end + +return M diff --git a/tests/support/local_ui_devhost.lua b/tests/support/local_ui_devhost.lua new file mode 100644 index 00000000..9063f39d --- /dev/null +++ b/tests/support/local_ui_devhost.lua @@ -0,0 +1,316 @@ +-- tests/support/local_ui_devhost.lua +-- +-- Shared devhost harness for the initial local UI port. It composes the real +-- HTTP, UI and GSM services over an in-process bus, with a fake HAL +-- control-store capability for durable APN tests and demos. + +local busmod = require 'bus' +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local exec = require 'fibers.io.exec' +local cjson = require 'cjson.safe' +local safe = require 'coxpcall' + +local http_service = require 'services.http.service' +local ui_service = require 'services.ui.service' +local gsm_service = require 'services.gsm' +local fake_control_store_mod = require 'tests.support.fake_control_store' +local probe = require 'tests.support.bus_probe' + +local M = {} + +local function assert_spawn(scope, fn, label) + local ok, err = scope:spawn(fn) + if ok ~= true then error((label or 'spawn failed') .. ': ' .. tostring(err), 0) end + return true +end + +local function topic_key(parts) + local out = {} + for i = 1, #(parts or {}) do out[#out + 1] = tostring(parts[i]) end + return table.concat(out, '/') +end + +local function dirname(path) + path = tostring(path or ''):gsub('\\', '/') + return path:match('^(.*)/[^/]*$') or '.' +end + +local function join_path(a, b) + b = tostring(b or ''):gsub('\\', '/') + if b:sub(1, 1) == '/' then return b end + a = tostring(a or ''):gsub('\\', '/') + if a == '' or a == '.' then return b end + return a .. '/' .. b +end + +local function normalise_path(path) + path = tostring(path or ''):gsub('\\', '/') + local absolute = path:sub(1, 1) == '/' + local parts = {} + for part in path:gmatch('[^/]+') do + if part == '' or part == '.' then + -- skip + elseif part == '..' then + if #parts > 0 and parts[#parts] ~= '..' then + parts[#parts] = nil + elseif not absolute then + parts[#parts + 1] = '..' + end + else + parts[#parts + 1] = part + end + end + local out = table.concat(parts, '/') + if absolute then return '/' .. out end + return out ~= '' and out or '.' +end + +local function file_exists(path) + local f = io.open(path, 'r') + if f then f:close(); return true end + return false +end + +local function repo_root_from_source() + local source = debug.getinfo(1, 'S').source or '' + source = source:gsub('^@', ''):gsub('\\', '/') + return normalise_path(join_path(dirname(source), '../..')) +end + +local REPO_ROOT = repo_root_from_source() + +function M.resolve_static_root(root) + root = tostring(root or 'src/services/ui/www') + if root:sub(1, 1) == '/' then return root end + local candidates = { + root, + join_path(REPO_ROOT, root), + join_path(REPO_ROOT, 'tests/' .. root), + join_path(REPO_ROOT, 'src/services/ui/www'), + } + for _, candidate in ipairs(candidates) do + candidate = normalise_path(candidate) + if file_exists(candidate .. '/index.html') then return candidate end + end + return root +end + +function M.ui_cfg(port, root) + return { + schema = 'devicecode.config/ui/1', + enabled = true, + http = { + enabled = true, + cap_id = 'main', + host = '127.0.0.1', + port = port, + max_active_requests = 8, + }, + static = { + root = root or 'src/services/ui/www', + index = 'index.html', + chunk_size = 16384, + }, + sse = { enabled = true, replay = true, queue_len = 16 }, + sessions = { prune_interval = false }, + updates = { + upload = { + enabled = false, + max_bytes = 0, + require_auth = false, + component = 'mcu', + create_job = false, + start_job = false, + }, + commit = { require_auth = false }, + }, + } +end + +function M.gsm_cfg(store_id) + return { + schema = 'devicecode.config/gsm/1', + apn_store = { + kind = 'control-store', + id = store_id or 'gsm', + key = 'custom-apns-v1', + }, + modems = { + default = { enabled = false, signal_freq = 60 }, + known = {}, + }, + } +end + +local function retain_config(conn, service, data, rev) + conn:retain({ 'cfg', service }, { + schema = 'devicecode.config.snapshot/1', + service = service, + rev = rev or 1, + data = data, + }) +end + +function M.publish_demo_state(conn) + conn:retain({ 'state', 'device', 'identity' }, { + schema = 'devicecode.device.identity/1', + serial = 'BBX-DEMO-0001', + model = 'Big Box devhost', + }) + conn:retain({ 'state', 'device', 'components' }, { + cm5 = { id = 'cm5', label = 'CM5', state = 'running' }, + mcu = { id = 'mcu', label = 'MCU', state = 'running' }, + }) + conn:retain({ 'state', 'net', 'summary' }, { + schema = 'devicecode.net.summary/1', + state = 'running', + wan = 'cellular', + }) + conn:retain({ 'state', 'net', 'wan_runtime' }, { + schema = 'devicecode.net.wan-runtime/1', + selected = 'gsm-main', + internet = true, + }) + conn:retain({ 'state', 'system', 'stats' }, { + schema = 'devicecode.system.stats/1', + state = 'ok', + cpu = { utilisation = 12.5 }, + }) + conn:retain({ 'state', 'fabric', 'summary' }, { + schema = 'devicecode.fabric.summary/1', + state = 'running', + links = 0, + }) + conn:retain({ 'raw', 'host', 'secret' }, { should_not = 'be in local-ui bootstrap' }) + conn:retain({ 'cfg', 'secret' }, { should_not = 'be in local-ui bootstrap' }) +end + +function M.start(scope, opts) + opts = opts or {} + local port = opts.port or tonumber(os.getenv('LOCAL_UI_DEVHOST_PORT')) or 18089 + local static_root = M.resolve_static_root(opts.static_root or 'src/services/ui/www') + local bus = opts.bus or busmod.new() + local control_store = opts.control_store or fake_control_store_mod.new({ id = opts.store_id or 'gsm' }) + + local config_conn = bus:connect({ origin_base = { kind = 'test', service = 'config' } }) + control_store:start(bus:connect({ origin_base = { kind = 'test', service = 'fake-hal-control-store' } }), { + id = opts.store_id or 'gsm', + scope = scope, + }) + retain_config(config_conn, 'gsm', M.gsm_cfg(opts.store_id or 'gsm'), 1) + retain_config(config_conn, 'ui', M.ui_cfg(port, static_root), 1) + if opts.demo_state ~= false then M.publish_demo_state(config_conn) end + + assert_spawn(scope, function (service_scope) + http_service.run(service_scope, { + conn = bus:connect({ origin_base = { service = 'http' } }), + id = 'main', + backend_timeout = 2, + connection_setup_timeout = 2, + intra_stream_timeout = 2, + max_accept_queue = 32, + }) + end, 'http service') + + assert_spawn(scope, function () + gsm_service.start(bus:connect({ origin_base = { service = 'gsm' } }), { + name = 'gsm', + env = 'dev', + heartbeat_s = 60, + }) + end, 'gsm service') + + assert_spawn(scope, function (service_scope) + ui_service.run(service_scope, { + conn = bus:connect({ origin_base = { service = 'ui' } }), + bus = bus, + connect = function () return bus:connect({ origin_base = { service = 'ui-request' } }) end, + service_id = 'ui', + config = M.ui_cfg(port, static_root), + read_model_opts = { queue_len = 128 }, + http_call_opts = { timeout = 3 }, + command_timeout = 3, + apn_timeout = 3, + encode_json = function (value) + local encoded, err = cjson.encode(value) + if type(encoded) ~= 'string' then error(err or 'json_encode_failed', 0) end + return encoded + end, + }) + end, 'ui service') + + local probe_conn = bus:connect({ origin_base = { kind = 'test', service = 'probe' } }) + probe.wait_retained_payload(probe_conn, { 'cap', 'http', 'main', 'status' }, { timeout = 2 }) + probe.wait_retained_payload(probe_conn, { 'cap', 'gsm', 'main', 'status' }, { timeout = 2 }) + + return { + bus = bus, + port = port, + base_url = ('http://127.0.0.1:%d'):format(port), + control_store = control_store, + config_conn = config_conn, + } +end + +local function parse_curl_output(out) + out = tostring(out or '') + local status = out:match('\n__HTTP_STATUS__:(%d+)%s*$') + local body = out:gsub('\n__HTTP_STATUS__:%d+%s*$', '') + return status, body +end + +function M.curl(args) + local cmd = exec.command('curl', unpack(args)) + local out, st, code, sig, err = fibers.perform(cmd:combined_output_op()) + if not (st == 'exited' and code == 0) then + error(('curl failed: status=%s code=%s signal=%s err=%s output=%s'):format( + tostring(st), tostring(code), tostring(sig), tostring(err), tostring(out) + ), 0) + end + return parse_curl_output(out) +end + +function M.curl_json(method, url, payload) + local args = { + '--silent', '--show-error', '--max-time', '5', + '--write-out', '\n__HTTP_STATUS__:%{http_code}', + '--request', method, + } + if payload ~= nil then + args[#args + 1] = '--header' + args[#args + 1] = 'Content-Type: application/json' + args[#args + 1] = '--data-binary' + args[#args + 1] = assert(cjson.encode(payload)) + end + args[#args + 1] = url + local status, body = M.curl(args) + local decoded = nil + if body ~= '' then + local err + decoded, err = cjson.decode(body) + if decoded == nil then + error(('expected JSON response from %s %s, got status=%s body=%q decode_err=%s'):format( + tostring(method), tostring(url), tostring(status), tostring(body), tostring(err) + ), 2) + end + end + return status, decoded, body +end + +function M.wait_http_ready(base_url, opts) + opts = opts or {} + local ok = probe.wait_until(function () + local success, ready = safe.pcall(function () + local st = M.curl({ '--silent', '--show-error', '--max-time', '2', '--output', '/dev/null', '--write-out', '\n__HTTP_STATUS__:%{http_code}', base_url .. '/api/local-ui/bootstrap' }) + return st == '200' + end) + return success == true and ready == true + end, { timeout = opts.timeout or 4, interval = 0.05 }) + if not ok then error('local UI HTTP endpoint did not become ready: ' .. tostring(base_url), 0) end + return true +end + +function M.topic_key(parts) return topic_key(parts) end + +return M diff --git a/tests/support/pty.lua b/tests/support/pty.lua new file mode 100644 index 00000000..b7545f48 --- /dev/null +++ b/tests/support/pty.lua @@ -0,0 +1,250 @@ +-- tests/support/pty.lua + +local posix = require 'posix' +local file = require 'fibers.io.file' +local exec = require 'fibers.io.exec' +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local safe = require 'coxpcall' + +local perform = fibers.perform +local M = {} + +local function close_fd(fd) + local fn = posix.close + if type(fn) ~= 'function' then + local ok, unistd = pcall(require, 'posix.unistd') + if ok and type(unistd.close) == 'function' then + fn = unistd.close + end + end + if type(fn) == 'function' and fd ~= nil then + pcall(function() fn(fd) end) + end +end + +local function close_stream_best_effort(stream) + if not (stream and stream.close_op) then + return true, nil + end + + local ok, a, b = safe.pcall(function() + return perform(stream:close_op()) + end) + if not ok then + return nil, tostring(a) + end + return a, b +end + +local function wait_read_some(stream, max_n, timeout_s) + local which, a, b = perform(op.named_choice({ + data = stream:read_some_op(max_n), + timeout = sleep.sleep_op(timeout_s or 1.0):wrap(function() + return true + end), + })) + + if which == 'timeout' then + return nil, 'timeout' + end + + return a, b +end + +local function write_all(stream, data) + local off = 1 + while off <= #data do + local n, err = perform(stream:write_op(data:sub(off))) + if n == nil then + return nil, err + end + off = off + n + end + return true, nil +end + +local function read_exact(stream, nbytes, timeout_s) + local deadline = fibers.now() + (timeout_s or 1.0) + local parts = {} + local got = 0 + + while got < nbytes do + local remain = deadline - fibers.now() + if remain <= 0 then + return nil, 'timeout' + end + + local chunk, err = wait_read_some(stream, nbytes - got, remain) + if chunk == nil then + return nil, err + end + + parts[#parts + 1] = chunk + got = got + #chunk + end + + return table.concat(parts), nil +end + +local function stty_raw_noecho(path) + local cmd = exec.command('stty', '-F', tostring(path), 'raw', '-echo') + local out, st, code, sig, err = perform(cmd:combined_output_op()) + + if st == 'exited' and code == 0 then + return true, nil + end + + local detail = err or out or ('status=' .. tostring(st)) + if st == 'exited' then + detail = tostring(detail) .. ' (exit ' .. tostring(code) .. ')' + elseif st == 'signalled' then + detail = tostring(detail) .. ' (signal ' .. tostring(sig) .. ')' + end + return nil, detail +end + +---@class TestPTY +---@field master Stream +---@field slave_name string +---@field _anchor_fd integer|nil +local PTY = {} +PTY.__index = PTY + +function PTY:write(data) + local ok, err = write_all(self.master, data) + if ok ~= true then + return nil, err + end + return true, nil +end + +function PTY:read_some(max_n, timeout_s) + return wait_read_some(self.master, max_n, timeout_s) +end + +function PTY:expect_some(max_n, timeout_s, label) + local data, err = self:read_some(max_n, timeout_s) + if data == nil then + error(('%s timed out: %s'):format(label or 'PTY read', tostring(err)), 0) + end + return data +end + +function PTY:expect_exact(nbytes, timeout_s, label) + local data, err = read_exact(self.master, nbytes, timeout_s) + if data == nil then + error(('%s timed out: %s'):format(label or 'PTY read exact', tostring(err)), 0) + end + return data +end + +function PTY:expect_no_data(timeout_s, label) + local data, _err = self:read_some(4096, timeout_s or 0.10) + if data ~= nil then + error(('%s unexpectedly received data: %q'):format(label or 'PTY read', tostring(data)), 0) + end + return true +end + +function PTY:close() + if self.master then + close_stream_best_effort(self.master) + self.master = nil + end + + if self._anchor_fd ~= nil then + close_fd(self._anchor_fd) + self._anchor_fd = nil + end + + return true +end + +---@param scope Scope|nil +---@return TestPTY +function M.open(scope) + local master_fd, slave_fd, slave_name, err = posix.openpty() + if not master_fd then + error('openpty failed: ' .. tostring(slave_fd or err), 0) + end + + -- Put the slave side into raw/noecho mode up front so code under test + -- that opens slave_name directly sees predictable byte-stream behaviour. + local ok_raw, err_raw = stty_raw_noecho(slave_name) + if ok_raw ~= true then + close_fd(master_fd) + close_fd(slave_fd) + error('failed to configure PTY slave raw mode: ' .. tostring(err_raw), 0) + end + + local master = file.fdopen(master_fd, 'r+', 'pty-master:' .. tostring(slave_name)) + master:setvbuf('no') + + local rec = setmetatable({ + master = master, + slave_name = slave_name, + _anchor_fd = slave_fd, + }, PTY) + + if scope and scope.finally then + scope:finally(function() + rec:close() + end) + end + + return rec +end + +function M.open_slave_stream(path, opts) + opts = opts or {} + + if opts.raw ~= false then + local ok, err = stty_raw_noecho(path) + if ok ~= true then + return nil, 'stty failed: ' .. tostring(err) + end + end + + local s, err = file.open(path, 'r+') + if not s then + return nil, err + end + + pcall(function() + if s.setvbuf then s:setvbuf('no') end + end) + + return s, nil +end + +function M.read_some(stream, max_n, timeout_s) + return wait_read_some(stream, max_n, timeout_s) +end + +function M.expect_some(stream, max_n, timeout_s, label) + local data, err = wait_read_some(stream, max_n, timeout_s) + if data == nil then + error(('%s timed out: %s'):format(label or 'stream read', tostring(err)), 0) + end + return data +end + +function M.expect_exact(stream, nbytes, timeout_s, label) + local data, err = read_exact(stream, nbytes, timeout_s) + if data == nil then + error(('%s timed out: %s'):format(label or 'stream read exact', tostring(err)), 0) + end + return data +end + +function M.expect_no_data(stream, timeout_s, label) + local data, _err = wait_read_some(stream, 4096, timeout_s or 0.10) + if data ~= nil then + error(('%s unexpectedly received data: %q'):format(label or 'stream read', tostring(data)), 0) + end + return true +end + +return M diff --git a/tests/support/run_fibers.lua b/tests/support/run_fibers.lua new file mode 100644 index 00000000..600fbbe8 --- /dev/null +++ b/tests/support/run_fibers.lua @@ -0,0 +1,76 @@ +-- tests/support/run_fibers.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local pulse = require 'fibers.pulse' +local op = require 'fibers.op' + +local safe = require 'coxpcall' + +local M = {} + +function M.run(fn, opts) + opts = opts or {} + local timeout = opts.timeout or 2.0 + + local ok, err = safe.pcall(function() + fibers.run(function(root_scope) + local test_scope, cerr = root_scope:child() + if not test_scope then + error(cerr, 0) + end + + local done = pulse.new() + local done_ver = done:version() + local body_err = nil + + local ok_spawn, spawn_err = test_scope:spawn(function(s) + local ok_body, err_body = safe.pcall(function() + fn(s) + end) + + body_err = ok_body and nil or err_body + done:signal() + + if not ok_body then + error(err_body, 0) + end + end) + + if not ok_spawn then + error(spawn_err, 0) + end + + local ok_timer, timer_err = root_scope:spawn(function() + local completed = fibers.perform(op.boolean_choice( + done:changed_op(done_ver):wrap(function() return true end), + sleep.sleep_op(timeout):wrap(function() return false end) + )) + + if completed == false then + test_scope:cancel('test timeout') + error(('test timed out after %.3fs'):format(timeout), 0) + end + end) + + if not ok_timer then + error(timer_err, 0) + end + + done:changed(done_ver) + + test_scope:cancel('test complete') + fibers.perform(test_scope:join_op()) + + if body_err then + error(body_err, 0) + end + end) + end) + + if not ok then + error(err, 0) + end +end + +return M diff --git a/tests/support/stack_diag.lua b/tests/support/stack_diag.lua new file mode 100644 index 00000000..d4f185c6 --- /dev/null +++ b/tests/support/stack_diag.lua @@ -0,0 +1,197 @@ +-- tests/support/stack_diag.lua +-- +-- Small diagnostics helper for stack/integration tests. + +local cjson = require 'cjson.safe' +local fibers = require 'fibers' + +local safe = require 'coxpcall' + +local M = {} + +local function encode_one(v) + local ok, s = safe.pcall(cjson.encode, v) + if ok and s then + return s + end + return tostring(v) +end + +local function topic_str(topic) + if type(topic) ~= 'table' then + return tostring(topic) + end + local parts = {} + for i = 1, #topic do + parts[i] = tostring(topic[i]) + end + return table.concat(parts, '/') +end + +local function push_limited(t, v, maxn) + t[#t + 1] = v + if #t > maxn then + table.remove(t, 1) + end +end + +---@param scope Scope +---@param bus Bus +---@param specs table[] -- { { label = "obs", topic = {"obs", "#"} }, ... } +---@param opts? { max_records?: integer } +---@return table recorder +function M.start(scope, bus, specs, opts) + opts = opts or {} + local max_records = opts.max_records or 200 + + local rec = { + records = {}, + errors = {}, + } + + local conn = bus:connect() + + local function append_record(r) + push_limited(rec.records, r, max_records) + end + + local function append_error(s) + push_limited(rec.errors, s, max_records) + end + + for i = 1, #specs do + local spec = specs[i] + local label = spec.label or ('sub' .. tostring(i)) + local topic = assert(spec.topic, 'stack_diag: missing spec.topic') + + local ok_spawn, err = scope:spawn(function() + local sub = conn:subscribe(topic, { + queue_len = 128, + full = 'drop_oldest', + }) + + append_record({ + t = fibers.now(), + label = label, + kind = 'subscribed', + topic = topic, + }) + + while true do + local msg, rerr = sub:recv() + if not msg then + append_error(('[%s] subscription ended: %s'):format(label, tostring(rerr))) + return + end + + append_record({ + t = fibers.now(), + label = label, + kind = 'msg', + topic = msg.topic, + payload = msg.payload, + id = msg.id, + }) + end + end) + + if not ok_spawn then + append_error(('[%s] failed to spawn recorder: %s'):format(label, tostring(err))) + end + end + + return rec +end + +---@param rec table +---@param opts? { max_records?: integer } +---@return string +function M.render(rec, opts) + opts = opts or {} + local max_records = opts.max_records or 120 + + local out = {} + + local records = rec.records or {} + local start_idx = math.max(1, #records - max_records + 1) + + out[#out + 1] = ('records=%d errors=%d'):format(#records, #(rec.errors or {})) + + if rec.errors and #rec.errors > 0 then + out[#out + 1] = '-- recorder errors --' + for i = 1, #rec.errors do + out[#out + 1] = tostring(rec.errors[i]) + end + end + + out[#out + 1] = '-- bus trace --' + for i = start_idx, #records do + local r = records[i] + if r.kind == 'subscribed' then + out[#out + 1] = ('[%0.6f] %-10s subscribed %s'):format( + tonumber(r.t) or 0, + tostring(r.label), + topic_str(r.topic) + ) + else + out[#out + 1] = ('[%0.6f] %-10s %s -> %s'):format( + tonumber(r.t) or 0, + tostring(r.label), + topic_str(r.topic), + encode_one(r.payload) + ) + end + end + + return table.concat(out, '\n') +end + +---@param fake_hal any +---@param opts? { max_calls?: integer } +---@return string +function M.render_fake_hal(fake_hal, opts) + opts = opts or {} + local max_calls = opts.max_calls or 80 + + local calls = (fake_hal and fake_hal.calls) or {} + local start_idx = math.max(1, #calls - max_calls + 1) + + local out = {} + out[#out + 1] = ('fake_hal.calls=%d'):format(#calls) + out[#out + 1] = '-- fake HAL calls --' + + for i = start_idx, #calls do + local c = calls[i] + out[#out + 1] = ('[%d] %s req=%s'):format( + i, + tostring(c.method), + encode_one(c.req) + ) + end + + return table.concat(out, '\n') +end + +---@param message string +---@param rec table +---@param fake_hal any +---@return string +function M.explain(message, rec, fake_hal) + local parts = { + tostring(message), + '', + M.render(rec), + '', + M.render_fake_hal(fake_hal), + } + return table.concat(parts, '\n') +end + +---@param rec table +---@param opts? { max_records?: integer } +---@return string +function M.render_records(rec, opts) + return M.render(rec, opts) +end + +return M diff --git a/tests/support/test_diag.lua b/tests/support/test_diag.lua new file mode 100644 index 00000000..6cc6cf73 --- /dev/null +++ b/tests/support/test_diag.lua @@ -0,0 +1,311 @@ +local cjson = require 'cjson.safe' +local probe = require 'tests.support.bus_probe' +local stack_diag = require 'tests.support.stack_diag' +local registry = require 'tests.support.diag_registry' + +local M = {} +M.__index = M + +local function encode_one(v) + local ok, s = pcall(cjson.encode, v) + if ok and s then return s end + return tostring(v) +end + +local function value_key(v) + return encode_one(v) +end + +function M.shallow_merge(a, b) + local out = {} + if type(a) == 'table' then + for k, v in pairs(a) do out[k] = v end + end + if type(b) == 'table' then + for k, v in pairs(b) do out[k] = v end + end + return out +end + +local function render_calls(label, calls, opts) + opts = opts or {} + local max_calls = opts.max_calls or 80 + calls = calls or {} + local start_idx = math.max(1, #calls - max_calls + 1) + local out = {} + out[#out + 1] = ('%s=%d'):format(tostring(label), #calls) + out[#out + 1] = ('-- %s --'):format(tostring(label)) + for i = start_idx, #calls do + local c = calls[i] + if type(c) == 'table' then + out[#out + 1] = ('[%d] %s'):format(i, encode_one(c)) + else + out[#out + 1] = ('[%d] %s'):format(i, tostring(c)) + end + end + return table.concat(out, '\n') +end + +local function render_table(label, value) + return ('-- %s --\n%s'):format(tostring(label), encode_one(value)) +end + +local function plugin_named(name) + local plugin = registry.plugins[name] + assert(type(plugin) == 'table', 'unknown diag plugin: ' .. tostring(name)) + return plugin +end + +local function profile_named(name) + local profile = registry.profiles[name] + assert(type(profile) == 'function', 'unknown stack profile: ' .. tostring(name)) + return profile +end + +local function plugin_section(name, opts) + local plugin = plugin_named(name) + assert(type(plugin.section) == 'function', 'diag plugin has no section builder: ' .. tostring(name)) + return plugin.section(M, opts or {}) +end + +local function append_plugin_topics(out, name) + local plugin = plugin_named(name) + local groups = plugin.topic_groups or {} + for i = 1, #groups do + out[#out + 1] = groups[i] + end +end + +function M.add_section(diag, section) + diag.extra_sections[#diag.extra_sections + 1] = section + return diag +end + +function M.add_calls(diag, label, calls, opts) + diag.extra_sections[#diag.extra_sections + 1] = { label = label, calls = calls, opts = opts } + return diag +end + +function M.add_render(diag, label, fn) + diag.extra_sections[#diag.extra_sections + 1] = { + render = function() + return ('-- %s --\n%s'):format(tostring(label), tostring(fn())) + end, + } + return diag +end + +function M.add_table(diag, label, value_fn) + diag.extra_sections[#diag.extra_sections + 1] = { + render = function() + local value = (type(value_fn) == 'function') and value_fn() or value_fn + return render_table(label, value) + end, + } + return diag +end + +function M.add_subsystem(diag, kind, opts) + diag.extra_sections[#diag.extra_sections + 1] = plugin_section(kind, opts) + return diag +end + +function M.retained_fn(conn, topic, opts) + opts = opts or {} + return function() + local ok, payload = pcall(function() + return probe.wait_payload(conn, topic, { timeout = opts.timeout or 0.01 }) + end) + if ok then return payload end + return { __error = tostring(payload), topic = topic } + end +end + +function M.assert_true(diag, cond, message) + if not cond then diag:fail(message or 'assert_true failed') end + return true +end + +function M.assert_eq(diag, want, got, message) + if want ~= got then + diag:fail((message or 'assert_eq failed') .. ('\nwant=%s\ngot=%s'):format(encode_one(want), encode_one(got))) + end + return true +end + +function M.assert_match(diag, got, partial, message) + if type(got) ~= 'table' or type(partial) ~= 'table' then + diag:fail((message or 'assert_match failed') .. ('\nwant_partial=%s\ngot=%s'):format(encode_one(partial), encode_one(got))) + end + for k, v in pairs(partial) do + if got[k] ~= v then + diag:fail((message or 'assert_match failed') .. ('\nkey=%s\nwant=%s\ngot=%s\nfull=%s'):format(tostring(k), encode_one(v), encode_one(got[k]), encode_one(got))) + end + end + return true +end + +function M.assert_eventually_eq(diag, label, getter, want, opts) + local got + diag:assert_until((label or 'assert_eventually_eq failed') .. ('\nwant=%s\ngot=%s'):format(encode_one(want), encode_one(got)), function() + got = getter() + return got == want + end, opts) + return true +end + +function M.assert_no_event(diag, pred, window_s, message) + local ok = probe.wait_until(function() + return pred() == true + end, { timeout = window_s or 0.1, interval = 0.01 }) + if ok then + diag:fail(message or 'assert_no_event failed: observed forbidden event') + end + return true +end + +function M.start_profile(scope, bus, name, opts) + opts = opts or {} + local spec = profile_named(name)(M, opts) + local diag = M.for_stack(scope, bus, spec.stack or {}) + local plugins = spec.plugins or {} + + for i = 1, #plugins do + local item = plugins[i] + local merged_opts = M.shallow_merge(opts, item.opts or {}) + diag.extra_sections[#diag.extra_sections + 1] = plugin_section(item.name, merged_opts) + end + + if type(spec.sections) == 'table' then + for i = 1, #spec.sections do + diag.extra_sections[#diag.extra_sections + 1] = spec.sections[i] + end + end + if type(spec.extra_sections) == 'table' then + for i = 1, #spec.extra_sections do + diag.extra_sections[#diag.extra_sections + 1] = spec.extra_sections[i] + end + end + return diag +end + +M.for_stack_profile = M.start_profile + +function M.assert_transitions(diag, label, getter, expected, opts) + opts = opts or {} + local timeout = opts.timeout or 1.0 + local interval = opts.interval or 0.005 + local selector = opts.selector or function(v) return v end + local matches = opts.matches or function(got, want) return got == want end + local ignore_nil = (opts.ignore_nil ~= false) + local allow_other = (opts.allow_other ~= false) + local collapse = (opts.collapse ~= false) + local history = {} + local idx = 1 + local last_key = nil + + local function maybe_record(v) + local key = value_key(v) + if not collapse or key ~= last_key then + history[#history + 1] = v + last_key = key + end + end + + local ok = probe.wait_until(function() + local got_ok, raw = pcall(getter) + if not got_ok then return false end + local got = selector(raw) + if got == nil and ignore_nil then return false end + maybe_record(got) + if matches(got, expected[idx]) then + idx = idx + 1 + return idx > #expected + end + if not allow_other then + return false + end + return false + end, { timeout = timeout, interval = interval }) + + if not ok or idx <= #expected then + diag:fail((label or 'assert_transitions failed') + .. ('\nexpected=%s\nobserved=%s'):format(encode_one(expected), encode_one(history))) + end + + return history +end + +function M.assert_retained_transitions(diag, conn, topic, expected, opts) + opts = opts or {} + local getter = M.retained_fn(conn, topic, { timeout = opts.sample_timeout or 0.02 }) + return M.assert_transitions(diag, opts.label or ('retained transitions for ' .. encode_one(topic)), getter, expected, opts) +end + +function M.for_stack(scope, bus, opts) + opts = opts or {} + local topics = {} + for _, name in ipairs({ 'update', 'device', 'fabric', 'config', 'obs', 'rpc', 'ui' }) do + if opts[name] then append_plugin_topics(topics, name) end + end + if type(opts.topics) == 'table' then + for i = 1, #opts.topics do topics[#topics + 1] = opts.topics[i] end + end + return M.start(scope, bus, { + topics = topics, + max_records = opts.max_records, + fake_hal = opts.fake_hal, + extra_sections = opts.extra_sections, + }) +end + +function M.start(scope, bus, opts) + opts = opts or {} + local rec = stack_diag.start(scope, bus, opts.topics or {}, { max_records = opts.max_records }) + return setmetatable({ + rec = rec, + fake_hal = opts.fake_hal, + extra_sections = opts.extra_sections or {}, + }, M) +end + +function M:render(message) + local parts = { + tostring(message), + '', + stack_diag.render(self.rec), + } + if self.fake_hal then + parts[#parts + 1] = '' + parts[#parts + 1] = stack_diag.render_fake_hal(self.fake_hal) + end + for i = 1, #self.extra_sections do + local sec = self.extra_sections[i] + parts[#parts + 1] = '' + if type(sec) == 'function' then + parts[#parts + 1] = tostring(sec()) + elseif type(sec) == 'table' and sec.render then + parts[#parts + 1] = tostring(sec.render()) + elseif type(sec) == 'table' and sec.calls then + parts[#parts + 1] = render_calls(sec.label or ('calls' .. tostring(i)), sec.calls, sec.opts) + else + parts[#parts + 1] = tostring(sec) + end + end + return table.concat(parts, '\n') +end + +function M:fail(message) + error(self:render(message), 0) +end + +function M:assert_until(message, pred, opts) + if not probe.wait_until(pred, opts) then + self:fail(message) + end +end + +M.render_calls = render_calls +M.render_table = render_table + +return M diff --git a/tests/support/time_harness.lua b/tests/support/time_harness.lua new file mode 100644 index 00000000..84896cc7 --- /dev/null +++ b/tests/support/time_harness.lua @@ -0,0 +1,113 @@ +local fibers = require 'fibers' +local runtime = require 'fibers.runtime' + +local perform = fibers.perform +local pack = rawget(table, 'pack') or function(...) + return { n = select('#', ...), ... } +end +local unpack = rawget(table, 'unpack') or unpack + +---@class VirtualClock +---@field advance fun(self: VirtualClock, dt: number): number + +---@class TimeHarnessTickOpts +---@field max_ticks? integer + +---@class TimeHarnessWaitTicksOpts +---@field max_ticks? integer +---@field on_miss? fun(tick: integer) + +---@class TimeHarnessWaitWithinOpts +---@field step? number +---@field max_ticks? integer + +local M = {} +local NOT_READY = {} + +---@param op_or_factory any|fun(): any +---@return any +local function resolve_op(op_or_factory) + if type(op_or_factory) == 'function' then + return op_or_factory() + end + return op_or_factory +end + +---@param op_or_factory any|fun(): any +---@return boolean +---@return ... +function M.try_op_now(op_or_factory) + local op = resolve_op(op_or_factory) + local out = pack(perform(op:or_else(function() return NOT_READY end))) + if out.n == 1 and out[1] == NOT_READY then + return false + end + return true, unpack(out, 1, out.n) +end + +---@param max_ticks? integer +function M.flush_ticks(max_ticks) + max_ticks = max_ticks or 1 + for _ = 1, max_ticks do + runtime.yield() + end +end + +---@param op_or_factory any|fun(): any +---@param opts? TimeHarnessWaitTicksOpts +---@return boolean +---@return ... +function M.wait_op_ticks(op_or_factory, opts) + opts = opts or {} + + local max_ticks = opts.max_ticks or 1 + local on_miss = opts.on_miss + + for tick = 0, max_ticks do + local out = pack(M.try_op_now(op_or_factory)) + if out[1] then + return unpack(out, 1, out.n) + end + + if tick == max_ticks then break end + + if on_miss then + on_miss(tick + 1) + else + runtime.yield() + end + end + + return false +end + +---@param clock VirtualClock +---@param timeout_s number +---@param op_or_factory any|fun(): any +---@param opts? TimeHarnessWaitWithinOpts +---@return boolean +---@return ... +function M.wait_op_within(clock, timeout_s, op_or_factory, opts) + opts = opts or {} + + local step = opts.step or timeout_s + local max_ticks = opts.max_ticks or 1 + local elapsed = 0 + + while true do + local out = pack(M.try_op_now(op_or_factory)) + if out[1] then + return unpack(out, 1, out.n) + end + if elapsed >= timeout_s then + return false + end + + local advance = math.min(step, timeout_s - elapsed) + clock:advance(advance) + M.flush_ticks(max_ticks) + elapsed = elapsed + advance + end +end + +return M diff --git a/tests/support/virtual_time.lua b/tests/support/virtual_time.lua new file mode 100644 index 00000000..bfc9436f --- /dev/null +++ b/tests/support/virtual_time.lua @@ -0,0 +1,128 @@ +local runtime = require 'fibers.runtime' +local time = require 'fibers.utils.time' + +---@class VirtualTimeInstallOpts +---@field monotonic? number +---@field realtime? number +---@field follow_realtime? boolean + +---@class VirtualClock +---@field monotonic fun(self: VirtualClock): number +---@field realtime fun(self: VirtualClock): number +---@field set_monotonic fun(self: VirtualClock, value: number): number +---@field set_realtime fun(self: VirtualClock, value: number): number +---@field advance fun(self: VirtualClock, dt: number): number +---@field restore fun(self: VirtualClock) + +local M = {} + +---@type VirtualClock|nil +local active_clock = nil + +---@param value number|nil +---@param fallback number +---@return number +local function now_or(value, fallback) + if value ~= nil then return value end + return fallback +end + +---@param opts? VirtualTimeInstallOpts +---@return VirtualClock +function M.install(opts) + if active_clock then + error('virtual_time.install: a clock is already installed') + end + + opts = opts or {} + + local scheduler = runtime.current_scheduler + local original = { + monotonic = time.monotonic, + realtime = time.realtime, + block = time._block, + scheduler_time = scheduler.get_time, + wheel_now = scheduler.wheel.now, + } + + local state = { + scheduler = scheduler, + original = original, + monotonic = now_or(opts.monotonic, scheduler:now()), + realtime = now_or(opts.realtime, time.realtime()), + follow_realtime = opts.follow_realtime ~= false, + restored = false, + } + + local function monotonic_now() + return state.monotonic + end + + local function realtime_now() + return state.realtime + end + + time.monotonic = monotonic_now + time.realtime = realtime_now + time._block = function() + return true + end + + scheduler.get_time = monotonic_now + scheduler.wheel.now = state.monotonic + + local clock = {} + + function clock:monotonic() + return state.monotonic + end + + function clock:realtime() + return state.realtime + end + + function clock:set_monotonic(value) + assert(type(value) == 'number', 'virtual_time.set_monotonic: value must be a number') + assert(value >= state.monotonic, 'virtual_time.set_monotonic: cannot move time backwards') + state.monotonic = value + return state.monotonic + end + + function clock:set_realtime(value) + assert(type(value) == 'number', 'virtual_time.set_realtime: value must be a number') + state.realtime = value + return state.realtime + end + + function clock:advance(dt) + assert(type(dt) == 'number', 'virtual_time.advance: dt must be a number') + assert(dt >= 0, 'virtual_time.advance: dt must be non-negative') + state.monotonic = state.monotonic + dt + if state.follow_realtime then + state.realtime = state.realtime + dt + end + return state.monotonic + end + + function clock:restore() + if state.restored then return end + if active_clock ~= clock then + error('virtual_time.restore: attempted to restore a non-active clock') + end + + scheduler.get_time = original.scheduler_time + scheduler.wheel.now = original.wheel_now + time.monotonic = original.monotonic + time.realtime = original.realtime + time._block = original.block + + state.restored = true + active_clock = nil + end + + ---@cast clock VirtualClock + active_clock = clock + return clock +end + +return M diff --git a/tests/test.lua b/tests/test.lua deleted file mode 100644 index 6a1eed94..00000000 --- a/tests/test.lua +++ /dev/null @@ -1,41 +0,0 @@ -package.path = "../src/lua-fibers/?.lua;" -- fibers submodule src - .. "../src/lua-trie/src/?.lua;" -- trie submodule src - .. "../src/lua-bus/src/?.lua;" -- bus submodule src - .. "../src/?.lua;" - .. "./test_utils/?.lua;" - .. package.path - .. ";/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua" - -local base_pkg_path = package.path - -_G._TEST = true -- global test flag - -local tests = { - "submodules", - "service", - "hal_utils", - -- "modem_driver", - -- "modemcard_manager", - -- "hal_capabilities", - -- "hal", - "metrics", - "system", - "wifi" -} - -for _, test in ipairs(tests) do - dofile("test_" .. test .. ".lua") - package.path = base_pkg_path - -- package.loaded = nil -end - --- Run all accumulated tests -local fiber = require "fibers.fiber" -local luaunit = require "luaunit" - -fiber.spawn(function() - luaunit.LuaUnit.run() - fiber.stop() -end) - -fiber.main() diff --git a/tests/test_hal.lua b/tests/test_hal.lua deleted file mode 100644 index 6700727a..00000000 --- a/tests/test_hal.lua +++ /dev/null @@ -1,703 +0,0 @@ -local fiber = require "fibers.fiber" -local context = require "fibers.context" -local channel = require "fibers.channel" -local sleep = require "fibers.sleep" -local op = require "fibers.op" -local queue = require "fibers.queue" -local file = require "fibers.stream.file" -local json = require "dkjson" -local bus_pkg = require "bus" -local new_msg = bus_pkg.new_msg -local test_utils = require "test_utils.utils" -local assertions = require "test_utils.assertions" - --- loader makes sure that hal starts from a completely reset state for each test -local hal_loader = test_utils.new_module_loader() -hal_loader:add_uncacheable("services.hal.mmcli") -hal_loader:add_uncacheable("services.hal.modem_driver") -hal_loader:add_uncacheable("services.hal.modem_manager") -hal_loader:add_uncacheable("services.hal") - --- commmon contexts for tests -local function make_contexts() - local bg_ctx = context.background() - local service_ctx = context.with_cancel(bg_ctx) - local fiber_ctx = context.with_cancel(service_ctx) - - return service_ctx, fiber_ctx -end - --- template of a typical device event -local function mock_device_event() - local info_file, err = file.open('test_modemcard_manager/shims/monitor_modems/services/hal/modemcard_info.json', "r") - if err then return nil end - local info_data = info_file:read_all_chars() - local info = json.decode(info_data) - return { - connected = true, - type = 'usb', - capabilities = {}, - device_control = {}, - id_field = 'port', - data = { - device = 'modemcard', - port = '/sys/devices/platform/scb/fd500000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0/usb1/1-1/1-1.2', - info = info - } - } -end - -local function test_capability_valid_cap() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local expected_result = { result = 'success', error = nil } - - -- make a fake capability that returns a simple result - local caps = { - modem = { - control = { - enable = function() - return expected_result - end}, - info_streams = {} - } - } - - local device_event = mock_device_event() - -- override the capabilities - device_event.capabilities = caps - device_event_q:put(device_event) - - sleep.sleep(0.1) - - -- send request for the enable endpoint - local sub = bus_conn:request(new_msg({ "hal", "capability", "modem", 1, "control", "enable" }, {})) - - -- expect a response from hal with the expected output - local ret_msg, timeout = sub:next_msg(0.2) - - assert(timeout == nil, 'expected timeout to be nil') - assertions.assert_table(expected_result, ret_msg.payload, 'cap_result') - service_ctx:cancel('test-end') -end - --- ask for a capability that does not exist --- an example of a capability would be modem, time etc -local function test_capability_no_cap() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - sleep.sleep(0.1) - - local bus_conn = bus:connect() - - local expected_result = { result = nil, error = 'capability does not exist' } - - -- use foo as fake capability - local sub = bus_conn:request(new_msg({ "hal", "capability", "foo", 1, "control", "enable" }, {})) - - -- expect a response with no result and a capability error - local ret_msg, timeout = sub:next_msg(0.2) - - assert(timeout == nil, 'expected timeout to be nil') - assertions.assert_table(expected_result, ret_msg.payload, 'cap_result') - service_ctx:cancel('test-end') -end - --- use an instance index that does not exist -local function test_capability_no_inst() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local expected_result = { result = nil, error = 'capability instance does not exist' } - - local caps = { - modem = { - control = {}, - info_streams = {} - } - } - - local device_event = mock_device_event() - device_event.capabilities = caps - device_event_q:put(device_event) - - sleep.sleep(0.1) - - -- no devices with modem capability have been added yet - -- so modem instance 100 cannot exist - local sub = bus_conn:request(new_msg({ "hal", "capability", "modem", 100, "control", "enable" }, {})) - - -- expect a response with no result and a instance error - local ret_msg, timeout = sub:next_msg(0.2) - - assert(timeout == nil, 'expected timeout to be nil') - assertions.assert_table(expected_result, ret_msg.payload, 'cap_result') - service_ctx:cancel('test-end') -end - --- ask for a capability endpoint (or method) that does not exist -local function test_capability_no_method() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local expected_result = { result = nil, error = 'endpoint does not exist' } - - local caps = { - modem = { - control = {}, - info_streams = {} - } - } - - local device_event = mock_device_event() - device_event.capabilities = caps - device_event_q:put(device_event) - - sleep.sleep(0.1) - - -- a modem capability has been added with no enpoints so - -- enable will not exist - local sub = bus_conn:request(new_msg({ "hal", "capability", "modem", 1, "control", "enable" }, {})) - - -- expect a response with no result and an endpoint error - local ret_msg, timeout = sub:next_msg(0.2) - - assert(timeout == nil, 'expected timeout to be nil') - assertions.assert_table(expected_result, ret_msg.payload, 'cap_result') - service_ctx:cancel('test-end') -end - --- some capabilities may require arguments to be provided to their endpoints -local function test_capability_args() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local res1 = 'Hello ' - local res2 = "World" - local expected_result = { result = res1 .. res2, err = nil } - - -- create a fake capability that takes two strings - -- and concatenates them - local caps = { - modem = { - control = { - enable = function(self, args) - return { result = args[1] .. args[2], err = nil } - end - }, - info_streams = {} - } - } - - local device_event = mock_device_event() - device_event.capabilities = caps - device_event_q:put(device_event) - - sleep.sleep(0.1) - - -- arguments are supplied in the payload - local sub = bus_conn:request(new_msg({ "hal", "capability", "modem", 1, "control", "enable" }, { res1, res2 })) - - -- expect a result of the two arguments concatenated - local ret_msg, timeout = sub:next_msg(0.2) - - assert(timeout == nil, 'expected timeout to be nil') - assertions.assert_table(expected_result, ret_msg.payload, 'cap_result') - service_ctx:cancel('test-end') -end - -local function test_device_event_add() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - -- this is the message that is published on the bus - local expected_result = { - connected = true, - type = 'usb', - index = 1, - identity = 'modemcard' - } - - -- send a device connected event, the modem manager would be responsible for this - local device_event = mock_device_event() - device_event_q:put(device_event) - - local device_sub = bus_conn:subscribe({ "hal", "device", "usb", "#" }) - - -- expect a connection event on the bus - local device_msg, timeout = device_sub:next_msg(0.2) - - assert(timeout == nil, 'expected timeout to be nil') - assertions.assert_table(expected_result, device_msg.payload, 'cap_result') - service_ctx:cancel('test-end') -end - --- the same as above but we add and then remove a device -local function test_device_event_remove() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local expected_result = { - connected = false, - type = 'usb', - index = 1, - identity = 'modemcard' - } - - local device_event_connect = mock_device_event() - device_event_q:put(device_event_connect) - - local device_event_disconnect = mock_device_event() - device_event_disconnect.connected = false - device_event_q:put(device_event_disconnect) - - sleep.sleep(0.1) - - local device_sub = bus_conn:subscribe({ "hal", "device", "usb", "+" }) - - -- no need to worry about the connection message as it will be - -- overwritten by the disconnection message - local device_msg, timeout = device_sub:next_msg(0.2) - - assert(timeout == nil, 'expected timeout to be nil') - assertions.assert_table(expected_result, device_msg.payload, 'cap_result') - service_ctx:cancel('test-end') -end - -local function test_device_event_remove_no_exist() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - -- remove a device without adding one - local device_event_disconnect = mock_device_event() - device_event_disconnect.connected = false - device_event_q:put(device_event_disconnect) - - sleep.sleep(0.1) - - local device_sub = bus_conn:subscribe({ "hal", "device", "usb", "#" }) - - -- hal should not publish anything on the bus, therefore a timeout. A device error log should be printed - local device_msg, timeout = device_sub:next_msg(0.2) - - assert(timeout == 'Timeout', 'expected message to timeout') - assert(device_msg == nil, string.format('expected device message to be nil but got %s', device_msg)) - service_ctx:cancel('test-end') -end - --- a device event requires a type (e.g. usb), pass nil instead -local function test_device_event_nil_type() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local device_event = mock_device_event() - device_event.type = nil - device_event_q:put(device_event) - - sleep.sleep(0.1) - - local device_sub = bus_conn:subscribe({ "hal", "device", "usb", "#" }) - - -- hal should not publish anything on the bus, therefore a timeout. A type error log should be printed - local device_msg, timeout = device_sub:next_msg(0.2) - - assert(timeout == 'Timeout', 'expected message to timeout') - assert(device_msg == nil, string.format('expected device message to be nil but got %s', device_msg)) - service_ctx:cancel('test-end') -end - -local function test_device_event_nil_connected() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local device_event = mock_device_event() - device_event.connected = nil - device_event_q:put(device_event) - - sleep.sleep(0.1) - - local device_sub = bus_conn:subscribe({ "hal", "device", "usb", "#" }) - - -- hal should not publish anything on the bus, therefore a timeout. A connected field error log should be printed - local device_msg, timeout = device_sub:next_msg(0.2) - - assert(timeout == 'Timeout', 'expected message to timeout') - assert(device_msg == nil, string.format('expected device message to be nil but got %s', device_msg)) - service_ctx:cancel('test-end') -end - -local function test_device_event_nil_id_field() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local device_event = mock_device_event() - device_event.id_field = nil - device_event_q:put(device_event) - - sleep.sleep(0.1) - - local device_sub = bus_conn:subscribe({ "hal", "device", "usb", "#" }) - - -- hal should not publish anything on the bus, therefore a timeout. A identity error log should be printed - local device_msg, timeout = device_sub:next_msg(0.2) - - assert(timeout == 'Timeout', 'expected message to timeout') - assert(device_msg == nil, string.format('expected device message to be nil but got %s', device_msg)) - service_ctx:cancel('test-end') -end - -local function test_device_event_nil_data() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local device_event = mock_device_event() - device_event.data = nil - device_event_q:put(device_event) - - sleep.sleep(0.1) - - local device_sub = bus_conn:subscribe({ "hal", "device", "usb", "#" }) - - -- hal should not publish anything on the bus, therefore a timeout. A identity error log should be printed - local device_msg, timeout = device_sub:next_msg(0.2) - - assert(timeout == 'Timeout', 'expected message to timeout') - assert(device_msg == nil, string.format('expected device message to be nil but got %s', device_msg)) - service_ctx:cancel('test-end') -end - -local function test_device_event_nil_capabilities() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local device_event = mock_device_event() - device_event.capabilities = nil - device_event_q:put(device_event) - - sleep.sleep(0.1) - - local device_sub = bus_conn:subscribe({ "hal", "device", "usb", "#" }) - - -- hal should not publish anything on the bus, therefore a timeout. A identity error log should be printed - local device_msg, timeout = device_sub:next_msg(0.2) - - assert(timeout == 'Timeout', 'expected message to timeout') - assert(device_msg == nil, string.format('expected device message to be nil but got %s', device_msg)) - service_ctx:cancel('test-end') -end - -local function test_capability_event() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local event_channel = channel.new() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local device_event = mock_device_event() - device_event.capabilities = { - modem = { - control = {}, - info_streams = { { name = 'test_event', channel = event_channel, endpoints = 'single' } } - } - } - - local expected_event = { - test = 'event' - } - - device_event_q:put(device_event) - - sleep.sleep(0.1) - - local event_sub = bus_conn:subscribe({ "hal", "capability", "modem", 1, "info", "test_event" }) - - fiber.spawn(function() - event_channel:put(expected_event) - end) - - local event_msg, timeout = event_sub:next_msg(0.2) - - assert(timeout == nil, 'event not recieved') - assertions.assert_table(expected_event, event_msg.payload, 'event_result') - service_ctx:cancel('test-end') -end -local function test_device_event_nil_identifier() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal.control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local device_event = mock_device_event() - device_event.identifier = nil - device_event_q:put(device_event) - - sleep.sleep(0.1) - - local device_sub = bus_conn:subscribe({ "hal", "device", "usb", "#" }) - - -- hal should not publish anything on the bus, therefore a timeout. A identity error log should be printed - local device_msg, timeout = device_sub:next_msg(0.2) - - assert(timeout == 'Timeout', 'expected message to timeout') - assert(device_msg == nil, string.format('expected device message to be nil but got %s', device_msg)) - service_ctx:cancel('test-end') -end -local function test_device_info_valid() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal:control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local expected_result = { result = 0.8, error = nil } - - -- similar to capability, make a fake endpoint - local device_event = mock_device_event() - device_event.device_control = { - utilisation = function() - return 0.8 - end - } - device_event_q:put(device_event) - - sleep.sleep(0.1) - - local info_sub = bus_conn:request(new_msg({ "hal", "device", "usb", "1", "info", "utilisation" }, {})) - local info_response, timeout = info_sub:next_msg(0.2) - - assert(timeout == nil, 'expected timeout to be nil') - assertions.assert_table(expected_result, info_response.payload, 'cap_result') - service_ctx:cancel('test-end') -end - -local function test_device_info_no_device_type() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal.control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local expected_result = { result = nil, error = 'device type does not exist' } - - sleep.sleep(0.1) - - -- there are no supported devices called foo, so we expect an error response - local info_sub = bus_conn:request(new_msg({ "hal", "device", "foo", "1", "info", "utilisation" }, {})) - local info_response, timeout = info_sub:next_msg(0.2) - - assert(timeout == nil, 'expected timeout to be nil') - assertions.assert_table(expected_result, info_response.payload, 'cap_result') - service_ctx:cancel('test-end') -end - -local function test_device_info_no_device() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal.control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local expected_result = { result = nil, error = 'device does not exist' } - - sleep.sleep(0.1) - - -- no device has been added so usb instance 100 does not exist - local info_sub = bus_conn:request(new_msg({ "hal", "device", "usb", "100", "info", "utilisation" }, {})) - local info_response, timeout = info_sub:next_msg(0.2) - - assert(timeout == nil, 'expected timeout to be nil') - assertions.assert_table(expected_result, info_response.payload, 'cap_result') - service_ctx:cancel('test-end') -end - -local function test_device_info_no_method() - local hal = hal_loader:require("services.hal") - local service_ctx, fiber_ctx = make_contexts() - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local device_event_q = queue.new() - - fiber.spawn(function() - hal.control_main(fiber_ctx, bus:connect(), device_event_q) - end) - - local bus_conn = bus:connect() - - local expected_result = { result = nil, error = 'control method does not exist' } - - local device_event = mock_device_event() - device_event_q:put(device_event) - - sleep.sleep(0.1) - - -- asking for an info endpoint that does not exist - local info_sub = bus_conn:request(new_msg({ "hal", "device", "usb", "1", "info", "utilisation" }, {})) - local info_response, timeout = info_sub:next_msg(0.2) - - assert(timeout == nil, 'expected timeout to be nil') - assertions.assert_table(expected_result, info_response.payload, 'cap_result') - service_ctx:cancel('test-end') -end - -fiber.spawn(function() - -- main control fiber capability tests - test_capability_valid_cap() - test_capability_no_cap() - test_capability_no_inst() - test_capability_no_method() - test_capability_args() - - -- -- main control fiber device event tests - test_device_event_add() - test_device_event_remove() - test_device_event_remove_no_exist() - test_device_event_nil_type() - test_device_event_nil_connected() - test_device_event_nil_id_field() - test_device_event_nil_data() - test_device_event_nil_capabilities() - - test_capability_event() - -- -- main control fiber device info - -- not running this as kinda redundant for now - -- test_device_info_valid() - -- test_device_info_no_device_type() - -- test_device_info_no_device() - -- test_device_info_no_method() - - fiber.stop() -end) - -print("running hal tests") -fiber.main() -print("tests passed") diff --git a/tests/test_hal_capabilities.lua b/tests/test_hal_capabilities.lua deleted file mode 100644 index 1e261c38..00000000 --- a/tests/test_hal_capabilities.lua +++ /dev/null @@ -1,41 +0,0 @@ -local capabilities = require "services.hal.hal_capabilities" -local queue = require "fibers.queue" -local fiber = require "fibers.fiber" -local assertions = require "test_utils.assertions" -local context = require "fibers.context" - -local function test_capability() - local ctx = context.with_cancel(context.background()) - local test_q = queue.new() - local modem_cap = capabilities.new_modem_capability(test_q) - - local expected_result = { - result = 'somthing' - } - - fiber.spawn(function() - local result = modem_cap:enable() - assertions.assert_table(expected_result, result, "cap_return") - ctx:cancel() - end) - - local cmd = test_q:get() - local expected_cmd = { - command = "enable", - args = nil - } - - assertions.assert_table(expected_cmd, cmd, 'cmd') - - cmd.return_channel:put(expected_result) - ctx:done_op():perform() -end - -fiber.spawn(function() - test_capability() - fiber.stop() -end) - -print("running hal capabilities tests") -fiber.main() -print("tests passed") diff --git a/tests/test_hal_qmi.lua b/tests/test_hal_qmi.lua deleted file mode 100644 index d0cbd9df..00000000 --- a/tests/test_hal_qmi.lua +++ /dev/null @@ -1,212 +0,0 @@ -package.path = "../src/lua-fibers/?.lua;" -- fibers submodule src - .. "../src/lua-trie/src/?.lua;" -- trie submodule src - .. "../src/lua-bus/src/?.lua;" -- bus submodule src - .. "../src/?.lua;" - .. "./test_utils/?.lua;" - .. package.path - .. ";/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua" - -local fiber = require "fibers.fiber" -local context = require "fibers.context" - -local function test_uim_get_gids_simple() - -- Clear cached module - package.loaded["services.hal.drivers.modem.mode.qmi"] = nil - - -- Mock dependencies - local mock_context = { - with_timeout = function(ctx, timeout) - return ctx - end - } - - local mock_cmd = { - setpgid_called = false, - setpgid = function(self, val) - self.setpgid_called = true - end, - combined_output = function(self) - local output = [[ -[/dev/cdc-wdm1] Successfully read information from the UIM: -Card result: - SW1: '0x90' - SW2: '0x00' -Read result: - FF]] - return output, nil - end - } - - local mock_qmicli = { - uim_read_transparent = function(ctx, port, id) - return mock_cmd - end - } - - local mock_wraperr = { - new = function(err) return err end - } - - -- Load the qmi module with mocked dependencies - package.loaded["services.hal.drivers.modem.qmicli"] = mock_qmicli - package.loaded["fibers.context"] = mock_context - package.loaded["wraperr"] = mock_wraperr - package.loaded["services.hal.utils"] = require("services.hal.utils") - package.loaded["services.log"] = require("services.log") - - local qmi_module = require("services.hal.drivers.modem.mode.qmi") - - -- Create a mock modem object - local modem = { - ctx = context.background(), - primary_port = "/dev/cdc-wdm1" - } - - -- Initialize the modem with qmi functions - qmi_module(modem) - - -- Test the function - local gids, err = modem.uim_get_gids() - - assert(err == nil, "expected err to be nil but got " .. tostring(err)) - assert(gids ~= nil, "expected gids to be a table but got nil") - assert(gids.gid1 == "FF", "expected gid1 to be 'FF' but got '" .. tostring(gids.gid1) .. "'") - assert(mock_cmd.setpgid_called, "expected setpgid to be called") -end - -local function test_uim_get_gids_complex() - -- Clear cached module - package.loaded["services.hal.drivers.modem.mode.qmi"] = nil - - -- Mock dependencies - local mock_context = { - with_timeout = function(ctx, timeout) - return ctx - end - } - - local mock_cmd = { - setpgid_called = false, - setpgid = function(self, val) - self.setpgid_called = true - end, - combined_output = function(self) - local output = [[ -[/dev/cdc-wdm1] Successfully read information from the UIM: -Card result: - SW1: '0x90' - SW2: '0x00' -Read result: - 85:FF:FF:FF:FF:FF:47:45:4E:49:45:49:4E:20:20:20:20:20:20:20]] - return output, nil - end - } - - local mock_qmicli = { - uim_read_transparent = function(ctx, port, id) - return mock_cmd - end - } - - local mock_wraperr = { - new = function(err) return err end - } - - -- Load the qmi module with mocked dependencies - package.loaded["services.hal.drivers.modem.qmicli"] = mock_qmicli - package.loaded["fibers.context"] = mock_context - package.loaded["wraperr"] = mock_wraperr - package.loaded["services.hal.utils"] = require("services.hal.utils") - package.loaded["services.log"] = require("services.log") - - local qmi_module = require("services.hal.drivers.modem.mode.qmi") - - -- Create a mock modem object - local modem = { - ctx = context.background(), - primary_port = "/dev/cdc-wdm1" - } - - -- Initialize the modem with qmi functions - qmi_module(modem) - - -- Test the function - local gids, err = modem.uim_get_gids() - - assert(err == nil, "expected err to be nil but got " .. tostring(err)) - assert(gids ~= nil, "expected gids to be a table but got nil") - - local expected = "85FFFFFFFFFF47454E4945494E20202020202020" - assert(gids.gid1 == expected, - "expected gid1 to be '" .. expected .. "' but got '" .. tostring(gids.gid1) .. "'") - assert(mock_cmd.setpgid_called, "expected setpgid to be called") -end - -local function test_uim_get_gids_error() - -- Clear cached module - package.loaded["services.hal.drivers.modem.mode.qmi"] = nil - - -- Mock dependencies - local mock_context = { - with_timeout = function(ctx, timeout) - return ctx - end - } - - local mock_cmd = { - setpgid_called = false, - setpgid = function(self, val) - self.setpgid_called = true - end, - combined_output = function(self) - return "", "command failed" - end - } - - local mock_qmicli = { - uim_read_transparent = function(ctx, port, id) - return mock_cmd - end - } - - local mock_wraperr = { - new = function(err) return "wrapped: " .. err end - } - - -- Load the qmi module with mocked dependencies - package.loaded["services.hal.drivers.modem.qmicli"] = mock_qmicli - package.loaded["fibers.context"] = mock_context - package.loaded["wraperr"] = mock_wraperr - package.loaded["services.hal.utils"] = require("services.hal.utils") - package.loaded["services.log"] = require("services.log") - - local qmi_module = require("services.hal.drivers.modem.mode.qmi") - - -- Create a mock modem object - local modem = { - ctx = context.background(), - primary_port = "/dev/cdc-wdm1" - } - - -- Initialize the modem with qmi functions - qmi_module(modem) - - -- Test the function - local gids, err = modem.uim_get_gids() - - assert(err ~= nil, "expected err to be set but got nil") - assert(err == "wrapped: command failed", "expected wrapped error but got " .. tostring(err)) - assert(gids ~= nil, "expected gids to be a table but got nil") - assert(next(gids) == nil, "expected gids to be empty but it has values") -end - -fiber.spawn(function () - test_uim_get_gids_simple() - test_uim_get_gids_complex() - test_uim_get_gids_error() - fiber.stop() -end) - -print("running hal qmi tests") -fiber.main() -print("passed") diff --git a/tests/test_hal_utils.lua b/tests/test_hal_utils.lua deleted file mode 100644 index 5f3dc8c2..00000000 --- a/tests/test_hal_utils.lua +++ /dev/null @@ -1,250 +0,0 @@ -local fiber = require "fibers.fiber" -local utils = require "services.hal.utils" -local assertions = require "test_utils.assertions" - -local function check_parse_output(expected, result, err) - assert(err == nil, "expected err to be nil but got", err) - assert(result, "expected table but got nil") - assert(result.type == expected.type, - 'expected type to be ' .. (expected.type or 'nil') .. ' but got ' .. (result.type or "nil")) - assert(result.reason == expected.reason, - 'expected reason to be ' .. (expected.reason or 'nil') .. ' but got ' .. (result.reason or "nil")) - assert(result.prev_state == expected.prev_state, - 'expected prev_state to be ' .. (expected.prev_state or 'nil') .. ' but got ' .. (result.prev_state or 'nil')) - assert(result.curr_state == expected.curr_state, - 'expected curr_state to be ' .. (expected.curr_state or 'nil') .. ' but got ' .. (result.curr_state or 'nil')) -end - -local function test_parse_modem_monitor_init() - local message = "/org/freedesktop/ModemManager1/Modem/0: Initial state, 'disabled'" - - local result, err = utils.parse_modem_monitor(message) - local expected = { - type = 'initial', - reason = nil, - prev_state = 'disabled', - curr_state = 'disabled' - } - check_parse_output(expected, result, err) -end - -local function test_parse_modem_monitor_changed_with_reason() - local message = - "/org/freedesktop/ModemManager1/Modem/0: State changed, 'disabled' --> 'enabling' (Reason: User request)" - - local result, err = utils.parse_modem_monitor(message) - local expected = { - type = 'changed', - reason = 'User request', - prev_state = 'disabled', - curr_state = 'enabling' - } - check_parse_output(expected, result, err) -end - -local function test_parse_modem_monitor_removed() - local message = "/org/freedesktop/ModemManager1/Modem/0: Removed" - - local result, err = utils.parse_modem_monitor(message) - - local expected = { - type = 'removed', - reason = nil, - prev_state = nil, - curr_state = nil - } - check_parse_output(expected, result, err) -end - -local function test_invalid_parse_modem_monitor() - local message = "Hello World" - - local result, err = utils.parse_modem_monitor(message) - - assert(err == "Unknown modem monitor message: Hello World", - 'expected "Unknown modem monitor message: Hello World" but received nil ' .. (err or 'nil')) - assert(result == nil, 'expected nil but got ' .. (result or 'nil')) -end - -local function test_nil_parse_modem_monitor() - local result, err = utils.parse_modem_monitor(nil) - - assert(err == "Modem monitor message is nil", - 'expected "Modem monitor message is nil" but received ' .. (err or 'nil')) - assert(result == nil, 'expected nil but got ' .. (result or 'nil')) -end - -local function test_starts_with_true() - local main_string = 'this is a test string' - local start_string = 'this is a' - - local result = utils.starts_with(main_string, start_string) - - local expected_result = true - assert(result == expected_result, string.format("starts_with expected %s but got %s", expected_result, result)) -end - -local function test_starts_with_false() - local main_string = 'this is a test string' - local start_string = 'this is not a' - - local result = utils.starts_with(main_string, start_string) - - local expected_result = false - assert(result == expected_result, string.format("starts_with expected %s but got %s", expected_result, result)) -end - -local function test_starts_with_nil_1() - local start_string = 'this is a' - - local result = utils.starts_with(nil, start_string) - - local expected_result = false - assert(result == expected_result, string.format("starts_with expected %s but got %s", expected_result, result)) -end - -local function test_starts_with_nil_2() - local main_string = 'this is a test string' - - local result = utils.starts_with(main_string, nil) - - local expected_result = false - assert(result == expected_result, string.format("starts_with expected %s but got %s", expected_result, result)) -end --- test_parse_monitor_added -local function test_parse_monitor_added() - local line = '(+) /org/freedesktop/ModemManager1/Modem/1 [QUALCOMM INCORPORATED] QUECTEL Mobile Broadband Module' - local added, address, err = utils.parse_monitor(line) - - local expected_address = "/org/freedesktop/ModemManager1/Modem/1" - assert(added == true, 'added should equal true') - assert(address == expected_address, - string.format('address expected %s but got %s', expected_address, assertions.to_str(address))) - assert(err == nil, string.format('expected err to be nil but got %s', assertions.to_str(err))) -end - --- test_parse_monitor_removed - -local function test_parse_monitor_removed() - local line = '(-) /org/freedesktop/ModemManager1/Modem/0 [QUALCOMM INCORPORATED] QUECTEL Mobile Broadband Module' - local added, address, err = utils.parse_monitor(line) - - local expected_address = "/org/freedesktop/ModemManager1/Modem/0" - assert(added == false, 'added should equal true') - assert(address == expected_address, - string.format('address expected %s but got %s', expected_address, assertions.to_str(address))) - assert(err == nil, string.format('expected err to be nil but got %s', assertions.to_str(err))) -end - --- test_parse_monitor_nil - -local function test_parse_monitor_nil() - local added, address, err = utils.parse_monitor(nil) - - local expected_err = 'monitor message is nil' - - assert(added == nil, string.format('added should be nil but got %s', assertions.to_str(added))) - assert(address == nil, string.format('address should be nil but got %s', assertions.to_str(address))) - assert(err == expected_err, string.format('expected err to be nil but got %s', expected_err, assertions.to_str(err))) -end - --- test_parse_control_topic_valid - -local function test_parse_control_topic_valid() - local topic = 'hal/capability/modem/1/control/enable' - - local cap_name, index, endpoint, err = utils.parse_control_topic(topic) - - local exp_cap_name = 'modem' - local exp_index = 1 - local exp_endpoint = 'enable' - - assertions.expect_assert(exp_cap_name, cap_name, 'capability name') - assertions.expect_assert(exp_index, index, 'capability index') - assertions.expect_assert(exp_endpoint, endpoint, 'capability endpoint') - assertions.expect_assert(nil, err, 'capability error') -end - --- test_parse_control_topic_fewer_components - -local function test_parse_control_topic_fewer_components() - local topic = 'hal/capability/modem/1/control' - - local cap_name, index, endpoint, err = utils.parse_control_topic(topic) - - local exp_err = 'control topic does not contain enough components' - - assertions.expect_assert(nil, cap_name, 'capability name') - assertions.expect_assert(nil, index, 'capability index') - assertions.expect_assert(nil, endpoint, 'capability endpoint') - assertions.expect_assert(exp_err, err, 'capability error') -end - --- test_parse_control_topic_invalid_number - -local function test_parse_control_topic_invalid_number() - local topic = 'hal/capability/modem/foo/control/enable' - - local cap_name, index, endpoint, err = utils.parse_control_topic(topic) - - local exp_err = 'failed to convert instance to a number' - - assertions.expect_assert(nil, cap_name, 'capability name') - assertions.expect_assert(nil, index, 'capability index') - assertions.expect_assert(nil, endpoint, 'capability endpoint') - assertions.expect_assert(exp_err, err, 'capability error') -end - -local function test_parse_qmicli_output() - local qmi_output = [[ -[/dev/cdc-wdm0] Successfully got serving system: - Registration state: 'registered' - CS: 'attached' - PS: 'attached' - Selected network: '3gpp' - Radio interfaces: '1' - [0]: 'lte' - Roaming status: 'off' - Current PLMN: - MCC: '234' - MNC: '10' - Description: 'O2 - UK']] - - local result, err = utils.parse_qmicli_output(qmi_output) - - assert(err == nil, "expected err to be nil but got " .. (err or "nil")) - assert(result ~= nil, "expected result to be a table but got nil") - - assert(result["Registration state"] == "registered", "wrong registration state") - assert(result["CS"] == "attached", "wrong CS state") - assert(result["PS"] == "attached", "wrong PS state") - assert(result["Selected network"] == "3gpp", "wrong network") - - local plmn = result["Current PLMN"] - assert(plmn.MCC == "234", "wrong MCC") - assert(plmn.MNC == "10", "wrong MNC") - assert(plmn.Description == "O2 - UK", "wrong description") -end -fiber.spawn(function () - test_parse_modem_monitor_init() - test_parse_modem_monitor_changed_with_reason() - test_parse_modem_monitor_removed() - test_invalid_parse_modem_monitor() - test_nil_parse_modem_monitor() - test_starts_with_true() - test_starts_with_false() - test_starts_with_nil_1() - test_starts_with_nil_2() - test_parse_monitor_added() - test_parse_monitor_removed() - test_parse_monitor_nil() - test_parse_control_topic_valid() - test_parse_control_topic_fewer_components() - test_parse_control_topic_invalid_number() - test_parse_qmicli_output() - fiber.stop() -end) - -print("running hal util tests") -fiber.main() -print("passed") diff --git a/tests/test_metrics.lua b/tests/test_metrics.lua deleted file mode 100644 index 99bed80c..00000000 --- a/tests/test_metrics.lua +++ /dev/null @@ -1,535 +0,0 @@ --- Detect if this file is being run as the entry point -local this_file = debug.getinfo(1, "S").source:match("@?([^/]+)$") -local is_entry_point = arg and arg[0] and arg[0]:match("[^/]+$") == this_file - -if is_entry_point then - package.path = "../src/lua-fibers/?.lua;" -- fibers submodule src - .. "../src/lua-trie/src/?.lua;" -- trie submodule src - .. "../src/lua-bus/src/?.lua;" -- bus submodule src - .. "../src/?.lua;" - .. "./test_utils/?.lua;" - .. package.path - .. ";/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua" - - _G._TEST = true -- Enable test exports in source code -end - -local luaunit = require 'luaunit' -local processing = require 'services.metrics.processing' -local conf = require 'services.metrics.config' -local sc = require "fibers.utils.syscall" -local sleep = require "fibers.sleep" -local fiber = require "fibers.fiber" -local senml = require "services.metrics.senml" - -TestProcessing = {} - -function TestProcessing:test_diff_trigger_absolute() - local config = { - threshold = 5, - diff_method = "absolute", - initial_val = 10 - } - - local trigger, trig_err = processing.DiffTrigger.new(config) - luaunit.assertNil(trig_err) - local val, short, err = trigger:run(12) - luaunit.assertNil(err) - luaunit.assertEquals(short, true) -- Should short circuit as diff < threshold - - val, short, err = trigger:run(16) - luaunit.assertNil(err) - luaunit.assertEquals(short, false) -- Should pass as diff >= threshold - luaunit.assertEquals(val, 16) -end - -function TestProcessing:test_diff_trigger_percent() - local config = { - threshold = 10, -- 10% threshold - diff_method = "percent", - initial_val = 100 - } - - local trigger, trig_err = processing.DiffTrigger.new(config) - luaunit.assertNil(trig_err) - - -- 5% change - should short circuit - local val, short, err = trigger:run(105) - luaunit.assertNil(err) - luaunit.assertEquals(short, true) - - -- 15% change - should pass - val, short, err = trigger:run(115) - luaunit.assertNil(err) - luaunit.assertEquals(short, false) - luaunit.assertEquals(val, 115) -end - -function TestProcessing:test_diff_trigger_any_change() - local config = { - diff_method = "any-change", - initial_val = 10 - } - - local trigger, trig_err = processing.DiffTrigger.new(config) - luaunit.assertNil(trig_err) - - -- Same value - should short circuit - local val, short, err = trigger:run(10) - luaunit.assertNil(err) - luaunit.assertEquals(short, true) - - -- Any change - should pass - val, short, err = trigger:run(10.1) - luaunit.assertNil(err) - luaunit.assertEquals(short, false) - luaunit.assertEquals(val, 10.1) -end - -function TestProcessing:test_time_trigger() - local config = { duration = 0.1 } - local trigger = processing.TimeTrigger.new(config) - - local val, short, err = trigger:run(123) - luaunit.assertNil(err) - luaunit.assertEquals(short, true) -- Should short circuit initially - - sleep.sleep(0.2) - val, short, err = trigger:run(123) - luaunit.assertNil(err) - luaunit.assertEquals(short, false) -- Should pass after duration - luaunit.assertEquals(val, 123) -end - -function TestProcessing:test_delta_value() - local delta = processing.DeltaValue.new({ initial_val = 10 }) - - local val, short, err = delta:run(15) - luaunit.assertNil(err) - luaunit.assertEquals(short, false) - luaunit.assertEquals(val, 5) -- Should return difference - - delta:reset() - val, short, err = delta:run(20) - luaunit.assertNil(err) - luaunit.assertEquals(short, false) - luaunit.assertEquals(val, 5) -end - -function TestProcessing:test_clone_process() - local config = { - threshold = 5, - diff_method = "absolute", - initial_val = 10 - } - - local trigger, trig_err = processing.DiffTrigger.new(config) - luaunit.assertNil(trig_err) - - local clone, clone_err = trigger:clone() - luaunit.assertNil(clone_err) - luaunit.assertNotNil(clone) - - -- Test that clone behaves the same as original - local val1, short1, err1 = trigger:run(16) - local val2, short2, err2 = clone:run(16) - - luaunit.assertEquals(val1, val2) - luaunit.assertEquals(short1, short2) - luaunit.assertEquals(err1, err2) -end - -function TestProcessing:test_process_pipeline() - local pipeline = processing.new_process_pipeline() - - -- Create a pipeline with DiffTrigger and DeltaValue - local diff_trigger = processing.DiffTrigger.new({ - threshold = 5, - diff_method = "absolute", - initial_val = 10 - }) - local delta_value = processing.DeltaValue.new({ initial_val = 10 }) - - pipeline:add(diff_trigger) - pipeline:add(delta_value) - - -- First run should pass through both processes - local val, short, err = pipeline:run(20) - luaunit.assertNil(err) - luaunit.assertEquals(short, false) - luaunit.assertEquals(val, 10) -- DeltaValue should return difference from initial - - -- Second run should short circuit at DiffTrigger - val, short, err = pipeline:run(22) - luaunit.assertNil(err) - luaunit.assertEquals(short, true) -end - -function TestProcessing:test_pipeline_reset() - local pipeline = processing.new_process_pipeline() - local delta_value = processing.DeltaValue.new({ initial_val = 10 }) - pipeline:add(delta_value) - - -- Run once to get delta - local val, short, err = pipeline:run(20) - luaunit.assertNil(err) - luaunit.assertEquals(val, 10) - - -- Reset pipeline - pipeline:reset() - - -- Next run should use the last value as new base - val, short, err = pipeline:run(25) - luaunit.assertNil(err) - luaunit.assertEquals(val, 5) -- 25 - 20 -end - -TestConfig = {} - -function TestConfig:test_build_metric_pipeline_via_apply_config() - -- Test pipeline building indirectly through apply_config - -- which is the main way pipelines are created - local mock_conn = { - subscribe = function(self, topic) - return { - next_msg_op = function() end, - unsubscribe = function() end - } - end - } - - local config = { - templates = {}, - collections = { - ["test/endpoint"] = { - protocol = "log", - process = { - { - type = "DiffTrigger", - threshold = 5, - diff_method = "absolute" - } - } - } - }, - publish_period = 60 - } - - local metrics, period, cloud_config = conf.apply_config(mock_conn, config, {}) - luaunit.assertEquals(#metrics, 1) - luaunit.assertEquals(period, 60) - luaunit.assertNotNil(metrics[1].base_pipeline) -end - -function TestConfig:test_validate_http_config() - -- Valid config - local valid_config = { - url = "http://example.com", - thing_key = "test_key", - channels = { - { id = "ch1", name = "data", metadata = { channel_type = "data" } } - } - } - local valid, err = conf.validate_http_config(valid_config) - luaunit.assertTrue(valid) - luaunit.assertNil(err) - - -- Missing url - local invalid_config = { - thing_key = "test_key", - channels = {} - } - valid, err = conf.validate_http_config(invalid_config) - luaunit.assertFalse(valid) - luaunit.assertNotNil(err) - - -- Nil config - valid, err = conf.validate_http_config(nil) - luaunit.assertFalse(valid) - luaunit.assertNotNil(err) -end - -function TestConfig:test_merge_config() - local base = { - url = "http://base.com", - thing_key = "base_key", - nested = { - field1 = "value1", - field2 = "value2" - } - } - - local override = { - thing_key = "override_key", - nested = { - field2 = "override_value2", - field3 = "value3" - } - } - - local merged = conf.merge_config(base, override) - luaunit.assertEquals(merged.url, "http://base.com") - luaunit.assertEquals(merged.thing_key, "override_key") - luaunit.assertEquals(merged.nested.field1, "value1") - luaunit.assertEquals(merged.nested.field2, "override_value2") - luaunit.assertEquals(merged.nested.field3, "value3") -end - -function TestConfig:test_validate_topic_with_nil() - -- Test the actual metrics service validate_topic logic by directly calling _handle_metric - local metrics_service = require 'services.metrics' - local bus_pkg = require 'bus' - local context = require "fibers.context" - - -- Create a bus and connection (but don't start the service) - local test_bus = bus_pkg.new() - local conn = test_bus:connect() - - -- Create a context for the metrics service (needed for logging) - local bg_ctx = context.background() - local service_ctx = context.with_value(bg_ctx, "service_name", "metrics_test") - service_ctx = context.with_value(service_ctx, "fiber_name", "test_fiber") - - -- Set up metrics service state manually without starting it - metrics_service.ctx = service_ctx - metrics_service.metric_values = {} - metrics_service.pipelines = {} - - -- Create a pipeline for a test metric - local base_pipeline = processing.new_process_pipeline() - local diff_trigger = processing.DiffTrigger.new({ - threshold = 5, - diff_method = "absolute", - initial_val = 10 - }) - base_pipeline:add(diff_trigger) - - -- Create a metric definition - local metric = { - protocol = "log", - field = nil, - rename = nil, - base_pipeline = base_pipeline - } - - -- Test 1: Valid topic should work and add value to metric_values - local valid_msg = bus_pkg.new_msg({"test", "metric"}, 100) - metrics_service:_handle_metric(metric, valid_msg) - luaunit.assertNotNil(metrics_service.metric_values.log) - luaunit.assertNotNil(metrics_service.metric_values.log["test.metric"]) - luaunit.assertEquals(metrics_service.metric_values.log["test.metric"].value, 100) - - -- Reset for next test - metrics_service.metric_values = {} - - -- Test 2: Invalid topic with nil in the middle should be rejected (not added) - local invalid_msg_nil = bus_pkg.new_msg({"test", nil, "metric"}, 200) - metrics_service:_handle_metric(metric, invalid_msg_nil) - -- Should not create any metric values because topic is invalid - luaunit.assertTrue((not metrics_service.metric_values.log) or (not metrics_service.metric_values.log["test..metric"])) - - -- Test 3: Invalid topic with gap (sparse array) should be rejected - local sparse_topic = {} - sparse_topic[1] = "test" - sparse_topic[3] = "metric" -- index 2 is missing - local sparse_msg = bus_pkg.new_msg(sparse_topic, 300) - metrics_service:_handle_metric(metric, sparse_msg) - -- Should not create any metric values because topic is invalid - luaunit.assertTrue((not metrics_service.metric_values.log) or (not metrics_service.metric_values.log["test.metric"])) - - -- Test 4: Another valid topic should work - metrics_service.metric_values = {} - local valid_msg2 = bus_pkg.new_msg({"another", "valid", "topic"}, 50) - metrics_service:_handle_metric(metric, valid_msg2) - luaunit.assertNotNil(metrics_service.metric_values.log) - luaunit.assertNotNil(metrics_service.metric_values.log["another.valid.topic"]) - - -- Test 5: Empty topic should be rejected - metrics_service.metric_values = {} - local empty_msg = bus_pkg.new_msg({}, 400) - metrics_service:_handle_metric(metric, empty_msg) - luaunit.assertNil(metrics_service.metric_values.log) - - -- Clean up - conn:disconnect() -end - -TestSenML = {} - -function TestSenML:test_encode_basic() - -- Test encoding a string - local result, err = senml.encode("test/topic", "string_value") - luaunit.assertNil(err) - luaunit.assertEquals(result.n, "test/topic") - luaunit.assertEquals(result.vs, "string_value") - - -- Test encoding a number - result, err = senml.encode("test/topic", 42.5) - luaunit.assertNil(err) - luaunit.assertEquals(result.n, "test/topic") - luaunit.assertEquals(result.v, 42.5) - - -- Test encoding a boolean - result, err = senml.encode("test/topic", true) - luaunit.assertNil(err) - luaunit.assertEquals(result.n, "test/topic") - luaunit.assertEquals(result.vb, true) - - -- Test encoding with timestamp - result, err = senml.encode("test/topic", 42, 1234567890) - luaunit.assertNil(err) - luaunit.assertEquals(result.t, 1234567890) -end - -function TestSenML:test_encode_invalid_types() - -- Test encoding with invalid type - local result, err = senml.encode("test/topic", {}) - luaunit.assertNotNil(err) - luaunit.assertNil(result) - - -- Test encoding with nil - result, err = senml.encode("test/topic", nil) - luaunit.assertNotNil(err) - luaunit.assertNil(result) -end - -function TestSenML:test_encode_r_flat() - -- Test encoding a flat table - local values = { - temperature = 23.5, - humidity = 60, - status = "online" - } - - local result, err = senml.encode_r("device/sensors", values) - luaunit.assertNil(err) - luaunit.assertEquals(#result, 3) - - -- Check each entry - local found = { temp = false, humid = false, status = false } - for _, entry in ipairs(result) do - if entry.n == "device/sensors.temperature" and entry.v == 23.5 then - found.temp = true - elseif entry.n == "device/sensors.humidity" and entry.v == 60 then - found.humid = true - elseif entry.n == "device/sensors.status" and entry.vs == "online" then - found.status = true - end - end - - luaunit.assertTrue(found.temp) - luaunit.assertTrue(found.humid) - luaunit.assertTrue(found.status) -end - -function TestSenML:test_encode_r_nested() - -- Test encoding a nested table - local values = { - system = { - memory = 8192, - cpu = 45.6 - }, - network = { - status = "connected", - speed = 100 - } - } - - local result, err = senml.encode_r("device", values) - luaunit.assertNil(err) - luaunit.assertEquals(#result, 4) - - -- Check specific entries - local found = { memory = false, cpu = false, net_status = false, speed = false } - for _, entry in ipairs(result) do - if entry.n == "device.system.memory" and entry.v == 8192 then - found.memory = true - elseif entry.n == "device.system.cpu" and entry.v == 45.6 then - found.cpu = true - elseif entry.n == "device.network.status" and entry.vs == "connected" then - found.net_status = true - elseif entry.n == "device.network.speed" and entry.v == 100 then - found.speed = true - end - end - - luaunit.assertTrue(found.memory) - luaunit.assertTrue(found.cpu) - luaunit.assertTrue(found.net_status) - luaunit.assertTrue(found.speed) -end - -function TestSenML:test_encode_r_with_value_field() - -- Test encoding a table with __value field and a subtable - local values = { - system = { - __value = "active", -- This should be at the base topic - memory = { - __value = "healthy", - used = 4096, - free = 4096 - } - } - } - - local result, err = senml.encode_r("device", values) - luaunit.assertNil(err) - - -- Check for all expected entries - local found = { system = false, memory = false, used = false, free = false } - for _, entry in ipairs(result) do - if entry.n == "device.system" and entry.vs == "active" then - found.system = true - elseif entry.n == "device.system.memory" and entry.vs == "healthy" then - found.memory = true - elseif entry.n == "device.system.memory.used" and entry.v == 4096 then - found.used = true - elseif entry.n == "device.system.memory.free" and entry.v == 4096 then - found.free = true - end - end - - luaunit.assertTrue(found.system) - luaunit.assertTrue(found.memory) - luaunit.assertTrue(found.used) - luaunit.assertTrue(found.free) - luaunit.assertEquals(#result, 4) -end - -function TestSenML:test_encode_r_with_value_and_time() - -- Test encoding values with explicit value and time fields - local values = { - temperature = { - value = 23.5, - time = 1234567890 - }, - status = "online" - } - - local result, err = senml.encode_r("sensor", values) - luaunit.assertNil(err) - luaunit.assertEquals(#result, 2) - - -- Check entries - local found = { temp = false, status = false } - for _, entry in ipairs(result) do - if entry.n == "sensor.temperature" and entry.v == 23.5 and entry.t == 1234567890 then - found.temp = true - elseif entry.n == "sensor.status" and entry.vs == "online" then - found.status = true - end - end - - luaunit.assertTrue(found.temp) - luaunit.assertTrue(found.status) -end - --- Only run tests if this file is executed directly (not via dofile) -if is_entry_point then - fiber.spawn(function () - luaunit.LuaUnit.run() - fiber.stop() - end) - - fiber.main() -end diff --git a/tests/test_modem_driver.lua b/tests/test_modem_driver.lua deleted file mode 100644 index 56f4ab4f..00000000 --- a/tests/test_modem_driver.lua +++ /dev/null @@ -1,280 +0,0 @@ --- insert shim to stop mmcli execution on hardware --- package.path = "./test_modem_driver/?.lua;./test_utils/?.lua;" .. package.path - --- purge any module caching of shimmed module --- package.loaded["services.hal.mmcli"] = nil - -local fiber = require "fibers.fiber" -local op = require "fibers.op" -local channel = require "fibers.channel" -local context = require "fibers.context" -local sleep = require "fibers.sleep" -local bus_pkg = require "bus" - --- load mmcli shim to emulate hardware -local test_utils = require "test_utils.utils" -local shim_dir = "./test_modem_driver/" - --- -local function test_get_info() - test_utils.update_shim_path(shim_dir, "default_shim") - local modem_driver = test_utils.uncached_require("services.hal.drivers.modem") - local bg = context.background() - local ctx = context.with_cancel(bg) - - local driver = modem_driver.new(context.with_cancel(ctx), 0) - - local get_key = {"modem", "generic", "device"} - - local nil_get = driver.cache:get(get_key) - - assert(nil_get == nil, "cache should not have device set before get function is called") - - driver:get_info() - - local val_get = driver.cache:get(get_key) - - assert(val_get == "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-1", - "device did not match expected value") -end - -local function test_get_at_ports() - test_utils.update_shim_path(shim_dir, "default_shim") - local modem_driver = test_utils.uncached_require("services.hal.drivers.modem") - local bg = context.background() - local ctx = context.with_cancel(bg) - - local driver = modem_driver.new(context.with_cancel(ctx), 0) - - local get_key = {"modem", "generic", "at_ports"} - - local nil_get = driver.cache:get(get_key) - - assert(nil_get == nil, "cache should not have at_ports set before get function is called") - - driver:get_at_ports() - - local at_ports = driver.cache:get(get_key) - - assert(at_ports, "at_ports expected table but got nil") - assert(at_ports[1] == "ttyUSB2", "at_ports[1] expected ttyUSB2 but got " .. at_ports[1]) - assert(at_ports[2] == "ttyUSB3", "at_ports[2] expected ttyUSB3 but got " .. at_ports[2]) -end - -local function test_get() - test_utils.update_shim_path(shim_dir, "default_shim") - local modem_driver = test_utils.uncached_require("services.hal.drivers.modem") - local bg = context.background() - local ctx = context.with_cancel(bg) - - local driver = modem_driver.new(context.with_cancel(ctx), 0) - - local imei = driver:get("imei") - - assert(imei, "imei should not be nil") - assert(imei == "867929068986654", "imei expected 867929068986654 but got " .. imei) -end - -local function test_get_primary_port() - test_utils.update_shim_path(shim_dir, "default_shim") - local modem_driver = test_utils.uncached_require("services.hal.drivers.modem") - local bg = context.background() - local ctx = context.with_cancel(bg) - - local driver = modem_driver.new(context.with_cancel(ctx), 0) - - local expected_val = '/dev/cdc-wdm0' - local primary_port = driver:primary_port() - - assert(primary_port == expected_val, - "primary_port expected " .. expected_val .. " but got " .. (primary_port or "nil")) -end - -local function test_modem_driver_invalid_get() - test_utils.update_shim_path(shim_dir, "default_shim") - local modem_driver = test_utils.uncached_require("services.hal.drivers.modem") - local bg = context.background() - local ctx = context.with_cancel(bg) - - local driver = modem_driver.new(context.with_cancel(ctx), 0) - - local info, err = driver:get("foo") - assert(err, "err expected error but got nil") - assert(info == nil, "info expected nil but got " .. (info or "nil")) -end - -local function test_modem_driver_init() - test_utils.update_shim_path(shim_dir, "default_shim") - local modem_driver = test_utils.uncached_require("services.hal.drivers.modem") - local bg = context.background() - local ctx = context.with_cancel(bg) - - local driver = modem_driver.new(context.with_cancel(ctx), 0) - - local err = driver:init() - - assert(err == nil, err) -end - -local function test_modem_driver_invalid_init() - test_utils.update_shim_path(shim_dir, "no_modem_shim") - local modem_driver = test_utils.uncached_require("services.hal.drivers.modem") - local bg = context.background() - local ctx = context.with_cancel(bg) - - local driver = modem_driver.new(context.with_cancel(ctx), 0) - - -- force the error to be a string rather than wrapper - local err = string.format("%s", driver:init()) - - local expected_err = "no modem found" - - assert(err == expected_err, string.format('init error expected %s but got %s', expected_err, err)) -end -local function test_state_monitor() - test_utils.update_shim_path(shim_dir, "default_shim") - local modem_driver = test_utils.uncached_require("services.hal.drivers.modem") - local bg = context.background() - local ctx = context.with_cancel(bg) - local ctx2 = context.with_cancel(ctx) - - local driver = modem_driver.new(context.with_cancel(ctx), 0) - - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local bus_connection = bus:connect() - local bus_connection_2 = bus:connect() - - fiber.spawn(function () - driver:state_monitor(bus_connection_2, 0) - end) - - -- cancel test after 1 second - fiber.spawn(function () - sleep.sleep(1) - ctx:cancel('test done') - end) - - local modem_monitor_sub, err = bus_connection:subscribe("hal/capability/modem/0/info/state") - if err ~= nil then error(err) end - local states = {} - - -- collect state messages - while not ctx2:err() do - local msg = op.choice( - modem_monitor_sub:next_msg_op(), - ctx2:done_op() - ):perform() - if msg == nil then break end - table.insert(states, msg.payload) - end - - local expected_states = { - {type = 'initial', prev_state='disabled', curr_state=nil}, - {type = 'changed', prev_state='disabled', curr_state='enabling'}, - {type = 'changed', prev_state='enabling', curr_state='searching'}, - {type = 'changed', prev_state='searching', curr_state='disabling'}, - {type = 'changed', prev_state='disabling', curr_state='disabled'}, - {type = 'removed', prev_state=nil, cur_state=nil} - } - - -- evaluate state_messages - for i, state in ipairs(states) do - assert(expected_states[i].type == state.type, - string.format("type mismatch, expected %s got %s", expected_states[i].type, state.type)) - assert(expected_states[i].prev_state == state.prev_state, - string.format("prev_state mismatch, expected %s got %s", expected_states[i].prev_state, state.prev_state)) - assert(expected_states[i].curr_state == state.curr_state, - string.format("cur_state mismatch, expected %s got %s", expected_states[i].cur_state, state.cur_state)) - end -end - -local function test_command_manager_commands() - test_utils.update_shim_path(shim_dir, "default_shim") - local modem_driver = test_utils.uncached_require("services.hal.drivers.modem") - local commands = {"disconnect", "reset", "enable", "disable", "inhibit"} - - local bg = context.background() - local ctx = context.with_cancel(bg) - - local driver = modem_driver.new(context.with_cancel(ctx), 0) - - fiber.spawn(function () - driver:command_manager() - end) - - local command_q = driver.command_q - - -- send commands to driver and check for the command name to be returned - -- or in the case of inhibit true should be returned - for _, cmd_name in ipairs(commands) do - local ret_chan = channel.new() - local cmd_msg = { - command = cmd_name, - return_channel = ret_chan - } - command_q:put(cmd_msg) - - local ret_msg = ret_chan:get() - local res = ret_msg.result - local err = ret_msg.err - - assert(err == nil, 'error expected nil but got '..(err or 'nil')) - if cmd_name == 'inhibit' then assert(res == true, 'return mismatch: expected true, received ' - .. (res and "true" or "false")) - else assert(res == cmd_name, 'return mismatch: expected '..cmd_name..', received '..res) end - end - - ctx:cancel('tests done') -end - -local function test_command_manager_command_not_exist() - test_utils.update_shim_path(shim_dir, "default_shim") - local modem_driver = test_utils.uncached_require("services.hal.drivers.modem") - local expected_result = "command does not exist" - local bg = context.background() - local ctx = context.with_cancel(bg) - - local driver = modem_driver.new(context.with_cancel(ctx), 0) - - fiber.spawn(function () - driver:command_manager() - end) - - local command_q = driver.command_q - - local ret_chan = channel.new() - local cmd_msg = { - command = 'foo', - return_channel = ret_chan - } - command_q:put(cmd_msg) - - local ret_msg = ret_chan:get() - local res = ret_msg.result - local err = ret_msg.err - - assert(res == nil, 'expected res to be nil but got '..(res or 'nil')) - assert(err == expected_result, 'err mismatch: expected '..expected_result..' received '..err) - - ctx:cancel('tests done') -end - --- How to test starts_with function when is local? Make a hook into the module? - -fiber.spawn(function () - -- test_get_info() - -- test_get_at_ports() - -- test_get() - -- test_get_primary_port() - -- test_modem_driver_invalid_get() - test_modem_driver_init() - test_modem_driver_invalid_init() - test_state_monitor() - test_command_manager_commands() - test_command_manager_command_not_exist() - fiber.stop() -end) - -print("running modem driver tests") -fiber.main() -print("passed") diff --git a/tests/test_modem_driver/default_shim/services/hal/mmcli.lua b/tests/test_modem_driver/default_shim/services/hal/mmcli.lua deleted file mode 100644 index 06582a38..00000000 --- a/tests/test_modem_driver/default_shim/services/hal/mmcli.lua +++ /dev/null @@ -1,92 +0,0 @@ -local sleep = require "fibers.sleep" -local file = require "fibers.stream.file" -local json = require "dkjson" -local simu_commands = require "test_utils.SimuCommands" - -local function monitor_modems() -end - -local function inhibit(device) - return { - start = function() - sleep.sleep(0.05) - return nil - end, - kill = function() - sleep.sleep(0.05) - end - } -end - -local function connect(device) - return { - run = function() - sleep.sleep(0.05) - return "connect" - end - } -end - -local function disconnect(device) - return { - run = function() - sleep.sleep(0.05) - return "disconnect" - end - } -end - -local function reset(device) - return { - run = function() - sleep.sleep(0.05) - return "reset" - end - } -end - -local function enable(device) - return { - run = function() - sleep.sleep(0.05) - return "enable" - end - } -end - -local function disable(device) - return { - run = function() - sleep.sleep(0.05) - return "disable" - end - } -end - -local function monitor_state(device) - return simu_commands.new('./test_modem_driver/default_shim/services/hal/modem_states.json') -end - -local function information(ctx, device) - return { - combined_output = function() - sleep.sleep(0.05) - local cardfile, err = file.open("./test_modem_driver/default_shim/services/hal/modemcard_info.json", "r") - if err then return nil end - local carddata = cardfile:read_all_chars() - return carddata - end - } -end - -return { - monitor_modems = monitor_modems, - inhibit = inhibit, - connect = connect, - disconnect = disconnect, - reset = reset, - enable = enable, - disable = disable, - monitor_state = monitor_state, - information = information -} diff --git a/tests/test_modem_driver/default_shim/services/hal/modem_states.json b/tests/test_modem_driver/default_shim/services/hal/modem_states.json deleted file mode 100644 index f10fab77..00000000 --- a/tests/test_modem_driver/default_shim/services/hal/modem_states.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "events": [ - { - "out": "/org/freedesktop/ModemManager1/Modem/0: Initial state, 'disabled'", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'disabled' --> 'enabling' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'enabling' --> 'searching' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'searching' --> 'disabling' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'disabling' --> 'disabled' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: Removed", - "wait": 0 - }, - { - "out": "This will not parse", - "wait": 0 - } - ] -} diff --git a/tests/test_modem_driver/default_shim/services/hal/modemcard_info.json b/tests/test_modem_driver/default_shim/services/hal/modemcard_info.json deleted file mode 100644 index ac67a354..00000000 --- a/tests/test_modem_driver/default_shim/services/hal/modemcard_info.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "modem": { - "3gpp": { - "5gnr": { - "registration-settings": { - "drx-cycle": "--", - "mico-mode": "--" - } - }, - "enabled-locks": [ - "fixed-dialing" - ], - "eps": { - "initial-bearer": { - "dbus-path": "--", - "settings": { - "apn": "", - "ip-type": "ipv4v6", - "password": "--", - "user": "--" - } - }, - "ue-mode-operation": "csps-2" - }, - "imei": "867929068986654", - "operator-code": "--", - "operator-name": "--", - "packet-service-state": "--", - "pco": "--", - "registration-state": "--" - }, - "cdma": { - "activation-state": "--", - "cdma1x-registration-state": "--", - "esn": "--", - "evdo-registration-state": "--", - "meid": "--", - "nid": "--", - "sid": "--" - }, - "dbus-path": "/org/freedesktop/ModemManager1/Modem/14", - "generic": { - "access-technologies": [], - "bearers": [], - "carrier-configuration": "ROW_Generic_3GPP", - "carrier-configuration-revision": "0501081F", - "current-bands": [ - "egsm", - "dcs", - "pcs", - "g850", - "utran-1", - "utran-4", - "utran-6", - "utran-5", - "utran-8", - "utran-2", - "eutran-1", - "eutran-2", - "eutran-3", - "eutran-4", - "eutran-5", - "eutran-7", - "eutran-8", - "eutran-12", - "eutran-13", - "eutran-18", - "eutran-19", - "eutran-20", - "eutran-25", - "eutran-26", - "eutran-28", - "eutran-38", - "eutran-39", - "eutran-40", - "eutran-41", - "utran-19" - ], - "current-capabilities": [ - "gsm-umts, lte" - ], - "current-modes": "allowed: 2g, 3g, 4g; preferred: 4g", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-1", - "device-identifier": "a7591502473ae9ffc14e992ff1621f18cc4dd408", - "drivers": [ - "option1", - "qmi_wwan" - ], - "equipment-identifier": "867929068986654", - "hardware-revision": "10000", - "manufacturer": "QUALCOMM INCORPORATED", - "model": "QUECTEL Mobile Broadband Module", - "own-numbers": [], - "physdev": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-1", - "plugin": "quectel", - "ports": [ - "cdc-wdm0 (qmi)", - "ttyUSB0 (ignored)", - "ttyUSB1 (gps)", - "ttyUSB2 (at)", - "ttyUSB3 (at)", - "wwan0 (net)" - ], - "power-state": "on", - "primary-port": "cdc-wdm0", - "primary-sim-slot": "1", - "revision": "EG25GGBR07A08M2G", - "signal-quality": { - "recent": "yes", - "value": "0" - }, - "sim": "/org/freedesktop/ModemManager1/SIM/14", - "sim-slots": [ - "/org/freedesktop/ModemManager1/SIM/14", - "/" - ], - "state": "disabled", - "state-failed-reason": "--", - "supported-bands": [ - "egsm", - "dcs", - "pcs", - "g850", - "utran-1", - "utran-4", - "utran-6", - "utran-5", - "utran-8", - "utran-2", - "eutran-1", - "eutran-2", - "eutran-3", - "eutran-4", - "eutran-5", - "eutran-7", - "eutran-8", - "eutran-12", - "eutran-13", - "eutran-18", - "eutran-19", - "eutran-20", - "eutran-25", - "eutran-26", - "eutran-28", - "eutran-38", - "eutran-39", - "eutran-40", - "eutran-41", - "utran-19" - ], - "supported-capabilities": [ - "gsm-umts, lte" - ], - "supported-ip-families": [ - "ipv4", - "ipv6", - "ipv4v6" - ], - "supported-modes": [ - "allowed: 2g; preferred: none", - "allowed: 3g; preferred: none", - "allowed: 4g; preferred: none", - "allowed: 2g, 3g; preferred: 3g", - "allowed: 2g, 3g; preferred: 2g", - "allowed: 2g, 4g; preferred: 4g", - "allowed: 2g, 4g; preferred: 2g", - "allowed: 3g, 4g; preferred: 4g", - "allowed: 3g, 4g; preferred: 3g", - "allowed: 2g, 3g, 4g; preferred: 4g", - "allowed: 2g, 3g, 4g; preferred: 3g", - "allowed: 2g, 3g, 4g; preferred: 2g" - ], - "unlock-required": "sim-pin2", - "unlock-retries": [ - "sim-pin (3)", - "sim-puk (10)", - "sim-pin2 (3)", - "sim-puk2 (10)" - ] - } - } -} \ No newline at end of file diff --git a/tests/test_modem_driver/no_modem_shim/services/hal/mmcli.lua b/tests/test_modem_driver/no_modem_shim/services/hal/mmcli.lua deleted file mode 100644 index fb8524d5..00000000 --- a/tests/test_modem_driver/no_modem_shim/services/hal/mmcli.lua +++ /dev/null @@ -1,17 +0,0 @@ -local sleep = require "fibers.sleep" -local file = require "fibers.stream.file" -local json = require "dkjson" -local simu_commands = require "test_utils.SimuCommands" - -local function information(ctx, device) - return { - combined_output = function() - sleep.sleep(0.05) - return nil, "no modem found" - end - } -end - -return { - information = information -} diff --git a/tests/test_modem_driver/no_modem_shim/services/hal/modem_states.json b/tests/test_modem_driver/no_modem_shim/services/hal/modem_states.json deleted file mode 100644 index f10fab77..00000000 --- a/tests/test_modem_driver/no_modem_shim/services/hal/modem_states.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "events": [ - { - "out": "/org/freedesktop/ModemManager1/Modem/0: Initial state, 'disabled'", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'disabled' --> 'enabling' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'enabling' --> 'searching' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'searching' --> 'disabling' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'disabling' --> 'disabled' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: Removed", - "wait": 0 - }, - { - "out": "This will not parse", - "wait": 0 - } - ] -} diff --git a/tests/test_modem_driver/no_modem_shim/services/hal/modemcard_info.json b/tests/test_modem_driver/no_modem_shim/services/hal/modemcard_info.json deleted file mode 100644 index ac67a354..00000000 --- a/tests/test_modem_driver/no_modem_shim/services/hal/modemcard_info.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "modem": { - "3gpp": { - "5gnr": { - "registration-settings": { - "drx-cycle": "--", - "mico-mode": "--" - } - }, - "enabled-locks": [ - "fixed-dialing" - ], - "eps": { - "initial-bearer": { - "dbus-path": "--", - "settings": { - "apn": "", - "ip-type": "ipv4v6", - "password": "--", - "user": "--" - } - }, - "ue-mode-operation": "csps-2" - }, - "imei": "867929068986654", - "operator-code": "--", - "operator-name": "--", - "packet-service-state": "--", - "pco": "--", - "registration-state": "--" - }, - "cdma": { - "activation-state": "--", - "cdma1x-registration-state": "--", - "esn": "--", - "evdo-registration-state": "--", - "meid": "--", - "nid": "--", - "sid": "--" - }, - "dbus-path": "/org/freedesktop/ModemManager1/Modem/14", - "generic": { - "access-technologies": [], - "bearers": [], - "carrier-configuration": "ROW_Generic_3GPP", - "carrier-configuration-revision": "0501081F", - "current-bands": [ - "egsm", - "dcs", - "pcs", - "g850", - "utran-1", - "utran-4", - "utran-6", - "utran-5", - "utran-8", - "utran-2", - "eutran-1", - "eutran-2", - "eutran-3", - "eutran-4", - "eutran-5", - "eutran-7", - "eutran-8", - "eutran-12", - "eutran-13", - "eutran-18", - "eutran-19", - "eutran-20", - "eutran-25", - "eutran-26", - "eutran-28", - "eutran-38", - "eutran-39", - "eutran-40", - "eutran-41", - "utran-19" - ], - "current-capabilities": [ - "gsm-umts, lte" - ], - "current-modes": "allowed: 2g, 3g, 4g; preferred: 4g", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-1", - "device-identifier": "a7591502473ae9ffc14e992ff1621f18cc4dd408", - "drivers": [ - "option1", - "qmi_wwan" - ], - "equipment-identifier": "867929068986654", - "hardware-revision": "10000", - "manufacturer": "QUALCOMM INCORPORATED", - "model": "QUECTEL Mobile Broadband Module", - "own-numbers": [], - "physdev": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-1", - "plugin": "quectel", - "ports": [ - "cdc-wdm0 (qmi)", - "ttyUSB0 (ignored)", - "ttyUSB1 (gps)", - "ttyUSB2 (at)", - "ttyUSB3 (at)", - "wwan0 (net)" - ], - "power-state": "on", - "primary-port": "cdc-wdm0", - "primary-sim-slot": "1", - "revision": "EG25GGBR07A08M2G", - "signal-quality": { - "recent": "yes", - "value": "0" - }, - "sim": "/org/freedesktop/ModemManager1/SIM/14", - "sim-slots": [ - "/org/freedesktop/ModemManager1/SIM/14", - "/" - ], - "state": "disabled", - "state-failed-reason": "--", - "supported-bands": [ - "egsm", - "dcs", - "pcs", - "g850", - "utran-1", - "utran-4", - "utran-6", - "utran-5", - "utran-8", - "utran-2", - "eutran-1", - "eutran-2", - "eutran-3", - "eutran-4", - "eutran-5", - "eutran-7", - "eutran-8", - "eutran-12", - "eutran-13", - "eutran-18", - "eutran-19", - "eutran-20", - "eutran-25", - "eutran-26", - "eutran-28", - "eutran-38", - "eutran-39", - "eutran-40", - "eutran-41", - "utran-19" - ], - "supported-capabilities": [ - "gsm-umts, lte" - ], - "supported-ip-families": [ - "ipv4", - "ipv6", - "ipv4v6" - ], - "supported-modes": [ - "allowed: 2g; preferred: none", - "allowed: 3g; preferred: none", - "allowed: 4g; preferred: none", - "allowed: 2g, 3g; preferred: 3g", - "allowed: 2g, 3g; preferred: 2g", - "allowed: 2g, 4g; preferred: 4g", - "allowed: 2g, 4g; preferred: 2g", - "allowed: 3g, 4g; preferred: 4g", - "allowed: 3g, 4g; preferred: 3g", - "allowed: 2g, 3g, 4g; preferred: 4g", - "allowed: 2g, 3g, 4g; preferred: 3g", - "allowed: 2g, 3g, 4g; preferred: 2g" - ], - "unlock-required": "sim-pin2", - "unlock-retries": [ - "sim-pin (3)", - "sim-puk (10)", - "sim-pin2 (3)", - "sim-puk2 (10)" - ] - } - } -} \ No newline at end of file diff --git a/tests/test_modemcard_manager.lua b/tests/test_modemcard_manager.lua deleted file mode 100644 index 2d51b839..00000000 --- a/tests/test_modemcard_manager.lua +++ /dev/null @@ -1,264 +0,0 @@ -local assertions = require "assertions" -local fiber = require "fibers.fiber" -local context = require "fibers.context" -local channel = require "fibers.channel" -local op = require "fibers.op" -local sleep = require "fibers.sleep" -local queue = require "fibers.queue" -local bus_pkg = require "bus" -local test_utils = require "test_utils.utils" -local json = require "dkjson" -local file = require "fibers.stream.file" - -local shim_dir = "./test_modemcard_manager/shims/" - --- set module loading to mot cache driver or mmcli for shims -local module_loader = test_utils.new_module_loader() -module_loader:add_uncacheable("services.hal.drivers.modem") -module_loader:add_uncacheable("services.hal.drivers.modem.mmcli") - -local function setup_test_contexts() - local bg_context = context.background() - local test_context = context.with_cancel(bg_context) - local fiber_context = context.with_cancel(context.with_value(test_context, "service_name", "dummy-service")) - - return test_context, fiber_context -end - -local function make_communication() - return - channel.new(), -- detect - channel.new(), -- remove - channel.new(), -- config - queue.new() -- device events -end - --- Detects 5 Modem events: --- No modems connected (also invalid parse case), --- modem detected, --- modem detected (diff identitiy), --- modem disconnected, --- modem disconnected (diff identitiy) -local function test_monitor_modems() - test_utils.update_shim_path(shim_dir, "monitor_modems") - - local modem_manager = module_loader:require("services.hal.managers.modemcard") - local modem_manager_instance = modem_manager.new() - - local detect_channel, remove_channel, config_channel, _ = make_communication() - modem_manager_instance.modem_detect_channel = detect_channel - modem_manager_instance.modem_remove_channel = remove_channel - - local test_context, fiber_context = setup_test_contexts() - - local channel_outputs = {} - - -- modem event listening fiber - fiber.spawn(function () - while not fiber_context:err() do - op.choice( - detect_channel:get_op():wrap(function (address) - table.insert(channel_outputs, {state="detect", address = address}) - end), - remove_channel:get_op():wrap(function (address) - table.insert(channel_outputs, {state="remove", address = address}) - end), - fiber_context:done_op() - ):perform() - end - end) - - -- spinning up the detector with a mmcli.lua shim - fiber.spawn(function () - modem_manager_instance:detector(fiber_context) - end) - - sleep.sleep(1) - - test_context:cancel('test over') - - local expected_modem_events = { - {state = "detect", address = '/org/freedesktop/ModemManager1/Modem/0'}, - {state = "detect", address = '/org/freedesktop/ModemManager1/Modem/1'}, - {state = "remove", address = '/org/freedesktop/ModemManager1/Modem/0'}, - {state = "remove", address = '/org/freedesktop/ModemManager1/Modem/1'} - } - - assertions.assert_table(expected_modem_events, channel_outputs, "modem_events") - test_context:cancel('test-end') -end - -local function test_handle_detection() - test_utils.update_shim_path(shim_dir, "monitor_modems") - - local modem_manager = module_loader:require("services.hal.managers.modemcard") - local modem_manager_instance = modem_manager.new() - - local test_context, fiber_context = setup_test_contexts() - - local detect_channel, remove_channel, config_channel, device_event_q = make_communication() - modem_manager_instance.modem_detect_channel = detect_channel - modem_manager_instance.modem_remove_channel = remove_channel - - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - - -- emulate the connection of a modem - fiber.spawn(function() - detect_channel:put("address/0") - end) - - local expected_event = { - connected = true, - type = 'usb', - id_field = 'port', - data = { - device = 'modemcard', - port = "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-1" - } - } - - -- spinning up the manager - fiber.spawn(function() - modem_manager_instance:manager(fiber_context, bus:connect(), device_event_q) - end) - - -- listen for device events - local event = device_event_q:get() - - -- check all fields apart from capabilities - assertions.assert_table(expected_event, event, "device_event") - test_context:cancel('test-end') -end - -local function test_handle_detection_no_exist() - test_utils.update_shim_path(shim_dir, "no_device") - - local modem_manager = module_loader:require("services.hal.managers.modemcard") - local modem_manager_instance = modem_manager.new() - - local test_context, fiber_context = setup_test_contexts() - - local detect_channel, remove_channel, config_channel, device_event_q = make_communication() - modem_manager_instance.modem_detect_channel = detect_channel - modem_manager_instance.modem_remove_channel = remove_channel - - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - - -- emulate the connection of a modem - fiber.spawn(function() - detect_channel:put("address/0") - end) - - -- spinning up the manager - fiber.spawn(function() - modem_manager_instance:manager(fiber_context, bus:connect(), device_event_q) - end) - - -- listen for device events for specific amount of time - local event = op.choice( - device_event_q:get_op(), - sleep.sleep_op(0.5) - ):perform() - - -- check that nothing was received in time frame - assert(event == nil, string.format("expected event to be nil but got %s", assertions.to_str(event))) - test_context:cancel('test-end') -end - -local function test_handle_removal() - test_utils.update_shim_path(shim_dir, "monitor_modems") - - local modem_manager = module_loader:require("services.hal.managers.modemcard") - local modem_manager_instance = modem_manager.new() - - local test_context, fiber_context = setup_test_contexts() - - local detect_channel, remove_channel, config_channel, device_event_q = make_communication() - modem_manager_instance.modem_detect_channel = detect_channel - modem_manager_instance.modem_remove_channel = remove_channel - - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - - -- emulate the connection and disconnection of a modem - fiber.spawn(function() - detect_channel:put('address/0') - sleep.sleep(0.1) - remove_channel:put('address/0') - end) - - local expected_event = { - connected = false, - type = 'usb', - id_field = 'port', - data = { - device = 'modemcard', - port = "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-1" - } - } - - -- spinning up the manager - fiber.spawn(function() - modem_manager_instance:manager(fiber_context, bus:connect(), device_event_q) - end) - - -- listen for device events - - local event = device_event_q:get() - -- We only care about removals - while event.connected == true do - event = device_event_q:get() - end - - -- check all fields apart from capabilities - assertions.assert_table(expected_event, event, "device_event") - test_context:cancel('test-end') -end - --- test what happens if we remove a modem which never connected -local function test_handle_removal_no_exist() - test_utils.update_shim_path(shim_dir, "monitor_modems") - - local modem_manager = module_loader:require("services.hal.managers.modemcard") - local modem_manager_instance = modem_manager.new() - - local test_context, fiber_context = setup_test_contexts() - - local detect_channel, remove_channel, config_channel, device_event_q = make_communication() - modem_manager_instance.modem_detect_channel = detect_channel - modem_manager_instance.modem_remove_channel = remove_channel - - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - - -- emulate the connection and disconnection of a modem - fiber.spawn(function() - remove_channel:put('address/0') - end) - - -- spinning up the manager - fiber.spawn(function() - modem_manager_instance:manager(fiber_context, bus:connect(), device_event_q) - end) - - -- listen for device events, expect nothing - local event = op.choice( - device_event_q:get_op(), - sleep.sleep_op(0.5) - ):perform() - - -- check that nothing was recieved - assert(event == nil, string.format("expected event to be nil but got %s", assertions.to_str(event))) - test_context:cancel('test-end') -end - -fiber.spawn(function () - test_monitor_modems() - test_handle_detection() - test_handle_detection_no_exist() - test_handle_removal() - test_handle_removal_no_exist() - fiber.stop() -end) - -print("running modem card manager tests") -fiber.main() -print("passed") diff --git a/tests/test_modemcard_manager/shims/monitor_modems/services/hal/mmcli.lua b/tests/test_modemcard_manager/shims/monitor_modems/services/hal/mmcli.lua deleted file mode 100644 index 7b177ddf..00000000 --- a/tests/test_modemcard_manager/shims/monitor_modems/services/hal/mmcli.lua +++ /dev/null @@ -1,43 +0,0 @@ -local sleep = require "fibers.sleep" -local file = require "fibers.stream.file" -local simu_commands = require "SimuCommands" - --- Goes through a set of modem monitor connection/disconnection messages --- with time delays to simulate cards being added and removed -local function monitor_modems() - local cmd, _ = simu_commands.new('./test_modemcard_manager/shims/monitor_modems/services/hal/modemcard_states.json') - return cmd -end - -local function monitor_state(device) - return simu_commands.new('./test_modemcard_manager/shims/monitor_modems/services/hal/modem_states.json') -end - --- Preset modem card info in the format of an mmcli -m request -local function information(ctx, device) - return { - combined_output = function () - sleep.sleep(0.05) - local cardfile, err = file.open( - "test_modemcard_manager/shims/monitor_modems/services/hal/modemcard_info.json", "r") - if err then return nil end - local carddata = cardfile:read_all_chars() - return carddata - end - } -end - -local function location_information() - return { - combined_output = function() - sleep.sleep(0.05) - return "" - end - } -end -return { - monitor_modems = monitor_modems, - monitor_state = monitor_state, - information = information, - location_information = location_information -} diff --git a/tests/test_modemcard_manager/shims/monitor_modems/services/hal/modem_states.json b/tests/test_modemcard_manager/shims/monitor_modems/services/hal/modem_states.json deleted file mode 100644 index f10fab77..00000000 --- a/tests/test_modemcard_manager/shims/monitor_modems/services/hal/modem_states.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "events": [ - { - "out": "/org/freedesktop/ModemManager1/Modem/0: Initial state, 'disabled'", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'disabled' --> 'enabling' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'enabling' --> 'searching' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'searching' --> 'disabling' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: State changed, 'disabling' --> 'disabled' (Reason: User request)", - "wait": 0.1 - }, - { - "out": "/org/freedesktop/ModemManager1/Modem/0: Removed", - "wait": 0 - }, - { - "out": "This will not parse", - "wait": 0 - } - ] -} diff --git a/tests/test_modemcard_manager/shims/monitor_modems/services/hal/modemcard_info.json b/tests/test_modemcard_manager/shims/monitor_modems/services/hal/modemcard_info.json deleted file mode 100644 index 2f609002..00000000 --- a/tests/test_modemcard_manager/shims/monitor_modems/services/hal/modemcard_info.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "modem": { - "3gpp": { - "5gnr": { - "registration-settings": { - "drx-cycle": "--", - "mico-mode": "--" - } - }, - "enabled-locks": [ - "fixed-dialing" - ], - "eps": { - "initial-bearer": { - "dbus-path": "--", - "settings": { - "apn": "", - "ip-type": "ipv4v6", - "password": "--", - "user": "--" - } - }, - "ue-mode-operation": "csps-2" - }, - "imei": "867929068986654", - "operator-code": "--", - "operator-name": "--", - "packet-service-state": "--", - "pco": "--", - "registration-state": "--" - }, - "cdma": { - "activation-state": "--", - "cdma1x-registration-state": "--", - "esn": "--", - "evdo-registration-state": "--", - "meid": "--", - "nid": "--", - "sid": "--" - }, - "dbus-path": "/org/freedesktop/ModemManager1/Modem/14", - "generic": { - "access-technologies": [], - "bearers": [], - "carrier-configuration": "ROW_Generic_3GPP", - "carrier-configuration-revision": "0501081F", - "current-bands": [ - "egsm", - "dcs", - "pcs", - "g850", - "utran-1", - "utran-4", - "utran-6", - "utran-5", - "utran-8", - "utran-2", - "eutran-1", - "eutran-2", - "eutran-3", - "eutran-4", - "eutran-5", - "eutran-7", - "eutran-8", - "eutran-12", - "eutran-13", - "eutran-18", - "eutran-19", - "eutran-20", - "eutran-25", - "eutran-26", - "eutran-28", - "eutran-38", - "eutran-39", - "eutran-40", - "eutran-41", - "utran-19" - ], - "current-capabilities": [ - "gsm-umts, lte" - ], - "current-modes": "allowed: 2g, 3g, 4g; preferred: 4g", - "device": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-1", - "device-identifier": "a7591502473ae9ffc14e992ff1621f18cc4dd408", - "drivers": [ - "option1", - "qmi_wwan" - ], - "equipment-identifier": "867929068986654", - "hardware-revision": "10000", - "manufacturer": "QUALCOMM INCORPORATED", - "model": "QUECTEL Mobile Broadband Module", - "own-numbers": [], - "physdev": "/sys/devices/platform/axi/1000120000.pcie/1f00300000.usb/xhci-hcd.1/usb3/3-1", - "plugin": "quectel", - "ports": [ - "cdc-wdm0 (qmi)", - "ttyUSB0 (ignored)", - "ttyUSB1 (gps)", - "ttyUSB2 (at)", - "ttyUSB3 (at)", - "wwan0 (net)" - ], - "power-state": "on", - "primary-port": "cdc-wdm0", - "primary-sim-slot": "1", - "revision": "EG25GGBR07A08M2G", - "signal-quality": { - "recent": "yes", - "value": "0" - }, - "sim": "--", - "sim-slots": [ - "/org/freedesktop/ModemManager1/SIM/14", - "/" - ], - "state": "disabled", - "state-failed-reason": "--", - "supported-bands": [ - "egsm", - "dcs", - "pcs", - "g850", - "utran-1", - "utran-4", - "utran-6", - "utran-5", - "utran-8", - "utran-2", - "eutran-1", - "eutran-2", - "eutran-3", - "eutran-4", - "eutran-5", - "eutran-7", - "eutran-8", - "eutran-12", - "eutran-13", - "eutran-18", - "eutran-19", - "eutran-20", - "eutran-25", - "eutran-26", - "eutran-28", - "eutran-38", - "eutran-39", - "eutran-40", - "eutran-41", - "utran-19" - ], - "supported-capabilities": [ - "gsm-umts, lte" - ], - "supported-ip-families": [ - "ipv4", - "ipv6", - "ipv4v6" - ], - "supported-modes": [ - "allowed: 2g; preferred: none", - "allowed: 3g; preferred: none", - "allowed: 4g; preferred: none", - "allowed: 2g, 3g; preferred: 3g", - "allowed: 2g, 3g; preferred: 2g", - "allowed: 2g, 4g; preferred: 4g", - "allowed: 2g, 4g; preferred: 2g", - "allowed: 3g, 4g; preferred: 4g", - "allowed: 3g, 4g; preferred: 3g", - "allowed: 2g, 3g, 4g; preferred: 4g", - "allowed: 2g, 3g, 4g; preferred: 3g", - "allowed: 2g, 3g, 4g; preferred: 2g" - ], - "unlock-required": "sim-pin2", - "unlock-retries": [ - "sim-pin (3)", - "sim-puk (10)", - "sim-pin2 (3)", - "sim-puk2 (10)" - ] - } - } -} diff --git a/tests/test_modemcard_manager/shims/monitor_modems/services/hal/modemcard_states.json b/tests/test_modemcard_manager/shims/monitor_modems/services/hal/modemcard_states.json deleted file mode 100644 index 9dcbf528..00000000 --- a/tests/test_modemcard_manager/shims/monitor_modems/services/hal/modemcard_states.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "events": [ - { - "out": "No modems were found", - "wait": 0.1 - }, - { - "out": "(+) /org/freedesktop/ModemManager1/Modem/0 [QUALCOMM INCORPORATED] QUECTEL Mobile Broadband Module", - "wait": 0.1 - }, - { - "out": "(+) /org/freedesktop/ModemManager1/Modem/1 [QUALCOMM INCORPORATED] QUECTEL Mobile Broadband Module", - "wait": 0.1 - }, - { - "out": "(-) /org/freedesktop/ModemManager1/Modem/0 [QUALCOMM INCORPORATED] QUECTEL Mobile Broadband Module", - "wait": 0.1 - }, - { - "out": "(-) /org/freedesktop/ModemManager1/Modem/1 [QUALCOMM INCORPORATED] QUECTEL Mobile Broadband Module", - "wait": 0.1 - } - ] -} diff --git a/tests/test_modemcard_manager/shims/no_device/services/hal/mmcli.lua b/tests/test_modemcard_manager/shims/no_device/services/hal/mmcli.lua deleted file mode 100644 index 418fbace..00000000 --- a/tests/test_modemcard_manager/shims/no_device/services/hal/mmcli.lua +++ /dev/null @@ -1,17 +0,0 @@ -local sleep = require "fibers.sleep" - --- For testing a modem not existing we only need --- the information function, this will signify a fail --- during modem_driver init function -local function information(ctx, device) - return { - combined_output = function() - sleep.sleep(0.05) - return nil, "no device of address 0" - end - } -end - -return { - information = information -} diff --git a/tests/test_service.lua b/tests/test_service.lua deleted file mode 100644 index e69a644a..00000000 --- a/tests/test_service.lua +++ /dev/null @@ -1,289 +0,0 @@ -local service = require "service" -local sleep = require "fibers.sleep" -local fiber = require "fibers.fiber" -local bus_pkg = require "bus" -local new_msg = bus_pkg.new_msg -local context = require "fibers.context" -local sc = require "fibers.utils.syscall" - -local function test_service_states() - -- make a fake service - local dummy_service = {} - dummy_service.__index = dummy_service - - dummy_service.name = 'dummy-service' - - function dummy_service:start(ctx, bus_conn) - --nothing - end - - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local bus_connection = bus:connect() - - local bg_ctx = context.background() - local ctx = context.with_cancel(bg_ctx) - - local expected_states = {'active', 'disabled'} - local states = {} - - -- fiber to listen for service health updates - fiber.spawn(function () - local service_health_sub = bus_connection:subscribe({ 'dummy-service', 'health' }) - while not ctx:err() do - local msg = service_health_sub:next_msg() - if msg.payload ~= '' then - table.insert(states, msg.payload.state) - end - end - end) - - service.spawn(dummy_service, bus, ctx) - - -- send shutdown signal to service - bus_connection:publish(new_msg( - { 'dummy-service', 'control', 'shutdown' }, - { cause = "shutdown-service", deadline = sc.monotime() + 0.1 }, - { retained = true } - )) - - -- a little time for messages to propagate - sleep.sleep(0.11) - - ctx:cancel('shutdown') - - -- check service states - for i=1, 2 do - assert(states[i] == expected_states[i], "service states was "..(states[i] or 'nil').." expected "..expected_states[i]) - end - - assert(#states == 2, 'service should have gone through 2 states, '..#states..' detected') -end - -local function test_fiber_states() - local dummy_service = {} - dummy_service.__index = dummy_service - - dummy_service.name = 'dummy-service' - - -- service spins up a fiber that waits for 0.1 seconds before exiting - function dummy_service:start(ctx, bus_conn) - service.spawn_fiber('sleep-fiber', bus_conn, ctx, function (fctx) - sleep.sleep(0.1) - end) - end - - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local bus_connection = bus:connect() - - local bg_ctx = context.background() - local ctx = context.with_cancel(bg_ctx) - - local expected_states = {'initialising', 'active', 'disabled'} - local states = {} - - -- fiber to listen for fiber health updates - fiber.spawn(function () - local fiber_health_sub = bus_connection:subscribe({ 'dummy-service', 'health', 'fibers', 'sleep-fiber' }) - while not ctx:err() do - local msg = fiber_health_sub:next_msg() - if msg.payload ~= '' then - table.insert(states, msg.payload.state) - end - end - end) - - -- let listening fiber spin up - fiber.yield() - - service.spawn(dummy_service, bus, ctx) - - -- send shutdown signal to service - bus_connection:publish(new_msg( - { 'dummy-service', 'control', 'shutdown' }, - { cause = "shutdown-service", deadline = sc.monotime() + 0.1 }, - { retained = true } - )) - - -- let shutdown signal propagate - sleep.sleep(0.2) - - ctx:cancel('shutdown') - - -- check fiber states - for i=1, 3 do - assert(states[i] == expected_states[i], "service states was "..(states[i] or 'nil').." expected "..expected_states[i]) - end - - assert(#states == 3, 'service should have gone through 3 states, '..#states..' detected') -end - -local function check_fiber_state(bus_conn, service_name, fiber_name) - local sub = bus_conn:subscribe({ service_name, 'health', 'fibers', fiber_name }) - local state_msg = sub:next_msg() - sub:unsubscribe() - return state_msg.payload.state -end - -local function check_service_state(bus_conn, service_name) - local sub = bus_conn:subscribe({ service_name, 'health' }) - local state_msg = sub:next_msg() - sub:unsubscribe() - return state_msg.payload.state -end - -local function test_blocked_shutdown() - local dummy_service = {} - dummy_service.__index = dummy_service - - dummy_service.name = 'dummy-service' - - -- service will create a fiber that will run endlessly, therefore - -- blocking the shutdown of the service - function dummy_service:start(ctx, bus_conn) - service.spawn_fiber('stuck-loop', bus_conn, ctx, function (fctx) - local i = 0 - while true do - i = i + 1 - fiber.yield() - end - end) - end - - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local bus_connection = bus:connect() - - local bg_ctx = context.background() - local ctx = context.with_cancel(bg_ctx) - - service.spawn(dummy_service, bus, ctx) - - -- send shutdown signal to service - bus_connection:publish(new_msg( - { 'dummy-service', 'control', 'shutdown' }, - { cause = "shutdown-service", deadline = sc.monotime() + 0.1 }, - { retained = true } - )) - - -- wait for messages to propegate - sleep.sleep(0.1) - - local fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'stuck-loop') - assert(fiber_state == 'active', 'stuck-loop should be active but is '..fiber_state) - - local service_state = check_service_state(bus_connection, 'dummy-service') - assert(service_state == 'active', 'dummy-service should be active but is '..service_state) -end - -local function test_timed_shutdown() - local dummy_service = {} - dummy_service.__index = dummy_service - - dummy_service.name = 'dummy-service' - - -- service will spin up - function dummy_service:start(ctx, bus_conn) - service.spawn_fiber('time-dependant', bus_conn, ctx, function (fctx) - sleep.sleep(0.2) - end) - end - - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local bus_connection = bus:connect() - - local bg_ctx = context.background() - local ctx = context.with_cancel(bg_ctx) - - service.spawn(dummy_service, bus, ctx) - - -- send shutdown signal to service - bus_connection:publish(new_msg( - { 'dummy-service', 'control', 'shutdown' }, - { cause = "shutdown-service", deadline = sc.monotime() + 0.4 }, - { retained = true } - )) - - -- wait for service and fiber to spin up - sleep.sleep(0.1) - - local fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'time-dependant') - assert(fiber_state == 'active', 'time-dependant should be active but is '..fiber_state) - - local service_state = check_service_state(bus_connection, 'dummy-service') - assert(service_state == 'active', 'dummy-service should be active but is '..service_state) - - -- wait for fiber to finish - sleep.sleep(0.31) - - fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'time-dependant') - assert(fiber_state == 'disabled', 'time-dependant should be disabled but is '..fiber_state) - - service_state = check_service_state(bus_connection, 'dummy-service') - assert(service_state == 'disabled', 'dummy-service should be disabled but is '..service_state) -end - -local function test_context_shutdown() - local dummy_service = {} - dummy_service.__index = dummy_service - - dummy_service.name = 'dummy-service' - - -- service creates a fiber that requires a context cancellation in order to exit - function dummy_service:start(sctx, bus_conn) - service.spawn_fiber('ctx-dependant', bus_conn, sctx, function (fctx) - local i = 0 - while not fctx:err() do - i = i + 1 - fiber.yield() - end - end) - end - - local bus = bus_pkg.new({ q_length = 10, m_wild = '#', s_wild = '+', sep = "/" }) - local bus_connection = bus:connect() - - local bg_ctx = context.background() - local ctx = context.with_cancel(bg_ctx) - - service.spawn(dummy_service, bus, ctx) - - local fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'ctx-dependant') - assert(fiber_state == 'initialising', 'ctx-dependant should be initialising but is '..fiber_state) - - -- let fiber spin up - sleep.sleep(0.1) - - fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'ctx-dependant') - assert(fiber_state == 'active', 'ctx-dependant should be active but is '..fiber_state) - - local service_state = check_service_state(bus_connection, 'dummy-service') - assert(service_state == 'active', 'dummy-service should be active but is '..service_state) - - -- send shutdown signal to service - bus_connection:publish(new_msg( - { 'dummy-service', 'control', 'shutdown' }, - { cause = "shutdown-service", deadline = sc.monotime() + 0.1 }, - { retained = true } - )) - - -- wait for messages to propegate - sleep.sleep(0.11) - - fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'ctx-dependant') - assert(fiber_state == 'disabled', 'ctx-dependant should be disabled but is '..fiber_state) - - service_state = check_service_state(bus_connection, 'dummy-service') - assert(service_state == 'disabled', 'dummy-service should be disabled but is '..service_state) -end - -fiber.spawn(function () - test_service_states() - test_fiber_states() - test_blocked_shutdown() - test_timed_shutdown() - -- test_context_shutdown() -- check what is up with this test, it is failing - fiber.stop() -end) - -print("starting service tests") -fiber.main() -print("tests complete") diff --git a/tests/test_submodules.lua b/tests/test_submodules.lua deleted file mode 100644 index 53fe3621..00000000 --- a/tests/test_submodules.lua +++ /dev/null @@ -1,15 +0,0 @@ -package.path = "../src/lua-fibers/?.lua;" - .. "../src/lua-trie/src/?.lua;" - .. "../src/lua-bus/src/?.lua;" - .. "../src/?.lua;" - .. package.path - .. ";/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua" - -print("starting submodule tests") - -assert(require 'fibers.fiber') -assert(require 'trie') -assert(require 'uuid') -assert(require 'bus') - -print("tests complete") diff --git a/tests/test_system.lua b/tests/test_system.lua deleted file mode 100644 index bde55359..00000000 --- a/tests/test_system.lua +++ /dev/null @@ -1,68 +0,0 @@ --- Detect if this file is being run as the entry point -local this_file = debug.getinfo(1, "S").source:match("@?([^/]+)$") -local is_entry_point = arg and arg[0] and arg[0]:match("[^/]+$") == this_file - -if is_entry_point then - package.path = "../src/lua-fibers/?.lua;" -- fibers submodule src - .. "../src/lua-trie/src/?.lua;" -- trie submodule src - .. "../src/lua-bus/src/?.lua;" -- bus submodule src - .. "../src/?.lua;" - .. "./test_utils/?.lua;" - .. package.path - .. ";/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua" - - _G._TEST = true -- Enable test exports in source code -end - -local luaunit = require 'luaunit' -local system = require 'services.system' - -local build_table = system.build_table -local merge_tables = system.merge_tables - -TestSystemMetrics = {} - -function TestSystemMetrics:test_metric_shapes_and_nesting() - local stats = {} - - -- 1) Single-key, scalar value (e.g. temperature) - merge_tables(stats, build_table({ 'temperature' }, 42)) - luaunit.assertEquals(stats.temperature, 42) - - -- 2) Multi-key, scalar value (e.g. hardware.revision) - merge_tables(stats, build_table({ 'hardware', 'revision' }, 'rev1')) - luaunit.assertNotNil(stats.hardware) - luaunit.assertEquals(stats.hardware.revision, 'rev1') - - -- 3) Single-key, table value (e.g. cpu utilisation struct) - local cpu_value = { - overall_utilisation = 12.5, - core_utilisations = { cpu0 = 34.0 }, - } - merge_tables(stats, build_table({ 'cpu' }, cpu_value)) - - -- Ensure CPU table is nested under 'cpu', not flattened - luaunit.assertNotNil(stats.cpu) - luaunit.assertEquals(stats.cpu.overall_utilisation, 12.5) - luaunit.assertEquals(stats.cpu.core_utilisations.cpu0, 34.0) - luaunit.assertNil(stats.overall_utilisation) - - -- 4) Multi-key, table value (generic nested case) - local mem_extra = { foo = 1, bar = 2 } - merge_tables(stats, build_table({ 'mem', 'extra' }, mem_extra)) - - luaunit.assertNotNil(stats.mem) - luaunit.assertNotNil(stats.mem.extra) - luaunit.assertEquals(stats.mem.extra.foo, 1) - luaunit.assertEquals(stats.mem.extra.bar, 2) -end - --- Only run tests if this file is executed directly (not via dofile) -if is_entry_point then - local fiber = require 'fibers.fiber' - fiber.spawn(function() - luaunit.LuaUnit.run() - fiber.stop() - end) - fiber.main() -end diff --git a/tests/test_utils/SimuCommands.lua b/tests/test_utils/SimuCommands.lua deleted file mode 100644 index 95f64bb8..00000000 --- a/tests/test_utils/SimuCommands.lua +++ /dev/null @@ -1,49 +0,0 @@ -local json = require "dkjson" -local file = require "fibers.stream.file" -local queue = require "fibers.queue" -local fiber = require "fibers.fiber" -local sleep = require "fibers.sleep" - -local StreamCommand = {} -StreamCommand.__index = StreamCommand - --- Given a json file, acts as a fake command that outputs --- defined messages and blocks for a set time -function StreamCommand.new(events_directory) - local events_file, err = file.open(events_directory, "r") - if err then return nil end - local events_data = events_file:read_all_chars() - - local self = {} - self.event_q = queue.new() - self.events = json.decode(events_data).events - return setmetatable(self, StreamCommand) -end - -function StreamCommand:stdout_pipe() - return { - close = function () return end, - lines = function () - local event_iterator = function () - local message = self.event_q:get() - return message - end - return event_iterator - end - } -end - -function StreamCommand:start() - fiber.spawn(function () - for _, event in ipairs(self.events) do - self.event_q:put(event.out) - sleep.sleep(event.wait) - end - end) -end - -function StreamCommand:wait() - sleep.sleep(0.5) -end - -return {new = StreamCommand.new} diff --git a/tests/test_utils/assertions.lua b/tests/test_utils/assertions.lua deleted file mode 100644 index de9ea417..00000000 --- a/tests/test_utils/assertions.lua +++ /dev/null @@ -1,35 +0,0 @@ -local json = require 'dkjson' - -local function to_str(value) - if type(value) == 'nil' then return "nil" end - return value -end --- Traverses a table, excludes function attributes -local function assert_table(expected, recieved, root_key) - assert(type(expected) == 'table') - assert(type(recieved) == 'table', string.format( - '%s: expected a table but recieved %s', - root_key, - type(recieved) - )) - - for k, v in pairs(expected) do - local key = string.format("%s.%s", root_key, k) - if type(v) == 'table' then - assert_table(v, recieved[k], key) - else - if type(v) ~= 'function' then - assert(v == recieved[k], string.format("%s: expected %s but got %s", key, v, to_str(recieved[k]))) - end - end - end -end - -local function expect_assert(expected, result, name) - assert(expected == result, string.format('%s expected %s but got %s', name, to_str(expected), to_str(result))) -end -return { - assert_table = assert_table, - to_str = to_str, - expect_assert = expect_assert -} diff --git a/tests/test_utils/shim_shifter.lua b/tests/test_utils/shim_shifter.lua deleted file mode 100644 index 11210fe5..00000000 --- a/tests/test_utils/shim_shifter.lua +++ /dev/null @@ -1,35 +0,0 @@ -local ShimShifter = {} -ShimShifter.__index = ShimShifter - -local function new() - local self = setmetatable({}, ShimShifter) - self.uncached_modules = {} - self.package_path_og = package.path - return self -end - -function ShimShifter:add_uncacheable(module) - table.insert(self.uncached_modules, module) -end - -function ShimShifter:set_shim(path) - for _, uncache_module in ipairs(self.uncached_modules) do - package.loaded[uncache_module] = nil - end - package.path = path .. "/?.lua;" .. self.package_path_og -end - -function ShimShifter:require(module) - return require(module) -end - -function ShimShifter:reset() - for _, uncache_module in ipairs(self.uncached_modules) do - package.loaded[uncache_module] = nil - end - package.path = self.package_path_og -end - -return { - new = new -} diff --git a/tests/test_utils/utils.lua b/tests/test_utils/utils.lua deleted file mode 100644 index 65da94f5..00000000 --- a/tests/test_utils/utils.lua +++ /dev/null @@ -1,42 +0,0 @@ -local path = package.path - -local function update_shim_path(shim_base_dir, shim_name) - package.path = shim_base_dir .. shim_name .. "/?.lua;" .. path -end - --- Old implementation of loading packages after shim switching, used for driver tests -local function uncached_require(module) - package.loaded[module] = nil - package.loaded['services.hal.mmcli'] = nil - package.loaded['services.hal.qmicli'] = nil - package.loaded['services.hal.at'] = nil - package.loaded['services.hal.modem_driver'] = nil - return require(module) -end - --- new implemeentation of loading packages after shim switching --- build up a list of modules to be purged every time a package is loaded --- makes sure packages stay up to date -local ModuleLoader = {} -ModuleLoader.__index = ModuleLoader - -local function new_module_loader() - return setmetatable({ uncached_modules = {} }, ModuleLoader) -end - -function ModuleLoader:add_uncacheable(module) - table.insert(self.uncached_modules, module) -end - -function ModuleLoader:require(module) - for _, uncache_module in ipairs(self.uncached_modules) do - package.loaded[uncache_module] = nil - end - package.loaded[module] = nil - return require(module) -end -return { - update_shim_path = update_shim_path, - uncached_require = uncached_require, - new_module_loader = new_module_loader -} diff --git a/tests/test_wifi.lua b/tests/test_wifi.lua deleted file mode 100644 index 2d0ae9af..00000000 --- a/tests/test_wifi.lua +++ /dev/null @@ -1,235 +0,0 @@ --- Detect if this file is being run as the entry point -local this_file = debug.getinfo(1, "S").source:match("@?([^/]+)$") -local is_entry_point = arg and arg[0] and arg[0]:match("[^/]+$") == this_file - -if is_entry_point then - package.path = "../src/lua-fibers/?.lua;" -- fibers submodule src - .. "../src/lua-trie/src/?.lua;" -- trie submodule src - .. "../src/lua-bus/src/?.lua;" -- bus submodule src - .. "../src/?.lua;" - .. "./test_utils/?.lua;" - .. package.path - .. ";/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua" - - _G._TEST = true -- Enable test exports in source code -end - -local luaunit = require 'luaunit' -local fiber = require "fibers.fiber" -local context = require "fibers.context" -local sleep = require "fibers.sleep" - -TestWifiClientEvents = {} - -local function setup_wifi_test_environment() - local bus_pkg = require "bus" - local bus = bus_pkg.new() - luaunit.assertNotNil(bus) - - local wifi_service = require "services.wifi" - luaunit.assertNotNil(wifi_service.wifi_service) - luaunit.assertNotNil(wifi_service.new_radio) - - return wifi_service, bus, bus_pkg.new_msg -end - -local function ssid_path(radio_index, interface) - return { 'hal', 'capability', 'wireless', radio_index, 'info', 'interface', interface, 'ssid' } -end - -local function client_event_path(radio_index, interface, client_mac) - return { 'hal', 'capability', 'wireless', radio_index, 'info', 'interface', interface, 'client', client_mac } -end - -local function session_start_path() - return { 'wifi', 'clients', '+', 'sessions', '+', 'session_start' } -end - -local function session_end_path() - return { 'wifi', 'clients', '+', 'sessions', '+', 'session_end' } -end - -function TestWifiClientEvents:test_normal_client_connection() - -- Setup test environment - local wifi, bus, new_msg = setup_wifi_test_environment() - local conn = bus:connect() - local ctx = context.with_cancel(context.background()) - - -- Configure radio and interface - local radio_index = "radio0" - local interface = "wlan0" - local ssid = "test_ssid" - - local radio = wifi.new_radio(ctx, bus:connect(), radio_index) - radio.band_ch:put("2g") - radio.interface_ch:put({ name = ssid, index = 1 }) - conn:publish(new_msg(ssid_path(radio_index, interface), ssid)) - fiber.yield() - - -- Subscribe before publishing the connection event - local session_sub = conn:subscribe(session_start_path()) - - -- Publish client connection event - local client_mac = "AA:BB:CC:DD:EE:FF" - local timestamp = 123456789 - conn:publish(new_msg( - client_event_path(radio_index, interface, client_mac), - { connected = true, timestamp = timestamp } - )) - - -- Verify session_start message was published with correct timestamp - local session_msg, err = session_sub:next_msg_with_context(context.with_timeout(ctx, 0.1)) - luaunit.assertNil(err, "Expected session_start message") - luaunit.assertNotNil(session_msg.payload) - luaunit.assertEquals(session_msg.payload, timestamp) -end - -function TestWifiClientEvents:test_duplicate_connection_event() - -- Setup test environment - local wifi, bus, new_msg = setup_wifi_test_environment() - local conn = bus:connect() - local ctx = context.with_cancel(context.background()) - - -- Configure radio and interface - local radio_index = "radio0" - local interface = "wlan0" - local ssid = "test_ssid" - - local radio = wifi.new_radio(ctx, bus:connect(), radio_index) - radio.band_ch:put("2g") - radio.interface_ch:put({ name = ssid, index = 1 }) - conn:publish(new_msg(ssid_path(radio_index, interface), ssid)) - fiber.yield() - - local session_sub = conn:subscribe(session_start_path()) - local client_mac = "AA:BB:CC:DD:EE:FF" - - -- Publish first connection event - conn:publish(new_msg( - client_event_path(radio_index, interface, client_mac), - { connected = true, timestamp = 1 } - )) - - -- Publish duplicate connection event (same MAC, still connected) - conn:publish(new_msg( - client_event_path(radio_index, interface, client_mac), - { connected = true, timestamp = 2 } - )) - - -- Verify first connection event created a session - local session_msg, err = session_sub:next_msg_with_context(context.with_timeout(ctx, 0.1)) - luaunit.assertNil(err, "Expected first session_start message") - luaunit.assertEquals(session_msg.payload, 1) - - -- Verify duplicate connection event was ignored (no second session_start) - local session_msg2, err2 = session_sub:next_msg_with_context(context.with_timeout(ctx, 0.1)) - luaunit.assertNotNil(err2, "Expected timeout - duplicate should be ignored") - luaunit.assertNil(session_msg2) -end - -function TestWifiClientEvents:test_normal_disconnection_event() - -- Setup test environment - local wifi, bus, new_msg = setup_wifi_test_environment() - local conn = bus:connect() - local ctx = context.with_cancel(context.background()) - - -- Configure radio and interface - local radio_index = "radio0" - local interface = "wlan0" - local ssid = "test_ssid" - - local radio = wifi.new_radio(ctx, bus:connect(), radio_index) - radio.band_ch:put("2g") - radio.interface_ch:put({ name = ssid, index = 1 }) - conn:publish(new_msg(ssid_path(radio_index, interface), ssid)) - fiber.yield() - - local client_mac = "AA:BB:CC:DD:EE:FF" - local connect_time = 100 - local disconnect_time = 200 - - -- Connect the client - conn:publish(new_msg( - client_event_path(radio_index, interface, client_mac), - { connected = true, timestamp = connect_time } - )) - fiber.yield() - - -- Subscribe before disconnecting - local session_sub = conn:subscribe(session_end_path()) - - -- Disconnect the client - conn:publish(new_msg( - client_event_path(radio_index, interface, client_mac), - { connected = false, timestamp = disconnect_time } - )) - - -- Verify session_end message was published with correct timestamp - local session_msg, err = session_sub:next_msg_with_context(context.with_timeout(ctx, 0.1)) - luaunit.assertNil(err, "Expected session_end message") - luaunit.assertNotNil(session_msg.payload) - luaunit.assertEquals(session_msg.payload, disconnect_time) -end - -function TestWifiClientEvents:test_duplicate_disconnection_event() - local wifi, bus, new_msg = setup_wifi_test_environment() - local conn = bus:connect() - local bg_ctx = context.background() - local ctx = context.with_cancel(bg_ctx) - - -- Setup: Create radio and interface - local radio_index = "radio0" - local interface = "wlan0" - local ssid = "ssid_name" - - local radio = wifi.new_radio(ctx, bus:connect(), radio_index) - radio.band_ch:put("2g") - radio.interface_ch:put({ name = ssid, index = 1 }) - conn:publish(new_msg(ssid_path(radio_index, interface), ssid)) - fiber.yield() - - local client_mac = "AA:BB:CC:DD:EE:FF" - local connect_time = 100 - local disconnect_time = 200 - - -- Connect the client first - conn:publish(new_msg( - client_event_path(radio_index, interface, client_mac), - { connected = true, timestamp = connect_time } - )) - fiber.yield() - - local session_sub = conn:subscribe(session_end_path()) - - -- Action: Publish first disconnection event - conn:publish(new_msg( - client_event_path(radio_index, interface, client_mac), - { connected = false, timestamp = disconnect_time } - )) - - -- Action: Publish duplicate disconnection event (should be ignored) - conn:publish(new_msg( - client_event_path(radio_index, interface, client_mac), - { connected = false, timestamp = disconnect_time + 1 } - )) - - -- Verify: First disconnection created a session_end message - local session_msg, err = session_sub:next_msg_with_context(context.with_timeout(ctx, 0.1)) - luaunit.assertNil(err, "Expected first session_end message") - luaunit.assertEquals(session_msg.payload, disconnect_time, "Session end timestamp should match first disconnect") - - -- Verify: Duplicate disconnection was ignored (no second session_end) - local session_msg2, err2 = session_sub:next_msg_with_context(context.with_timeout(ctx, 0.1)) - luaunit.assertNotNil(err2, "Expected timeout - duplicate disconnect should be ignored") - luaunit.assertNil(session_msg2, "No second session_end should be published") -end - --- Only run tests if this file is executed directly (not via dofile) -if is_entry_point then - fiber.spawn(function() - luaunit.LuaUnit.run() - fiber.stop() - end) - - fiber.main() -end diff --git a/tests/unit/config/codec_spec.lua b/tests/unit/config/codec_spec.lua new file mode 100644 index 00000000..3609792b --- /dev/null +++ b/tests/unit/config/codec_spec.lua @@ -0,0 +1,45 @@ +-- tests/config_codec_spec.lua + +local codec = require 'services.config.codec' + +local T = {} + +function T.decode_rejects_non_table_root() + local out, err = codec.decode_blob_strict('"x"') + assert(out == nil) + assert(tostring(err):match('root must be a table')) +end + +function T.decode_requires_schema() + local blob = [[{"net":{"rev":1,"data":{"foo":"bar"}}}]] + local out, err = codec.decode_blob_strict(blob) + assert(out == nil) + assert(tostring(err):match('data%.schema')) +end + +function T.decode_strips_json_nulls() + local blob = [[{"net":{"rev":1,"data":{"schema":"x","a":null,"b":1}}}]] + local out, err = codec.decode_blob_strict(blob) + assert(out ~= nil, tostring(err)) + assert(out.net.data.a == nil) + assert(out.net.data.b == 1) +end + +function T.encode_is_strict() + local t = {} + t.self = t + local s, err = codec.encode_blob(t) + assert(s == nil) + assert(tostring(err):match('json_encode_failed')) +end + +function T.deepcopy_plain_copies_nested_tables() + local x = { a = { b = 1 } } + local y = codec.deepcopy_plain(x) + assert(y ~= x) + assert(y.a ~= x.a) + y.a.b = 2 + assert(x.a.b == 1) +end + +return T diff --git a/tests/unit/config/service_spec.lua b/tests/unit/config/service_spec.lua new file mode 100644 index 00000000..631be3e9 --- /dev/null +++ b/tests/unit/config/service_spec.lua @@ -0,0 +1,164 @@ +-- tests/config_service_spec.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local busmod = require 'bus' + +local safe = require 'coxpcall' + +local runfibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' +local fake_hal_mod = require 'tests.support.fake_hal' + +local config_service = require 'services.config' + +local T = {} + +local function config_timings() + return { + hal_wait_timeout_s = 0.25, + hal_wait_tick_s = 0.01, + heartbeat_s = 60.0, + persist_debounce_s = 0.02, + persist_max_delay_s = 0.05, + persist_retry_initial_s = 0.02, + persist_retry_max_s = 0.05, + } +end + +function T.config_loads_from_hal_and_publishes_retained() + runfibers.run(function(scope) + local bus = busmod.new() + local conn = bus:connect() + + local fake_hal = fake_hal_mod.new({ + backend = 'fakehal', + caps = { + read_state = true, + write_state = true, + }, + scripted = { + read_state = { + { + ok = true, + found = true, + data = [[{"net":{"rev":2,"data":{"schema":"devicecode.net/1","foo":"bar"}}}]], + }, + }, + }, + }) + + fake_hal:start(bus:connect(), { name = 'hal' }) + + local ok_spawn, err = scope:spawn(function() + config_service.start(bus:connect(), { + name = 'config', + env = 'dev', + timings = config_timings(), + }) + end) + assert(ok_spawn, tostring(err)) + + local payload = probe.wait_payload(conn, { 'cfg', 'net' }, { timeout = 0.5 }) + assert(type(payload) == 'table') + assert(payload.rev == 2) + assert(type(payload.data) == 'table') + assert(payload.data.schema == 'devicecode.net/1') + assert(payload.data.foo == 'bar') + end, { timeout = 1.0 }) +end + +function T.config_accepts_set_and_persists_debounced() + runfibers.run(function(scope) + local bus = busmod.new() + local conn = bus:connect() + + local fake_hal = fake_hal_mod.new({ + backend = 'fakehal', + caps = { + read_state = true, + write_state = true, + }, + scripted = { + read_state = { + { ok = true, found = false }, + }, + write_state = { + { ok = true }, + }, + }, + }) + + fake_hal:start(bus:connect(), { name = 'hal' }) + + local ok_spawn, err = scope:spawn(function() + config_service.start(bus:connect(), { + name = 'config', + env = 'dev', + timings = config_timings(), + }) + end) + assert(ok_spawn, tostring(err)) + + local req_conn = bus:connect() + + -- Retry call until the service has definitely bound and accepted it. + local deadline = fibers.now() + 0.5 + local seen = false + + while fibers.now() < deadline do + req_conn:call({ 'cmd', 'config', 'set' }, { + service = 'net', + data = { + schema = 'devicecode.net/1', + answer = 42, + }, + }, { timeout = 0.02 }) + + seen = probe.wait_until(function() + local ok, payload = safe.pcall(function() + return probe.wait_payload(conn, { 'cfg', 'net' }, { timeout = 0.02 }) + end) + return ok + and type(payload) == 'table' + and payload.rev == 1 + and type(payload.data) == 'table' + and payload.data.answer == 42 + end, { timeout = 0.03, interval = 0.005 }) + + if seen then + break + end + + sleep.sleep(0.01) + end + + assert(seen == true, 'expected retained cfg/net update') + + local persisted = probe.wait_until(function() + for i = 1, #fake_hal.calls do + if fake_hal.calls[i].method == 'write_state' then + return true + end + end + return false + end, { timeout = 0.5, interval = 0.005 }) + + assert(persisted == true, 'expected write_state to be called') + + local write_call = nil + for i = 1, #fake_hal.calls do + if fake_hal.calls[i].method == 'write_state' then + write_call = fake_hal.calls[i] + break + end + end + + assert(write_call ~= nil) + assert(write_call.req.ns == 'config') + assert(write_call.req.key == 'services') + assert(type(write_call.req.data) == 'string') + end, { timeout = 1.0 }) +end + +return T diff --git a/tests/unit/config/state_spec.lua b/tests/unit/config/state_spec.lua new file mode 100644 index 00000000..c4810c78 --- /dev/null +++ b/tests/unit/config/state_spec.lua @@ -0,0 +1,61 @@ +-- tests/config_state_spec.lua + +local state = require 'services.config.state' + +local T = {} + +local function fake_conn() + local retained = {} + return { + retained = retained, + retain = function(_, topic, payload) + retained[#retained + 1] = { topic = topic, payload = payload } + return true + end, + } +end + +function T.set_service_increments_revision_and_copies_input() + local conn = fake_conn() + local current = {} + + local payload = { + data = { + schema = 'devicecode.test/1', + x = { y = 1 }, + }, + } + + local ok, err = state.set_service(current, conn, nil, nil, 'net', payload, nil) + assert(ok == true, tostring(err)) + assert(current.net.rev == 1) + assert(current.net.data ~= payload.data) + assert(current.net.data.x ~= payload.data.x) + + payload.data.x.y = 99 + assert(current.net.data.x.y == 1) + + local ok2, err2 = state.set_service(current, conn, nil, nil, 'net', payload, nil) + assert(ok2 == true, tostring(err2)) + assert(current.net.rev == 2) +end + +function T.publish_all_retained_publishes_copies() + local conn = fake_conn() + local current = { + net = { + rev = 3, + data = { + schema = 'devicecode.test/1', + x = { y = 1 }, + }, + }, + } + + state.publish_all_retained(conn, nil, current) + assert(#conn.retained == 1) + assert(conn.retained[1].payload ~= current.net) + assert(conn.retained[1].payload.data ~= current.net.data) +end + +return T diff --git a/tests/unit/device/test_action_manager.lua b/tests/unit/device/test_action_manager.lua new file mode 100644 index 00000000..ab35bc74 --- /dev/null +++ b/tests/unit/device/test_action_manager.lua @@ -0,0 +1,281 @@ +-- tests/unit/device/test_action_manager.lua + +local fibers = require 'fibers' +local busmod = require 'bus' +local config = require 'services.device.config' +local model_mod = require 'services.device.model' +local action_manager = require 'services.device.action_manager' +local scoped_work = require 'devicecode.support.scoped_work' +local topics = require 'services.device.topics' + +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end + +local function req(payload) + local r = { payload = payload, replied = nil, failed = nil } + function r:reply(v) self.replied = v; return true end + function r:fail(e) self.failed = e; return true end + return r +end + +function tests.test_status_read_is_immediate_model_snapshot() + local cat = assert(config.to_catalogue({ + schema = config.SCHEMA, + components = { + mcu = { subtype = 'mcu', facts = { software = topics.raw_member_state('mcu', 'software') } }, + }, + })) + local m = model_mod.new() + m:apply_catalogue(3, cat) + m:apply_observation(3, { component = 'mcu', tag = 'fact_retained', fact = 'software', payload = { version = '1.0' } }) + local r = req() + local state = { model = m, now = function () return 10 end } + local ok, err = action_manager.status_reply(state, r, 'mcu') + assert_true(ok, err) + assert_eq(r.replied.component, 'mcu') + assert_eq(r.replied.software.version, '1.0') +end + + +function tests.test_action_start_failure_after_request_owner_setup_resolves_once() + local old_start = scoped_work.start + local captured_setup + local captured_setup_finaliser + local fake_scope = { + finally = function (_, fn) + captured_setup_finaliser = fn + return function () end + end, + } + + scoped_work.start = function (spec) + captured_setup = spec.setup(fake_scope) + return nil, 'synthetic post-setup action start failure', captured_setup + end + + local r = { fail_count = 0, failed = nil } + function r:fail(e) + self.fail_count = self.fail_count + 1 + self.failed = e + return true + end + + local cat = assert(config.to_catalogue({ + schema = config.SCHEMA, + components = { + mcu = { + subtype = 'mcu', + facts = { software = topics.raw_member_state('mcu', 'software') }, + actions = { restart = { kind = 'rpc', call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') } }, + }, + }, + })) + + local state = { + _action_seq = 0, + scope = {}, + conn = nil, + done_tx = {}, + pending_actions = {}, + action_timeout = 1, + active = { + generation = 11, + action_root = {}, + catalogue = cat, + }, + } + + local ok, err = action_manager.start_action(state, r, { + generation = 11, + component = 'mcu', + action = 'restart', + }) + + scoped_work.start = old_start + + assert_nil(ok) + assert_eq(err, 'synthetic post-setup action start failure') + assert_eq(r.fail_count, 1) + assert_eq(r.failed, 'synthetic post-setup action start failure') + assert_true(captured_setup and captured_setup.request_owner ~= nil, 'setup request owner was not returned') + + -- Later structural cleanup must not resolve the raw request again. + captured_setup.request_owner:finalise_unresolved('later structural cleanup') + assert_eq(r.fail_count, 1) +end + + +function tests.test_dynamic_action_spec_rejects_payload_as_request_failure() + local rejecting_module = { + kind = 'mcu', + action_spec = function (_, _, payload, _) + if not (type(payload) == 'table' and payload.allowed == true) then + return nil, 'payload_rejected' + end + return { call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') } + end, + } + + local cat = assert(config.to_catalogue({ + schema = config.SCHEMA, + components = { + mcu = { + subtype = 'mcu', + module = rejecting_module, + facts = { software = topics.raw_member_state('mcu', 'software') }, + actions = { restart = { kind = 'rpc', call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') } }, + }, + }, + })) + + local r = req({ denied = true }) + local state = { + _action_seq = 0, + pending_actions = {}, + active = { + generation = 23, + catalogue = cat, + action_root = {}, + }, + } + + local ok, err = action_manager.start_action(state, r, { + generation = 23, + component = 'mcu', + action = 'restart', + }) + + assert_true(ok, err) + assert_nil(err) + assert_eq(r.failed, 'payload_rejected') + assert_nil(r.replied) + assert_eq(next(state.pending_actions), nil, 'rejected request must not admit action work') +end + + + +function tests.test_dynamic_action_dependency_is_registered_and_blocks_admission() + fibers.run(function () + local dynamic_module = { + kind = 'mcu', + action_spec = function (_, _, _, base_spec) + local spec = {} + for k, v in pairs(base_spec or {}) do spec[k] = v end + spec.dependency = { class = 'dynamic-control', id = 'main' } + return spec + end, + } + + local cat = assert(config.to_catalogue({ + schema = config.SCHEMA, + components = { + mcu = { + subtype = 'mcu', + module = dynamic_module, + facts = { software = topics.raw_member_state('mcu', 'software') }, + actions = { restart = { kind = 'rpc', call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') } }, + }, + }, + })) + + local b = busmod.new() + local state = { + _action_seq = 0, + conn = b:connect(), + pending_actions = {}, + dependency_queue_len = 4, + active = { generation = 41, action_root = {}, catalogue = cat }, + update_dependency_model = function (st) + st.dependency_model_updated = true + return true, nil + end, + } + local r = req({}) + + local ok, err = action_manager.start_action(state, r, { generation = 41, component = 'mcu', action = 'restart' }) + assert_true(ok, err) + assert_nil(err) + assert_eq(r.failed, 'dependency_unavailable:action:mcu:restart') + assert_not_nil(state.active.action_deps) + assert_not_nil(state.active.action_deps:dependency('action:mcu:restart')) + assert_true(state.dependency_model_updated) + assert_eq(next(state.pending_actions), nil) + state.active.action_deps:terminate('test complete') + end) +end + +function tests.test_unbind_generation_skips_unowned_synthetic_event_sources() + local active = { + action_eps = { + ['mcu:restart'] = { + key = 'mcu:restart', + ep = { recv_op = function () end }, + }, + }, + } + + local ok, err = action_manager.unbind_generation(active, nil) + assert_true(ok, err) + assert_eq(next(active.action_eps), nil) +end + +function tests.test_action_admission_passes_caller_cancel_op_to_scoped_work() + local cond = require 'fibers.cond' + local op = require 'fibers.op' + fibers.run(function () + local old_start = scoped_work.start + local done = cond.new() + local r = { payload = {}, _done = false, _status = 'pending', _err = nil } + function r:reply(_) error('reply must not be called') end + function r:fail(_) error('fail must not be called') end + function r:abandon(reason) self._done = true; self._status = 'abandoned'; self._err = reason; done:signal(); return true end + function r:done_op() + return op.guard(function () + if self._done then return op.always(self._status, nil, self._err) end + return done:wait_op():wrap(function () return self._status, nil, self._err end) + end) + end + + scoped_work.start = function (spec) + assert_not_nil(spec.cancel_op, 'action scoped_work must receive caller cancel_op') + local fake_scope = { finally = function () return function () end end } + local setup = assert(spec.setup(fake_scope)) + r:abandon('user_timeout') + local reason = fibers.perform(spec.cancel_op) + assert_eq(reason, 'user_timeout') + assert_true(setup.request_owner:done(), 'caller cancellation should abandon local owner') + return { cancel = function () return true end }, nil + end + + local cat = assert(config.to_catalogue({ + schema = config.SCHEMA, + components = { + mcu = { + subtype = 'mcu', + facts = { software = topics.raw_member_state('mcu', 'software') }, + actions = { restart = { kind = 'rpc', call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') } }, + }, + }, + })) + + local state = { + _action_seq = 0, + scope = {}, + conn = nil, + done_tx = {}, + pending_actions = {}, + action_timeout = 1, + active = { generation = 31, action_root = {}, catalogue = cat }, + } + + local ok, err = action_manager.start_action(state, r, { generation = 31, component = 'mcu', action = 'restart' }) + scoped_work.start = old_start + assert_true(ok, err) + end) +end + +return tests diff --git a/tests/unit/device/test_catalogue.lua b/tests/unit/device/test_catalogue.lua new file mode 100644 index 00000000..e198ad46 --- /dev/null +++ b/tests/unit/device/test_catalogue.lua @@ -0,0 +1,114 @@ +-- tests/unit/device/test_catalogue.lua + +local config = require 'services.device.config' +local catalogue = require 'services.device.catalogue' +local topics = require 'services.device.topics' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end + +local function sample_config() + return { + schema = config.SCHEMA, + components = { + mcu = { + class = 'member', + subtype = 'mcu', + member = 'mcu', + required_facts = { 'software' }, + facts = { + software = topics.raw_member_state('mcu', 'software'), + }, + events = { + alert = topics.raw_member_cap_event('mcu', 'telemetry', 'main', 'alert'), + }, + actions = { + restart = { kind = 'rpc', call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') }, + }, + }, + }, + } +end + +function tests.test_config_to_catalogue_normalises_component_routes() + local cat, err = config.to_catalogue(sample_config()) + assert_nil(err) + assert_not_nil(cat) + assert_not_nil(cat.components.mcu) + assert_eq(cat.components.mcu.facts.software.name, 'software') + assert_eq(cat.components.mcu.events.alert.name, 'alert') + assert_eq(cat.components.mcu.actions.restart.kind, 'rpc') + assert_eq(cat.components.mcu.actions.restart.call_topic[1], 'raw') +end + + + +function tests.test_catalogue_rejects_transitional_action_and_component_aliases() + local cfg = sample_config() + cfg.components.mcu.actions.restart = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') + local cat, err = config.to_catalogue(cfg) + assert_nil(cat) + if not tostring(err):find('must be a table with kind and call_topic', 1, true) then + fail('unexpected error: ' .. tostring(err)) + end + + cfg = sample_config() + cfg.components.mcu.actions.restart = { kind = 'rpc', topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') } + cat, err = config.to_catalogue(cfg) + assert_nil(cat) + if not tostring(err):find('deprecated topic; use call_topic', 1, true) then + fail('unexpected error: ' .. tostring(err)) + end + + cfg = sample_config() + cfg.components.mcu.actions.restart = { kind = 'rpc', call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart'), timeout = 1 } + cat, err = config.to_catalogue(cfg) + assert_nil(cat) + if not tostring(err):find('deprecated timeout; use timeout_s', 1, true) then + fail('unexpected error: ' .. tostring(err)) + end +end + +function tests.test_fabric_stage_requires_target_and_rejects_receiver() + local cfg = sample_config() + cfg.components.mcu.actions['stage-update'] = { + kind = 'fabric_stage', + } + local cat, err = config.to_catalogue(cfg) + assert_nil(cat) + if not tostring(err):find('requires fabric_stage target', 1, true) then + fail('unexpected error: ' .. tostring(err)) + end + + cfg.components.mcu.actions['stage-update'] = { + kind = 'fabric_stage', + receiver = topics.raw_member_cap_rpc('mcu', 'update', 'main', 'stage'), + } + cat, err = config.to_catalogue(cfg) + assert_nil(cat) + if not tostring(err):find('deprecated receiver; use target', 1, true) then + fail('unexpected error: ' .. tostring(err)) + end + + cfg.components.mcu.actions['stage-update'] = { + kind = 'fabric_stage', + target = 'updater/main', + } + cat, err = config.to_catalogue(cfg) + assert_nil(err) + assert_not_nil(cat.components.mcu.actions['stage-update']) +end +function tests.test_catalogue_material_comparison_is_stable_for_copies() + local a = assert(config.to_catalogue(sample_config())) + local b = catalogue.copy(a) + assert_true(catalogue.materially_equal(a, b)) + b.components.mcu.required_facts[2] = 'updater' + if catalogue.materially_equal(a, b) then fail('expected material difference') end +end + +return tests diff --git a/tests/unit/device/test_dependencies.lua b/tests/unit/device/test_dependencies.lua new file mode 100644 index 00000000..729c0be7 --- /dev/null +++ b/tests/unit/device/test_dependencies.lua @@ -0,0 +1,49 @@ +local config = require 'services.device.config' +local topics = require 'services.device.topics' +local deps = require 'services.device.dependencies' + +local T = {} + +function T.derives_explicit_action_dependency() + local cat = assert(config.to_catalogue({ + schema = config.SCHEMA, + components = { + mcu = { + class = 'member', subtype = 'mcu', member = 'mcu', + facts = { software = topics.raw_member_state('mcu', 'software') }, + actions = { + restart = { + kind = 'rpc', + call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart'), + dependency = { raw_kind = 'member', source = 'mcu', class = 'control', id = 'main' }, + }, + }, + }, + }, + })) + local specs, map, err = deps.catalogue_dependencies(cat) + assert(err == nil) + assert(#specs == 1) + assert(specs[1].key == 'action:mcu:restart') + assert(specs[1].raw_kind == 'member') + assert(specs[1].source == 'mcu') + assert(specs[1].class == 'control') + assert(map.mcu.restart == specs[1].key) +end + +function T.actions_without_metadata_have_no_dependency() + local cat = assert(config.to_catalogue({ + schema = config.SCHEMA, + components = { + mcu = { + class = 'member', subtype = 'mcu', member = 'mcu', + facts = { software = topics.raw_member_state('mcu', 'software') }, + actions = { restart = { kind = 'rpc', call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') } }, + }, + }, + })) + local specs = assert(deps.catalogue_dependencies(cat)) + assert(#specs == 0) +end + +return T diff --git a/tests/unit/device/test_model.lua b/tests/unit/device/test_model.lua new file mode 100644 index 00000000..43c9d288 --- /dev/null +++ b/tests/unit/device/test_model.lua @@ -0,0 +1,103 @@ +-- tests/unit/device/test_model.lua + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local sleep = require 'fibers.sleep' +local queue = require 'devicecode.support.queue' +local config = require 'services.device.config' +local model_mod = require 'services.device.model' +local topics = require 'services.device.topics' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_false(v, msg) if v ~= false then fail(msg or ('expected false, got ' .. tostring(v))) end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end + +local function catalogue_one() + return assert(config.to_catalogue({ + schema = config.SCHEMA, + components = { + mcu = { + subtype = 'mcu', + facts = { software = topics.raw_member_state('mcu', 'software') }, + }, + }, + })) +end + +function tests.test_snapshot_returns_copy() + fibers.run(function () + local m = model_mod.new() + m:apply_catalogue(1, catalogue_one()) + local snap = m:snapshot() + snap.components.mcu.subtype = 'mutated' + assert_eq(m:snapshot().components.mcu.subtype, 'mcu') + end) +end + +function tests.test_observation_updates_current_generation_only() + fibers.run(function () + local m = model_mod.new() + m:apply_catalogue(7, catalogue_one()) + local changed, _, err = m:apply_observation(6, { + component = 'mcu', tag = 'fact_retained', fact = 'software', payload = { version = 'old' }, + }) + assert_false(changed) + assert_eq(err, 'stale_generation') + assert_nil(m:snapshot().components.mcu.raw_facts.software) + + changed, _, err = m:apply_observation(7, { + component = 'mcu', tag = 'fact_retained', fact = 'software', payload = { version = '1.0' }, + }) + assert_true(changed) + assert_nil(err) + assert_eq(m:snapshot().components.mcu.raw_facts.software.version, '1.0') + end) +end + +function tests.test_changed_op_wakes_on_material_change() + fibers.run(function (scope) + local m = model_mod.new() + local seen = m:version() + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + local ok, err = scope:spawn(function () + local version, snap, cerr = fibers.perform(m:changed_op(seen)) + queue.assert_admit_required(tx, { version = version, snap = snap, err = cerr }, 'device_model_changed') + end) + assert_true(ok, err) + fibers.perform(sleep.sleep_op(0.001)) + m:apply_catalogue(1, catalogue_one()) + local got = fibers.perform(rx:recv_op()) + assert_not_nil(got) + assert_eq(got.version, seen + 1) + assert_nil(got.err) + end) +end + +function tests.test_terminate_wakes_changed_op_with_reason() + fibers.run(function (scope) + local m = model_mod.new() + local seen = m:version() + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + local ok, err = scope:spawn(function () + local version, snap, cerr = fibers.perform(m:changed_op(seen)) + queue.assert_admit_required(tx, { version = version, snap = snap, err = cerr }, 'device_model_terminated') + end) + assert_true(ok, err) + fibers.perform(sleep.sleep_op(0.001)) + assert_true(m:terminate('test_done')) + local got = fibers.perform(rx:recv_op()) + assert_not_nil(got) + assert_nil(got.version) + assert_eq(got.err, 'test_done') + assert_true(m:is_terminated()) + assert_true(m:terminate('ignored')) + assert_eq(m:why(), 'test_done') + end) +end + +return tests diff --git a/tests/unit/device/test_phase2.lua b/tests/unit/device/test_phase2.lua new file mode 100644 index 00000000..00f8c1d9 --- /dev/null +++ b/tests/unit/device/test_phase2.lua @@ -0,0 +1,1319 @@ +-- tests/unit/device/test_phase2.lua + +local fibers = require 'fibers' +local op = require 'fibers.op' +local cond = require 'fibers.cond' +local mailbox = require 'fibers.mailbox' +local sleep = require 'fibers.sleep' + +local config = require 'services.device.config' +local catalogue = require 'services.device.catalogue' +local model_mod = require 'services.device.model' +local service = require 'services.device.service' +local action_worker = require 'services.device.action_worker' +local observer = require 'services.device.observer' +local topics = require 'services.device.topics' +local projection = require 'services.device.projection' +local component_mcu = require 'services.device.component_mcu' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_false(v, msg) if v ~= false then fail(msg or ('expected false, got ' .. tostring(v))) end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end + +local function wait_for_signal(c, msg) + local which = fibers.perform(fibers.named_choice{ + ready = c:wait_op(), + timeout = sleep.sleep_op(1), + }) + assert_eq(which, 'ready', msg or 'timed out waiting for test barrier') +end + +local function req(payload) + local r = { payload = payload, replied = nil, failed = nil } + function r:reply(v) self.replied = v; return true end + function r:fail(e) self.failed = e; return true end + return r +end + + +local function take_stage_source(params, expected) + assert_not_nil(params and params.source_owner, 'source_owner required') + local source = params.source_owner:value() + if expected ~= nil then assert_eq(source, expected) end + local detached, err = params.source_owner:detach() + assert_eq(detached, source, tostring(err)) + return source +end + +local function sample_config(display_name) + return { + schema = config.SCHEMA, + components = { + mcu = { + subtype = 'mcu', + display = { name = display_name or 'MCU' }, + facts = { software = topics.raw_member_state('mcu', 'software') }, + actions = { restart = { kind = 'rpc', call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') } }, + }, + }, + } +end + + +function tests.test_default_fabric_client_disables_bus_timeout_but_passes_transfer_budget() + local seen_topic, seen_payload, seen_opts + + fibers.run(function () + local conn = { + call_op = function (_, topic, payload, opts) + seen_topic = topic + seen_payload = payload + seen_opts = opts + return op.always({ ok = true, result = { accepted = true } }) + end, + } + local client = assert(service.default_fabric_client(conn)) + local result, err = fibers.perform(client:send_blob_op({ + request_id = 'r-fabric-stage', + target = 'updater/main', + chunk_size = 2048, + timeout = 300, + }, {})) + assert_nil(err) + assert_true(result.accepted) + end) + + assert_eq(table.concat(seen_topic, '/'), 'cap/transfer-manager/main/rpc/send-blob') + assert_eq(seen_payload.timeout_s, 300) + assert_eq(seen_payload.target, 'updater/main') + assert_eq(seen_opts.timeout, false, 'Device must not hide a lua-bus timeout inside Fabric staging') +end + +function tests.test_default_catalogue_includes_host_and_mcu_components() + local cat = catalogue.build(nil) + assert_not_nil(cat.components.cm5) + assert_not_nil(cat.components.mcu) + assert_not_nil(cat.components.mcu.facts.software) + assert_not_nil(cat.components.mcu.events.charger_alert) + assert_not_nil(cat.components.mcu.actions.restart) +end + +function tests.test_public_metadata_config_change_does_not_restart_generation_but_updates_model() + fibers.run(function (scope) + local state = service.build_state(scope, { + conn = nil, + enable_actions = false, + enable_observers = false, + }) + + local ok, err = service.apply_config_payload(state, sample_config('A')) + assert_true(ok, err) + local generation = state.active.generation + assert_eq(state.model:snapshot().components.mcu.display.name, 'A') + + ok, err = service.apply_config_payload(state, sample_config('B')) + assert_true(ok, err) + assert_eq(state.active.generation, generation) + assert_eq(state.model:snapshot().components.mcu.display.name, 'B') + assert_true(state.dirty.components.mcu) + end) +end + +function tests.test_next_event_prioritises_config_before_ready_action_request() + fibers.run(function (scope) + local config_tx, config_rx = mailbox.new(2, { full = 'reject_newest' }) + local action_tx, action_rx = mailbox.new(2, { full = 'reject_newest' }) + local state = service.build_state(scope, { + conn = nil, + config_rx = config_rx, + enable_actions = false, + enable_observers = false, + }) + + state.active = { + generation = 7, + action_eps = { + ['mcu:restart'] = { + key = 'mcu:restart', + generation = 7, + component = 'mcu', + action = 'restart', + ep = { recv_op = function () return action_rx:recv_op() end }, + }, + }, + } + + fibers.perform(action_tx:send_op(req())) + fibers.perform(config_tx:send_op({ kind = 'config_changed', payload = sample_config('prio') })) + + local ev = fibers.perform(service.next_event_op(state)) + assert_eq(ev.kind, 'config_changed') + assert_eq(ev.payload.components.mcu.display.name, 'prio') + end) +end + + +function tests.test_fabric_stage_source_owner_transfer_does_not_terminate_source_in_action_scope() + local terminate_count = 0 + fibers.run(function (scope) + local source = { id = 'src' } + local r = req({ source = source }) + local client = { + send_blob_op = function (_, params) + take_stage_source(params, source) + assert_eq(params.component, 'mcu') + assert_eq(params.target, 'updater/main') + return op.always({ ok = true, staged = true }) + end, + } + + local result = action_worker.run(scope, { + request = r, + component_id = 'mcu', + action = 'stage-update', + request_id = 'r1', + action_spec = { kind = 'fabric_stage', target = 'updater/main' }, + fabric_client = client, + terminate_source = function () terminate_count = terminate_count + 1; return true end, + }) + + assert_true(result.ok) + assert_not_nil(r.replied) + end) + assert_eq(terminate_count, 0) +end + + +function tests.test_fabric_stage_requires_client_to_take_source_ownership() + local terminate_count = 0 + fibers.run(function (scope) + local source = { id = 'src-no-proof' } + local r = req({ source = source }) + local client = { + send_blob_op = function () + return op.always({ ok = true, staged = true }) + end, + } + + local result = action_worker.run(scope, { + request = r, + component_id = 'mcu', + action = 'stage-update', + request_id = 'r-no-proof', + action_spec = { kind = 'fabric_stage', target = 'updater/main' }, + fabric_client = client, + terminate_source = function (v) + assert_eq(v, source) + terminate_count = terminate_count + 1 + return true + end, + }) + + assert_eq(result.ok, false) + assert_eq(r.failed, 'fabric_stage client did not take source ownership') + end) + assert_eq(terminate_count, 1) +end + + +function tests.test_fabric_stage_failed_admission_terminates_untransferred_source() + local terminate_count = 0 + fibers.run(function (scope) + local source = { id = 'src' } + local r = req({ source = source }) + local client = { + send_blob_op = function () return op.always(nil, 'no_route') end, + } + + local result = action_worker.run(scope, { + request = r, + component_id = 'mcu', + action = 'stage-update', + request_id = 'r2', + action_spec = { kind = 'fabric_stage', target = 'updater/main' }, + fabric_client = client, + terminate_source = function (v, reason) + assert_eq(v, source) + assert_eq(reason, 'ok') + terminate_count = terminate_count + 1 + return true + end, + }) + + assert_eq(result.ok, false) + assert_eq(r.failed, 'no_route') + end) + assert_eq(terminate_count, 1) +end + +function tests.test_repeated_observation_does_not_change_model_version() + fibers.run(function () + local cat = assert(config.to_catalogue(sample_config('A'))) + local m = model_mod.new() + m:apply_catalogue(1, cat) + local changed = m:apply_observation(1, { + component = 'mcu', tag = 'fact_retained', fact = 'software', payload = { version = '1.0' }, + }) + assert_true(changed) + local seen = m:version() + changed = m:apply_observation(1, { + component = 'mcu', tag = 'fact_retained', fact = 'software', payload = { version = '1.0' }, + }) + assert_eq(changed, false) + assert_eq(m:version(), seen) + end) +end + +local function fake_bound_conn() + local c = { bound = {}, events = {} } + function c:bind(topic, opts) + local k = table.concat(topic, '/') + if self.bound[k] then + error('already bound: ' .. k) + end + local ep = { topic = topic, key = k } + self.bound[k] = ep + self.events[#self.events + 1] = 'bind:' .. k + function ep:recv_op() + return mailbox.new(0, { full = 'reject_newest' }) + end + return ep + end + function c:unbind(ep) + local k = ep and ep.key or (ep and ep.topic and table.concat(ep.topic, '/') or tostring(ep)) + self.events[#self.events + 1] = 'unbind:' .. k + self.bound[k] = nil + return true + end + function c:retain() return true end + function c:unretain() return true end + function c:publish() return true end + return c +end + +function tests.test_generation_replacement_unbinds_old_endpoints_before_rebinding() + fibers.run(function (scope) + local conn = fake_bound_conn() + local state = service.build_state(scope, { + conn = conn, + enable_actions = true, + enable_observers = false, + }) + + assert_true(service.apply_config_payload(state, sample_config('A'))) + assert_not_nil(next(conn.bound)) + + local cfg2 = sample_config('B') + cfg2.components.host = { + class = 'host', + subtype = 'cm5', + facts = { software = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'software') }, + } + + local ok, err = service.apply_config_payload(state, cfg2) + assert_true(ok, err) + assert_eq(state.active.generation, 2) + -- The same mcu public endpoints are now rebound for generation 2, which + -- would have failed if generation 1 endpoints were left installed. + assert_not_nil(conn.bound['cap/component/mcu/rpc/restart']) + end) +end + +function tests.test_action_timeout_cancels_action_scope_and_finalises_request() + fibers.run(function () + local r = req() + local conn = { + call_op = function () + return sleep.sleep_op(60):wrap(function () + return { ok = true } + end) + end, + } + + local st, _rep, primary = fibers.run_scope(function (scope) + action_worker.run(scope, { + conn = conn, + request = r, + component_id = 'mcu', + action = 'restart', + request_id = 'timeout-1', + timeout = 0.02, + action_spec = { call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') }, + }) + end) + + assert_eq(st, 'cancelled') + assert_eq(primary, 'timeout') + assert_eq(r.failed, 'timeout') + assert_nil(r.replied) + end) +end + +function tests.test_remote_action_failure_is_public_result_not_worker_failure() + fibers.run(function (scope) + local r = req() + local conn = { + call_op = function () + return op.always({ ok = false, reason = 'remote_no' }) + end, + } + + local result = action_worker.run(scope, { + conn = conn, + request = r, + component_id = 'mcu', + action = 'restart', + request_id = 'r-public-fail', + timeout = 1, + action_spec = { call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') }, + }) + + assert_eq(result.ok, false) + assert_eq(result.public_status, 'remote_failed') + assert_eq(result.err, 'remote_no') + assert_eq(r.failed, 'remote_no') + end) +end + + +function tests.test_action_worker_requires_call_op_and_does_not_use_call_fallback() + fibers.run(function (scope) + local r = req() + local called = false + local conn = { + call = function () + called = true + return { ok = true } + end, + } + + local result = action_worker.run(scope, { + conn = conn, + request = r, + component_id = 'mcu', + action = 'restart', + request_id = 'no-call-op', + timeout = 1, + action_spec = { call_topic = topics.raw_member_cap_rpc('mcu', 'control', 'main', 'restart') }, + }) + + assert_eq(called, false) + assert_eq(result.ok, false) + assert_eq(result.public_status, 'unavailable') + assert_eq(result.err, 'connection does not support call_op') + assert_eq(r.failed, 'connection does not support call_op') + end) +end + +function tests.test_failed_generation_start_leaves_no_active_generation_and_rolls_back_endpoints() + fibers.run(function (scope) + local conn = fake_bound_conn() + local bind_count = 0 + local parent_bind = conn.bind + function conn:bind(topic, opts) + bind_count = bind_count + 1 + if bind_count == 2 then + error('synthetic bind failure') + end + return parent_bind(self, topic, opts) + end + + local state = service.build_state(scope, { + conn = conn, + enable_actions = true, + enable_observers = false, + }) + + local ok, err = service.apply_config_payload(state, sample_config('fails')) + assert_nil(ok) + assert_not_nil(err) + assert_nil(state.active) + assert_nil(next(conn.bound)) + end) +end + + +function tests.test_generation_replacement_cancels_old_generation_without_generation_shell_completion() + fibers.run(function (scope) + local state = service.build_state(scope, { + conn = nil, + enable_actions = false, + enable_observers = false, + }) + + assert_true(service.apply_config_payload(state, sample_config('A'))) + assert_eq(state.active.generation, 1) + + local cfg2 = sample_config('B') + cfg2.components.host = { + class = 'host', + subtype = 'cm5', + facts = { software = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'software') }, + } + + assert_true(service.apply_config_payload(state, cfg2)) + assert_eq(state.active.generation, 2) + + local ev = fibers.perform(state.done_rx:recv_op():or_else(function () + return nil + end)) + assert_nil(ev, 'generation lifetime should not report a synthetic generation_done event') + assert_eq(state.active.generation, 2) + end) +end + + +function tests.test_service_finaliser_cancels_generation_before_closing_reporting_queues() + local order = {} + + fibers.run(function () + local st, _rep, primary = fibers.run_scope(function (scope) + local state = service.build_state(scope, { + conn = nil, + enable_actions = false, + enable_observers = false, + }) + + local close_done = state.done_tx.close + function state.done_tx:close(reason) + order[#order + 1] = 'done_close:' .. tostring(reason) + return close_done(self, reason) + end + + local close_obs = state.observation_tx.close + function state.observation_tx:close(reason) + order[#order + 1] = 'observation_close:' .. tostring(reason) + return close_obs(self, reason) + end + + state.active = { + generation = 77, + action_eps = {}, + cancel = function (reason) + order[#order + 1] = 'generation_cancel:' .. tostring(reason) + return true, nil + end, + } + end) + + assert_eq(st, 'ok') + assert_nil(primary) + end) + + assert_eq(order[1], 'generation_cancel:ok') + assert_eq(order[2], 'done_close:ok') + assert_eq(order[3], 'observation_close:ok') +end + +function tests.test_observer_finaliser_owns_opened_handles_before_source_down_emit_can_fail() + local terminate_count = 0 + + fibers.run(function () + local tx = mailbox.new(0, { full = 'reject_newest' }) + local conn = { opened = 0 } + + function conn:watch_retained(_topic, _opts) + self.opened = self.opened + 1 + if self.opened == 1 then + return { id = 'watch-1' } + end + return nil, 'synthetic_watch_failure' + end + + function conn:unwatch_retained(watch) + assert_eq(watch.id, 'watch-1') + terminate_count = terminate_count + 1 + return true + end + + local st, _rep, primary = fibers.run_scope(function (scope) + observer.run(scope, { + conn = conn, + tx = tx, + generation = 1, + component_id = 'mcu', + component = { + facts = { + alpha = { watch_topic = { 'raw', 'alpha' } }, + beta = { watch_topic = { 'raw', 'beta' } }, + }, + }, + }) + end) + + assert_eq(st, 'failed') + assert_not_nil(primary) + end) + + assert_eq(terminate_count, 1) +end + +function tests.test_observer_done_records_outcome_and_forgets_live_handle() + fibers.run(function (scope) + local state = service.build_state(scope, { + conn = nil, + enable_actions = false, + enable_observers = false, + }) + + state.active = { + generation = 3, + observers = { mcu = { cancel = function () end } }, + observer_outcomes = {}, + } + + local ev = { + kind = 'observer_done', + generation = 3, + component = 'mcu', + status = 'ok', + result = { reason = 'done' }, + } + + local ok, err = service.reduce_event(state, ev) + assert_true(ok, err) + assert_nil(state.active.observers.mcu) + assert_eq(state.active.observer_outcomes.mcu, ev) + end) +end + +function tests.test_generation_cancellation_cascades_to_in_flight_action_and_finalises_request() + fibers.run(function (scope) + local request_done = cond.new() + local call_entered = cond.new() + local r = req() + function r:reply(v) + self.replied = v + request_done:signal() + return true + end + function r:fail(e) + self.failed = e + request_done:signal() + return true + end + + local conn = { + call_op = function () + call_entered:signal() + return sleep.sleep_op(60):wrap(function () return { ok = true } end) + end, + } + + local state = service.build_state(scope, { + conn = conn, + enable_actions = false, + enable_observers = false, + action_timeout = 60, + }) + + assert_true(service.apply_config_payload(state, sample_config('A'))) + local active = state.active + local ok, err = service.reduce_event(state, { + kind = 'component_action_request', + generation = active.generation, + component = 'mcu', + action = 'restart', + request = r, + }) + assert_true(ok, err) + assert_not_nil(next(state.pending_actions)) + wait_for_signal(call_entered, 'action worker did not enter raw RPC call') + + ok, err = service.cancel_active_generation(state, 'catalogue_changed') + assert_true(ok, err) + wait_for_signal(request_done, 'action request was not finalised after generation cancellation') + + assert_eq(r.failed, 'catalogue_changed') + assert_nil(r.replied) + end) +end + + +function tests.test_fabric_stage_open_source_may_be_an_explicit_op() + fibers.run(function (scope) + local source = { id = 'src-op' } + local r = req({}) + local client = { + send_blob_op = function (_, params) + take_stage_source(params, source) + return op.always({ ok = true, staged = true }) + end, + } + + local result = action_worker.run(scope, { + request = r, + component_id = 'mcu', + action = 'stage-update', + request_id = 'r-source-op', + action_spec = { kind = 'fabric_stage', target = 'updater/main' }, + fabric_client = client, + open_source_op = function () + return sleep.sleep_op(0.001):wrap(function () + return source, nil + end) + end, + terminate_source = function () + fail('source should have been handed off, not terminated') + end, + }) + + assert_true(result.ok) + assert_not_nil(r.replied) + end) +end + + +function tests.test_fabric_stage_timeout_cancels_child_stage_and_terminates_unhanded_source() + local terminate_count = 0 + + fibers.run(function () + local source = { id = 'slow-source' } + local r = req({ source = source }) + local client = { + send_blob_op = function () + return sleep.sleep_op(60):wrap(function () + return { ok = true } + end) + end, + } + + local st, _rep, primary = fibers.run_scope(function (scope) + action_worker.run(scope, { + request = r, + component_id = 'mcu', + action = 'stage-update', + request_id = 'fabric-timeout', + timeout = 0.02, + action_spec = { kind = 'fabric_stage', target = 'updater/main' }, + fabric_client = client, + terminate_source = function (v, reason) + assert_eq(v, source) + assert_eq(reason, 'timeout') + terminate_count = terminate_count + 1 + return true + end, + }) + end) + + assert_eq(st, 'cancelled') + assert_eq(primary, 'timeout') + assert_eq(r.failed, 'timeout') + end) + + assert_eq(terminate_count, 1) +end + +function tests.test_completion_queue_admission_failure_fails_observing_scope() + fibers.run(function () + local st, _rep, primary = fibers.run_scope(function (scope) + local r = req() + local conn = { + call_op = function () + return op.always({ ok = true, value = 'ok' }) + end, + } + + local state = service.build_state(scope, { + conn = conn, + done_queue_len = 0, + enable_actions = false, + enable_observers = false, + action_timeout = 60, + }) + + assert_true(service.apply_config_payload(state, sample_config('A'))) + local ok, err = service.reduce_event(state, { + kind = 'component_action_request', + generation = state.active.generation, + component = 'mcu', + action = 'restart', + request = r, + }) + assert_true(ok, err) + + -- The action result is produced immediately, but the completion cannot + -- be admitted to a zero-capacity done queue without a receiver. The + -- reporter must fail the observing scope rather than silently dropping it. + end) + + assert_eq(st, 'failed') + assert_not_nil(primary) + assert_true(tostring(primary):find('device_action_done_report_failed', 1, true) ~= nil, + 'expected completion report failure, got ' .. tostring(primary)) + end) +end + + +function tests.test_generation_cancellation_cascades_into_fabric_stage_child_scope() + local terminate_count = 0 + + fibers.run(function (scope) + local source = { id = 'gen-cancel-source' } + local entered_stage = cond.new() + local request_done = cond.new() + local r = req({ source = source }) + function r:reply(v) + self.replied = v + request_done:signal() + return true + end + function r:fail(e) + self.failed = e + request_done:signal() + return true + end + + local client = { + send_blob_op = function () + -- The Fabric client is called only after the stage scope has acquired + -- the source and installed its stage-scope cleanup finaliser. + entered_stage:signal() + return sleep.sleep_op(60):wrap(function () + return { ok = true } + end) + end, + } + + local cfg = sample_config('A') + cfg.components.mcu.actions['stage-update'] = { + kind = 'fabric_stage', + target = 'updater/main', + } + + local state = service.build_state(scope, { + conn = nil, + enable_actions = false, + enable_observers = false, + action_timeout = 60, + fabric_client = client, + terminate_source = function (v, reason) + assert_eq(v, source) + assert_eq(reason, 'catalogue_changed') + terminate_count = terminate_count + 1 + return true + end, + }) + + assert_true(service.apply_config_payload(state, cfg)) + local generation = state.active.generation + local ok, err = service.reduce_event(state, { + kind = 'component_action_request', + generation = generation, + component = 'mcu', + action = 'stage-update', + request = r, + }) + assert_true(ok, err) + wait_for_signal(entered_stage, 'fabric-stage child did not acquire the source') + + ok, err = service.cancel_active_generation(state, 'catalogue_changed') + assert_true(ok, err) + wait_for_signal(request_done, 'fabric-stage action request was not finalised after generation cancellation') + + assert_eq(r.failed, 'catalogue_changed') + assert_nil(r.replied) + end) + + assert_eq(terminate_count, 1) +end + +local function fake_live_conn() + local c = { + retained = {}, published = {}, endpoints = {}, watches = {}, subs = {}, + unwatched = 0, unsubscribed = 0, unbound = 0, + watch_opened = cond.new(), watch_closed = cond.new(), endpoint_bound = cond.new(), + } + + local function make_feed(topic) + local tx, rx = mailbox.new(8, { full = 'drop_oldest' }) + local f = { topic = topic, tx = tx, rx = rx, closed = false } + function f:recv_op() + return self.rx:recv_op() + end + function f:close(reason) + if not self.closed then + self.closed = true + self.tx:close(reason or 'closed') + end + return true + end + return f + end + + function c:watch_retained(topic, _opts) + local f = make_feed(topic) + self.watches[#self.watches + 1] = f + self.watch_opened:signal() + return f + end + + function c:unwatch_retained(watch) + self.unwatched = self.unwatched + 1 + if watch and watch.close then watch:close('unwatched') end + self.watch_closed:signal() + return true + end + + function c:subscribe(topic, _opts) + local f = make_feed(topic) + self.subs[#self.subs + 1] = f + return f + end + + function c:unsubscribe(sub) + self.unsubscribed = self.unsubscribed + 1 + if sub and sub.close then sub:close('unsubscribed') end + return true + end + + function c:bind(topic, _opts) + local k = table.concat(topic, '/') + if self.endpoints[k] then error('already bound: ' .. k) end + local tx, rx = mailbox.new(8, { full = 'reject_newest' }) + local ep = { topic = topic, key = k, tx = tx, rx = rx, closed = false } + function ep:recv_op() + return self.rx:recv_op() + end + function ep:close(reason) + if not self.closed then + self.closed = true + self.tx:close(reason or 'closed') + end + return true + end + self.endpoints[k] = ep + self.endpoint_bound:signal() + return ep + end + + function c:unbind(ep) + if ep then + self.endpoints[ep.key] = nil + if ep.close then ep:close('unbound') end + end + self.unbound = self.unbound + 1 + return true + end + + function c:retain(topic, payload) self.retained[table.concat(topic, '/')] = payload; return true end + function c:unretain(topic) self.retained[table.concat(topic, '/')] = nil; return true end + function c:publish(topic, payload) self.published[#self.published + 1] = { topic = topic, payload = payload }; return true end + return c +end + +function tests.test_config_replacement_with_live_observer_terminates_raw_watch() + fibers.run(function (scope) + local conn = fake_live_conn() + local state = service.build_state(scope, { + conn = conn, + enable_actions = false, + enable_observers = true, + }) + + assert_true(service.apply_config_payload(state, sample_config('A'))) + local first = state.active + wait_for_signal(conn.watch_opened, 'observer did not open a real-ish retained watch') + assert_not_nil(conn.watches[1]) + + local cfg2 = sample_config('B') + cfg2.components.host = { + class = 'host', + subtype = 'cm5', + facts = { software = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'software') }, + } + assert_true(service.apply_config_payload(state, cfg2)) + assert_eq(first.scope:status(), 'cancelled') + wait_for_signal(conn.watch_closed, 'old-generation observer did not release its retained watch') + assert_true(conn.unwatched >= 1) + + -- The replacement generation also owns a live observer. End the test by + -- cancelling it explicitly; otherwise fibers.run() will correctly wait for + -- that still-live child scope during structural join. + assert_true(service.cancel_active_generation(state, 'test_done')) + end) +end + + +function tests.test_config_replacement_with_live_fabric_stage_endpoint_cancels_action_and_terminates_source() + local terminate_count = 0 + fibers.run(function (scope) + local conn = fake_live_conn() + local entered_stage = cond.new() + local request_done = cond.new() + local source = { id = 'live-endpoint-source' } + local r = req({ source = source }) + function r:reply(v) + self.replied = v + request_done:signal() + return true + end + function r:fail(e) + self.failed = e + request_done:signal() + return true + end + + local client = { + send_blob_op = function () + entered_stage:signal() + return sleep.sleep_op(60):wrap(function () + return { ok = true } + end) + end, + } + + local cfg = sample_config('A') + cfg.components.mcu.actions['stage-update'] = { + kind = 'fabric_stage', + target = 'updater/main', + } + + local state = service.build_state(scope, { + conn = conn, + enable_actions = true, + enable_observers = false, + auto_publish = false, + action_timeout = 60, + fabric_client = client, + terminate_source = function (v, reason) + assert_eq(v, source) + assert_eq(reason, 'catalogue_changed') + terminate_count = terminate_count + 1 + return true + end, + }) + + assert_true(service.apply_config_payload(state, cfg)) + local ep = conn.endpoints['cap/component/mcu/rpc/stage-update'] + assert_not_nil(ep, 'stage-update endpoint was not bound') + assert_true(fibers.perform(ep.tx:send_op(r))) + + local ev = fibers.perform(service.next_event_op(state)) + assert_eq(ev.kind, 'component_action_request') + assert_true(service.reduce_event(state, ev)) + wait_for_signal(entered_stage, 'fabric-stage action did not start from endpoint request') + + local cfg2 = sample_config('B') + cfg2.components.host = { + class = 'host', + subtype = 'cm5', + facts = { software = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'software') }, + } + assert_true(service.apply_config_payload(state, cfg2)) + wait_for_signal(request_done, 'live endpoint fabric-stage request was not finalised') + assert_eq(r.failed, 'catalogue_changed') + assert_nil(r.replied) + end) + assert_eq(terminate_count, 1) +end + +function tests.test_action_start_cancelled_before_worker_body_resolves_request_without_calling_hal() + fibers.run(function (scope) + local called = false + local r = req() + local conn = { + call_op = function () + called = true + return op.always({ ok = true }) + end, + } + local state = service.build_state(scope, { + conn = conn, + enable_actions = false, + enable_observers = false, + action_timeout = 60, + }) + assert_true(service.apply_config_payload(state, sample_config('A'))) + local generation = state.active.generation + assert_true(service.reduce_event(state, { + kind = 'component_action_request', + generation = generation, + component = 'mcu', + action = 'restart', + request = r, + })) + + -- No yield has happened since admission, so the worker body has not had an + -- opportunity to call the HAL. The setup-installed request owner still lets + -- generation cancellation resolve the request immediately. + assert_true(service.cancel_active_generation(state, 'catalogue_changed')) + assert_eq(called, false) + assert_eq(r.failed, 'catalogue_changed') + assert_nil(r.replied) + end) +end + + +function tests.test_fabric_stage_public_reply_is_sanitised_of_ownership_internals() + fibers.run(function (scope) + local source = { id = 'sanitize-source' } + local r = req({ source = source }) + local client = { + send_blob_op = function (_, params) + take_stage_source(params, source) + return op.always({ ok = true, transfer_id = 'tx1', source_owner = params.source_owner }) + end, + } + + local result = action_worker.run(scope, { + request = r, + component_id = 'mcu', + action = 'stage-update', + request_id = 'sanitize', + action_spec = { kind = 'fabric_stage', target = 'updater/main' }, + fabric_client = client, + terminate_source = function () fail('source should be handed off') end, + }) + + assert_true(result.ok) + assert_eq(r.replied.transfer_id, 'tx1') + assert_nil(r.replied.source_owner) + assert_nil(result.reply_payload.source_owner) + end) +end + +function tests.test_backpressure_policy_is_explicit() + local backpressure = require 'services.device.backpressure' + local p = backpressure.snapshot() + assert_eq(p.completions.queue_full, 'fail_observing_scope') + assert_eq(p.observations.queue_full, 'fail_observing_scope') + assert_eq(p.action_endpoints.queue_full, 'reject_request') + assert_eq(p.publication.policy, 'coalesce_dirty_state') + assert_eq(p.publication.failure, 'fail_service') + assert_eq(p.availability.source_down, 'mark_degraded_or_unavailable') +end + +function tests.test_catalogue_entries_carry_component_modules() + local cat = catalogue.build(nil) + assert_eq(cat.components.mcu.module.kind, 'mcu') + assert_eq(cat.components.cm5.module.kind, 'host') +end + +function tests.test_mcu_fault_availability_is_stored_and_projected() + fibers.run(function () + local cat = assert(config.to_catalogue({ + schema = config.SCHEMA, + components = { + mcu = { + subtype = 'mcu', + required_facts = { 'software', 'updater' }, + facts = { + software = topics.raw_member_state('mcu', 'software'), + updater = topics.raw_member_state('mcu', 'updater'), + health = topics.raw_member_state('mcu', 'health'), + }, + }, + }, + })) + local m = model_mod.new() + m:apply_catalogue(1, cat) + m:apply_observation(1, { component = 'mcu', tag = 'fact_retained', fact = 'software', payload = { version = '1.0' } }) + m:apply_observation(1, { component = 'mcu', tag = 'fact_retained', fact = 'updater', payload = { state = 'idle' } }) + m:apply_observation(1, { component = 'mcu', tag = 'fact_retained', fact = 'health', payload = { state = 'failed' } }) + local rec = m:snapshot().components.mcu + assert_eq(rec.status.availability, 'unavailable') + assert_eq(rec.status.health, 'fault') + local payloads = projection.component_payloads('mcu', rec, 10) + assert_eq(payloads.component.availability, 'unavailable') + assert_eq(payloads.cap_status.state, 'unavailable') + end) +end + +function tests.test_host_missing_software_degrades_rather_than_disappears() + fibers.run(function () + local cat = assert(config.to_catalogue({ + schema = config.SCHEMA, + components = { + host = { + class = 'host', + subtype = 'cm5', + required_facts = { 'software' }, + facts = { + software = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'software'), + updater = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'updater'), + }, + }, + }, + })) + local m = model_mod.new() + m:apply_catalogue(1, cat) + m:apply_observation(1, { component = 'host', tag = 'fact_retained', fact = 'updater', payload = { state = 'idle' } }) + local status = m:snapshot().components.host.status + assert_eq(status.availability, 'degraded') + assert_eq(status.reason, 'missing_host_software') + end) +end + +function tests.test_component_module_normalises_mcu_software_fact() + local v, err = component_mcu.normalise_fact('software', { version = '2.3', build_id = 'abc' }) + assert_nil(err) + assert_eq(v.version, '2.3') + assert_eq(v.build_id, 'abc') +end + +function tests.test_component_module_ignores_non_contract_mcu_fact_aliases() + local software = assert(component_mcu.normalise_fact('software', { fw_version = 'legacy', build = 'legacy-build' })) + assert_nil(software.version) + assert_nil(software.build_id) + + local updater = assert(component_mcu.normalise_fact('updater', { status = 'available', err = 'boom' })) + assert_nil(updater.state) + assert_nil(updater.last_error) +end + + +function tests.test_stale_action_completion_is_archived_and_clears_pending_without_mutating_model() + fibers.run(function (scope) + local state = service.build_state(scope, { + conn = nil, + enable_actions = false, + enable_observers = false, + }) + assert_true(service.apply_config_payload(state, sample_config('A'))) + local old_generation = state.active.generation + state.pending_actions['old-action'] = { + generation = old_generation, + component = 'mcu', + action = 'restart', + } + + local cfg2 = sample_config('B') + cfg2.components.host = { + class = 'host', + subtype = 'cm5', + facts = { software = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'software') }, + } + assert_true(service.apply_config_payload(state, cfg2)) + local current_generation = state.active.generation + assert_true(current_generation ~= old_generation) + + local ok, err = service.reduce_event(state, { + kind = 'component_action_done', + generation = old_generation, + component = 'mcu', + action = 'restart', + request_id = 'old-action', + status = 'ok', + result = { public_status = 'succeeded', ok = true }, + }) + assert_true(ok, err) + assert_nil(state.pending_actions['old-action']) + assert_not_nil(state.action_outcomes['old-action']) + assert_not_nil(state.stale_action_outcomes['old-action']) + assert_eq(state.action_outcomes['old-action'].current, false) + assert_nil(state.model:snapshot().components.mcu.last_action, + 'stale action completion must not mutate current-generation model state') + end) +end + +function tests.test_stale_observer_completion_is_kept_on_generation_history() + fibers.run(function (scope) + local state = service.build_state(scope, { + conn = nil, + enable_actions = false, + enable_observers = false, + }) + assert_true(service.apply_config_payload(state, sample_config('A'))) + local first = state.active + first.observers.mcu = { cancel = function () return true end } + + local cfg2 = sample_config('B') + cfg2.components.host = { + class = 'host', + subtype = 'cm5', + facts = { software = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'software') }, + } + assert_true(service.apply_config_payload(state, cfg2)) + + local ev = { + kind = 'observer_done', + generation = first.generation, + component = 'mcu', + status = 'cancelled', + primary = 'catalogue_changed', + } + local ok, err = service.reduce_event(state, ev) + assert_true(ok, err) + assert_nil(first.observers.mcu) + assert_eq(first.observer_outcomes.mcu, ev) + assert_eq(state.generation_history[first.generation].observer_outcomes.mcu, ev) + end) +end + +function tests.test_generation_cancel_marks_pending_actions_until_completion_arrives() + fibers.run(function (scope) + local state = service.build_state(scope, { + conn = nil, + enable_actions = false, + enable_observers = false, + }) + assert_true(service.apply_config_payload(state, sample_config('A'))) + local gen = state.active.generation + local cancelled = false + state.pending_actions['pending'] = { + generation = gen, + component = 'mcu', + action = 'restart', + handle = { + cancel = function (_, reason) + cancelled = reason + return true, nil + end, + }, + } + + assert_true(service.cancel_active_generation(state, 'catalogue_changed')) + assert_eq(cancelled, 'catalogue_changed') + assert_not_nil(state.pending_actions['pending']) + assert_eq(state.pending_actions['pending'].cancelled, true) + assert_eq(state.pending_actions['pending'].cancel_reason, 'catalogue_changed') + + assert_true(service.reduce_event(state, { + kind = 'component_action_done', + generation = gen, + component = 'mcu', + action = 'restart', + request_id = 'pending', + status = 'cancelled', + primary = 'catalogue_changed', + })) + assert_nil(state.pending_actions['pending']) + assert_not_nil(state.stale_action_outcomes['pending']) + end) +end + + +function tests.test_mcu_charger_bitfields_are_expanded_in_device_projection() + local raw = { + vin_mV = 24153, + vsys_mV = 24100, + iin_mA = 623, + state_bits = 0x0240, -- absorb + cccv + status_bits = 0x0005, -- iin_limit + const_voltage + system_bits = 0x2045, -- enabled + ok_to_charge + vin_gt_vbat + intvcc_gt_2p8v + seq = 7, + uptime_ms = 1234, + } + local fact, err = component_mcu.normalise_fact('power_charger', raw) + assert_nil(err) + assert_eq(fact.vin_mV, 24153) + assert_eq(fact.state_bits, raw.state_bits) + assert_true(fact.state.absorb_charge) + assert_true(fact.state.cccv_charge) + assert_false(fact.state.bat_missing_fault) + assert_true(fact.status.iin_limit_active) + assert_true(fact.status.const_voltage) + assert_false(fact.status.const_current) + assert_true(fact.system.charger_enabled) + assert_true(fact.system.ok_to_charge) + assert_true(fact.system.vin_gt_vbat) + assert_true(fact.system.intvcc_gt_2p8v) +end + +return tests diff --git a/tests/unit/device/test_publisher.lua b/tests/unit/device/test_publisher.lua new file mode 100644 index 00000000..fd21f3da --- /dev/null +++ b/tests/unit/device/test_publisher.lua @@ -0,0 +1,47 @@ +-- tests/unit/device/test_publisher.lua + +local config = require 'services.device.config' +local model_mod = require 'services.device.model' +local publisher = require 'services.device.publisher' +local topics = require 'services.device.topics' + +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end + +local function key(topic) return table.concat(topic, '/') end + +local function fake_conn() + local c = { retained = {}, published = {} } + function c:retain(topic, payload) self.retained[key(topic)] = payload; return true end + function c:unretain(topic) self.retained[key(topic)] = nil; return true end + function c:publish(topic, payload) self.published[#self.published + 1] = { topic = topic, payload = payload }; return true end + return c +end + +function tests.test_publish_component_and_summary_are_immediate_effects() + local cat = assert(config.to_catalogue({ + schema = config.SCHEMA, + components = { + mcu = { subtype = 'mcu', facts = { software = topics.raw_member_state('mcu', 'software') } }, + }, + })) + local m = model_mod.new() + m:apply_catalogue(1, cat) + m:apply_observation(1, { component = 'mcu', tag = 'fact_retained', fact = 'software', payload = { version = '1.0' } }) + + local conn = fake_conn() + local snap = m:snapshot() + local ok, err = publisher.publish_component_now(conn, snap, 'mcu', { now = function () return 123 end }) + assert_true(ok, err) + assert_eq(conn.retained['state/device/component/mcu'].component, 'mcu') + assert_eq(conn.retained['state/device/component/mcu/software'].version, '1.0') + assert_eq(conn.retained['cap/component/mcu/status'].state, 'available') + + ok, err = publisher.publish_summary_now(conn, snap, { now = function () return 123 end }) + assert_true(ok, err) + assert_eq(conn.retained['state/device/components'].counts.total, 1) +end + +return tests diff --git a/tests/unit/device/test_service.lua b/tests/unit/device/test_service.lua new file mode 100644 index 00000000..420f4cc1 --- /dev/null +++ b/tests/unit/device/test_service.lua @@ -0,0 +1,504 @@ +-- tests/unit/device/test_service.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' +local service = require 'services.device.service' +local device = require 'services.device' +local config = require 'services.device.config' +local topics = require 'services.device.topics' + +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end +local function assert_close(a, b, msg) + if math.abs(a - b) > 0.000001 then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end +end + +local function key(topic) return table.concat(topic, '/') end +local function fake_conn() + local c = { retained = {}, published = {}, binds = {}, subscriptions = {}, retain_calls = 0, unretain_calls = 0, publish_calls = 0, bind_calls = 0, unbind_calls = 0, subscribe_calls = 0, unsubscribe_calls = 0 } + function c:retain(topic, payload) self.retain_calls = self.retain_calls + 1; self.retained[key(topic)] = payload; return true end + function c:unretain(topic) + self.unretain_calls = self.unretain_calls + 1 + if self.fail_unretain then return false, self.fail_unretain end + self.retained[key(topic)] = nil + return true + end + function c:publish(topic, payload) self.publish_calls = self.publish_calls + 1; self.published[#self.published + 1] = { topic = topic, payload = payload }; return true end + function c:bind(topic, opts) + self.bind_calls = self.bind_calls + 1 + local ep = { topic = topic, opts = opts, id = 'ep-' .. tostring(self.bind_calls) } + self.binds[ep.id] = ep + return ep + end + function c:subscribe(topic, opts) + self.subscribe_calls = self.subscribe_calls + 1 + local _tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local sub = { topic = topic, opts = opts, id = 'sub-' .. tostring(self.subscribe_calls), rx = rx } + function sub:recv_op() return self.rx:recv_op() end + self.subscriptions[sub.id] = sub + return sub + end + function c:unsubscribe(sub) + self.unsubscribe_calls = self.unsubscribe_calls + 1 + if sub and sub.id then self.subscriptions[sub.id] = nil end + return true + end + function c:unbind(ep) + self.unbind_calls = self.unbind_calls + 1 + if self.fail_unbind then return false, self.fail_unbind end + if ep and ep.id then self.binds[ep.id] = nil end + return true + end + return c +end + +local function fake_svc() + local svc = { metrics = {} } + function svc:obs_metric(name, payload) + self.metrics[#self.metrics + 1] = { name = name, payload = payload } + end + return svc +end + +local function metrics_by_namespace(svc) + local out = {} + for i = 1, #(svc.metrics or {}) do + local rec = svc.metrics[i] + out[table.concat(rec.payload.namespace or {}, '.')] = rec + end + return out +end + +local function assert_metric(map, namespace, name, value) + local rec = map[namespace] + assert_not_nil(rec, 'missing metric ' .. namespace) + assert_eq(rec.name, name, 'metric topic key mismatch for ' .. namespace) + if type(value) == 'number' then + assert_close(rec.payload.value, value, 'metric value mismatch for ' .. namespace) + else + assert_eq(rec.payload.value, value, 'metric value mismatch for ' .. namespace) + end +end + +local function cfg_one(extra) + local c = { + schema = config.SCHEMA, + components = { + mcu = { subtype = 'mcu', facts = { software = topics.raw_member_state('mcu', 'software') } }, + }, + } + if extra then for k, v in pairs(extra) do c[k] = v end end + return c +end + +local function cfg_mcu_metrics() + return { + schema = config.SCHEMA, + components = { + mcu = { + subtype = 'mcu', + facts = { + runtime_memory = topics.raw_member_state('mcu', 'runtime', 'memory'), + environment_temperature = topics.raw_member_state('mcu', 'environment', 'temperature'), + environment_humidity = topics.raw_member_state('mcu', 'environment', 'humidity'), + power_battery = topics.raw_member_state('mcu', 'power', 'battery'), + power_charger = topics.raw_member_state('mcu', 'power', 'charger'), + }, + }, + }, + } +end + + +function tests.test_start_requires_conn_and_fiber() + local ok, err = pcall(function () service.start(nil, { watch_config = false }) end) + assert_eq(ok, false) + assert_true(tostring(err):find('conn required', 1, true) ~= nil, tostring(err)) + + ok, err = pcall(function () service.start(fake_conn(), { watch_config = false }) end) + assert_eq(ok, false) + assert_true(tostring(err):find('inside a fiber', 1, true) ~= nil, tostring(err)) +end + + +function tests.test_start_opens_cfg_device_with_shared_config_watch_and_closes_it() + fibers.run(function () + local conn = fake_conn() + local st, _rep, primary = fibers.run_scope(function (scope) + assert_true(scope:spawn(function () + device.start(conn, { + auto_publish = false, + enable_actions = false, + enable_observers = false, + }) + end)) + fibers.perform(sleep.sleep_op(0.001)) + assert_eq(conn.subscribe_calls, 1) + local sub = conn.subscriptions['sub-1'] + assert_not_nil(sub) + assert_eq(key(sub.topic), 'cfg/device') + scope:cancel('test_stop') + end) + + assert_eq(st, 'cancelled') + assert_eq(primary, 'test_stop') + assert_eq(conn.unsubscribe_calls, 1) + assert_nil(conn.subscriptions['sub-1']) + end) +end + +function tests.test_start_publishes_service_lifecycle_status() + fibers.run(function () + local conn = fake_conn() + local seen = {} + local parent_retain = conn.retain + function conn:retain(topic, payload) + local k = key(topic) + if k == 'svc/device/status' then + seen[#seen + 1] = payload.state .. ':' .. tostring(payload.ready) + end + return parent_retain(self, topic, payload) + end + + local st, _rep, primary = fibers.run_scope(function (scope) + assert_true(scope:spawn(function () + device.start(conn, { + watch_config = false, + auto_publish = false, + enable_actions = false, + enable_observers = false, + }) + end)) + fibers.perform(sleep.sleep_op(0.001)) + scope:cancel('test_stop') + end) + + assert_eq(st, 'cancelled') + assert_eq(primary, 'test_stop') + assert_true(seen[1] == 'starting:false' or seen[1] == 'running:false', 'missing starting lifecycle status') + assert_true(seen[#seen]:find('stopped:false', 1, true) ~= nil, 'missing stopped lifecycle status') + assert_nil(conn.retained['svc/device/status'], 'service_base finaliser should unretain lifecycle topics') + end) +end + +function tests.test_apply_config_starts_generation_and_publishes_public_state() + fibers.run(function (scope) + local conn = fake_conn() + local state = service.build_state(scope, { + conn = conn, + enable_actions = false, + enable_observers = false, + now = function () return 100 end, + }) + + local ok, err = service.apply_config_payload(state, cfg_one()) + assert_true(ok, err) + assert_eq(state.active.generation, 1) + assert_eq(state.model:snapshot().components.mcu.subtype, 'mcu') + + ok, err = service.flush_publication(state) + assert_true(ok, err) + assert_eq(conn.retained['state/device/component/mcu'].component, 'mcu') + assert_eq(conn.retained['state/device/components'].counts.total, 1) + end) +end + +function tests.test_same_effective_catalogue_does_not_restart_generation() + fibers.run(function (scope) + local state = service.build_state(scope, { + conn = fake_conn(), + enable_actions = false, + enable_observers = false, + }) + assert_true(service.apply_config_payload(state, cfg_one())) + local generation = state.active.generation + assert_true(service.apply_config_payload(state, cfg_one())) + assert_eq(state.active.generation, generation) + end) +end + +function tests.test_material_catalogue_change_replaces_generation() + fibers.run(function (scope) + local state = service.build_state(scope, { + conn = fake_conn(), + enable_actions = false, + enable_observers = false, + }) + assert_true(service.apply_config_payload(state, cfg_one())) + local first = state.active + assert_true(first.observer_root ~= nil, 'observer root was not created') + assert_true(first.action_root ~= nil, 'action root was not created') + assert_true(service.apply_config_payload(state, cfg_one({ components = { + mcu = { subtype = 'mcu', facts = { software = topics.raw_member_state('mcu', 'software') } }, + host = { class = 'host', subtype = 'cm5', facts = { software = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'software') } }, + } }))) + assert_eq(state.active.generation, 2) + local st = first.scope:status() + assert_eq(st, 'cancelled') + local observer_st = first.observer_root:status() + local action_st = first.action_root:status() + assert_eq(observer_st, 'cancelled') + assert_eq(action_st, 'cancelled') + end) +end + +function tests.test_publication_coalesces_repeated_materially_identical_observation() + fibers.run(function (scope) + local conn = fake_conn() + local state = service.build_state(scope, { + conn = conn, + enable_actions = false, + enable_observers = false, + now = function () return 100 end, + }) + + assert_true(service.apply_config_payload(state, cfg_one())) + assert_true(service.flush_publication(state)) + + local ev = { + kind = 'component_observation', + generation = state.active.generation, + component = 'mcu', + tag = 'fact_retained', + fact = 'software', + payload = { version = '1.0' }, + at = 123, + } + + assert_true(service.reduce_event(state, ev)) + assert_true(service.flush_publication(state)) + local retain_after_first = conn.retain_calls + local publish_after_first = conn.publish_calls + + assert_true(service.reduce_event(state, ev)) + assert_true(service.flush_publication(state)) + assert_eq(conn.retain_calls, retain_after_first) + assert_eq(conn.publish_calls, publish_after_first) + end) +end + +function tests.test_mcu_dirty_publication_emits_mcu_metrics() + fibers.run(function (scope) + local conn = fake_conn() + local state = service.build_state(scope, { + conn = conn, + enable_actions = false, + enable_observers = false, + now = function () return 100 end, + }) + + assert_true(service.apply_config_payload(state, cfg_mcu_metrics())) + local generation = state.active.generation + local observations = { + { fact = 'runtime_memory', payload = { alloc_bytes = 32768 } }, + { fact = 'environment_temperature', payload = { deci_c = 234 } }, + { fact = 'environment_humidity', payload = { rh_x100 = 4512 } }, + { fact = 'power_battery', payload = { + pack_mV = 12800, + ibat_mA = -120, + temp_mC = 24500, + } }, + { fact = 'power_charger', payload = { + vin_mV = 24000, + vsys_mV = 12800, + iin_mA = 500, + state_bits = 0x0142, + status_bits = 0x000d, + system_bits = 0x2045, + } }, + } + for i = 1, #observations do + local obs = observations[i] + assert_true(service.reduce_event(state, { + kind = 'component_observation', + generation = generation, + component = 'mcu', + tag = 'fact_retained', + fact = obs.fact, + payload = obs.payload, + })) + end + + local svc = fake_svc() + state.svc = svc + local ok, err = service.flush_publication(state) + assert_true(ok, err) + + local metrics = metrics_by_namespace(svc) + assert_metric(metrics, 'mcu.sys.mem.alloc', 'alloc', 32768) + assert_metric(metrics, 'mcu.env.temperature.core', 'core', 23.4) + assert_metric(metrics, 'mcu.env.humidity.core', 'core', 45.12) + assert_metric(metrics, 'mcu.power.temperature.internal', 'internal', 24.5) + assert_metric(metrics, 'mcu.power.battery.internal.vbat', 'vbat', 12800) + assert_metric(metrics, 'mcu.power.battery.internal.ibat', 'ibat', -120) + assert_metric(metrics, 'mcu.power.charger.internal.vin', 'vin', 24000) + assert_metric(metrics, 'mcu.power.charger.internal.vsys', 'vsys', 12800) + assert_metric(metrics, 'mcu.power.charger.internal.iin', 'iin', 500) + assert_metric(metrics, + 'mcu.power.charger.internal.system.charger_enabled', 'charger_enabled', true) + assert_metric(metrics, 'mcu.power.charger.internal.system.ok_to_charge', 'ok_to_charge', true) + assert_metric(metrics, 'mcu.power.charger.internal.system.vin_gt_vbat', 'vin_gt_vbat', true) + assert_metric(metrics, 'mcu.power.charger.internal.status.iin_limited', 'iin_limited', true) + assert_metric(metrics, 'mcu.power.charger.internal.status.uvcl_active', 'uvcl_active', true) + assert_metric(metrics, 'mcu.power.charger.internal.status.cc_phase', 'cc_phase', false) + assert_metric(metrics, 'mcu.power.charger.internal.status.cv_phase', 'cv_phase', true) + assert_metric(metrics, 'mcu.power.charger.internal.state.bat_missing', 'bat_missing', true) + assert_metric(metrics, 'mcu.power.charger.internal.state.cccv', 'cccv', true) + assert_metric(metrics, 'mcu.power.charger.internal.state.suspended', 'suspended', true) + assert_metric(metrics, 'mcu.power.charger.internal.state.bat_short', 'bat_short', false) + end) +end + +function tests.test_mcu_metrics_skip_empty_component_values() + fibers.run(function (scope) + local conn = fake_conn() + local state = service.build_state(scope, { + conn = conn, + enable_actions = false, + enable_observers = false, + now = function () return 100 end, + }) + + assert_true(service.apply_config_payload(state, cfg_mcu_metrics())) + local svc = fake_svc() + state.svc = svc + local ok, err = service.flush_publication(state) + assert_true(ok, err) + assert_eq(#svc.metrics, 0) + end) +end + +function tests.test_non_mcu_dirty_publication_does_not_emit_mcu_metrics() + fibers.run(function (scope) + local conn = fake_conn() + local state = service.build_state(scope, { + conn = conn, + enable_actions = false, + enable_observers = false, + now = function () return 100 end, + }) + + assert_true(service.apply_config_payload(state, { + schema = config.SCHEMA, + components = { + host = { + class = 'host', + subtype = 'cm5', + facts = { + software = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'software'), + }, + }, + }, + })) + local svc = fake_svc() + state.svc = svc + local ok, err = service.flush_publication(state) + assert_true(ok, err) + assert_eq(#svc.metrics, 0) + end) +end + + +function tests.test_publication_is_selected_as_semantic_event_after_dirty_change() + fibers.run(function (scope) + local conn = fake_conn() + local state = service.build_state(scope, { + conn = conn, + enable_actions = false, + enable_observers = false, + now = function () return 100 end, + }) + + assert_true(service.apply_config_payload(state, cfg_one())) + assert_eq(conn.retain_calls, 0, 'configuration should mark publication dirty, not publish inline') + assert_true(state.publication_requested) + + local ev = fibers.perform(service.next_event_op(state)) + assert_eq(ev.kind, 'publication_flush') + assert_true(service.reduce_event(state, ev)) + assert_eq(state.publication_requested, false) + assert_eq(conn.retained['state/device/component/mcu'].component, 'mcu') + end) +end + + +local function count_keys(t) + local n = 0 + for _ in pairs(t or {}) do n = n + 1 end + return n +end + +function tests.test_unbind_failure_during_generation_replacement_is_surfaced_and_records_kept() + fibers.run(function (scope) + local conn = fake_conn() + local state = service.build_state(scope, { + conn = conn, + enable_observers = false, + auto_publish = false, + }) + + local ok, err = service.apply_config_payload(state, cfg_one()) + assert_true(ok, err) + local first = state.active + assert_not_nil(first) + assert_true(count_keys(first.action_eps) > 0, 'first generation should expose action endpoints') + + conn.fail_unbind = 'synthetic unbind failure' + ok, err = service.apply_config_payload(state, cfg_one({ components = { + mcu = { subtype = 'mcu', facts = { software = topics.raw_member_state('mcu', 'software') } }, + host = { class = 'host', subtype = 'cm5', facts = { software = topics.raw_host_cap_state('updater', 'updater', 'cm5', 'software') } }, + } })) + + assert_nil(ok) + assert_true(tostring(err):find('synthetic unbind failure', 1, true) ~= nil, tostring(err)) + assert_true(count_keys(first.action_eps) > 0, 'old endpoint records must not be cleared when unbind fails') + + -- Let the enclosing test scope finalisers clean up the generation. + conn.fail_unbind = nil + end) +end + +function tests.test_publication_cleanup_is_checked_and_keeps_records_on_failure() + fibers.run(function (scope) + local conn = fake_conn() + local state = service.build_state(scope, { + conn = conn, + enable_actions = false, + enable_observers = false, + now = function () return 100 end, + }) + assert_true(service.apply_config_payload(state, cfg_one())) + assert_true(service.flush_publication(state)) + assert_true(state.published_components.mcu) + + conn.fail_unretain = 'synthetic unretain failure' + local ok, err = service.cleanup_publication_now(state) + assert_nil(ok) + assert_true(tostring(err):find('synthetic unretain failure', 1, true) ~= nil, tostring(err)) + assert_true(state.published_components.mcu, 'publication record must remain until durable cleanup succeeds') + + -- Let the enclosing test scope finalisers clean up retained publication. + conn.fail_unretain = nil + end) +end + + +function tests.test_cleanup_publication_ignores_summary_identity_when_never_published() + fibers.run(function (scope) + local state = service.build_state(scope, { + conn = {}, + enable_actions = false, + enable_observers = false, + }) + + local ok, err = service.cleanup_publication_now(state) + assert_true(ok, err) + end) +end + +return tests diff --git a/tests/unit/device/test_static.lua b/tests/unit/device/test_static.lua new file mode 100644 index 00000000..064b5ac3 --- /dev/null +++ b/tests/unit/device/test_static.lua @@ -0,0 +1,204 @@ +-- tests/unit/device/test_static.lua + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_false(v, msg) if v ~= false then fail(msg or ('expected false, got ' .. tostring(v))) end end + +local files = { + '../src/services/device.lua', + '../src/services/device/service.lua', + '../src/services/device/observer.lua', + '../src/services/device/observer_manager.lua', + '../src/services/device/action_manager.lua', + '../src/services/device/action_worker.lua', + '../src/services/device/model.lua', + '../src/services/device/publisher.lua', + '../src/services/device/projection.lua', + '../src/services/device/mcu_metrics.lua', + '../src/services/device/catalogue.lua', + '../src/services/device/component_mcu.lua', + '../src/services/device/component_host.lua', + '../src/services/device/topics.lua', + '../src/services/device/availability.lua', + '../src/services/device/fabric_stage.lua', + '../src/services/device/backpressure.lua', +} + +local function read(path) + local f = assert(io.open(path, 'r')) + local s = f:read('*a') + f:close() + return s +end + +local function each_line(s) + local lines = {} + for line in (s .. '\n'):gmatch('([^\n]*)\n') do + lines[#lines + 1] = line + end + return lines +end + +local function finaliser_lines(path, s) + local out = {} + local in_finaliser = false + local start_line = nil + + for i, line in ipairs(each_line(s)) do + if not in_finaliser and line:find(':finally%s*%(%s*function', 1) then + in_finaliser = true + start_line = i + end + + if in_finaliser then + out[#out + 1] = { path = path, line_no = i, line = line, start_line = start_line } + if line:match('^%s*end%)') then + in_finaliser = false + start_line = nil + end + end + end + + return out +end + +local function coordinator_lines(path, s) + local out = {} + local in_coord = false + local start_line = nil + local interesting = { + 'cancel_active_generation', + 'apply_config_payload', + 'reduce_event', + 'coordinator_loop', + 'flush_publication', + 'handle_observation', + 'handle_observer_done', + 'handle_action_done', + 'handle_generation_done', + } + + for i, line in ipairs(each_line(s)) do + if not in_coord then + for _, name in ipairs(interesting) do + if line:find('local%s+function%s+' .. name .. '%s*%(', 1) then + in_coord = true + start_line = i + break + end + end + end + + if in_coord then + out[#out + 1] = { path = path, line_no = i, line = line, start_line = start_line } + if line:match('^end$') then + in_coord = false + start_line = nil + end + end + end + + return out +end + +local function assert_no_waiting_cleanup_or_inline_join(records, label) + for _, rec in ipairs(records) do + local line = rec.line + assert_false(line:find(':close_op%s*%(', 1) ~= nil, + ('%s in %s:%d calls close_op'):format(label, rec.path, rec.line_no)) + assert_false(line:find(':join_op%s*%(', 1) ~= nil, + ('%s in %s:%d calls join_op'):format(label, rec.path, rec.line_no)) + assert_false(line:find('perform_raw', 1, true) ~= nil, + ('%s in %s:%d calls perform_raw'):format(label, rec.path, rec.line_no)) + end +end + +function tests.test_device_service_code_does_not_use_perform_raw() + for _, path in ipairs(files) do + local s = read(path) + assert_false(s:find('perform_raw', 1, true) ~= nil, path .. ' uses perform_raw') + end +end + +function tests.test_device_finalisers_do_not_wait_or_join() + for _, path in ipairs(files) do + assert_no_waiting_cleanup_or_inline_join(finaliser_lines(path, read(path)), 'finaliser') + end +end + +function tests.test_device_coordinator_paths_do_not_close_or_join() + for _, path in ipairs(files) do + assert_no_waiting_cleanup_or_inline_join(coordinator_lines(path, read(path)), 'coordinator') + end +end + + +function tests.test_device_service_uses_shared_config_watch_helper() + local s = read('../src/services/device/service.lua') + assert_false(s:find("devicecode.support.config_watch", 1, true) == nil, + 'device service should require the shared config_watch helper') + assert_false(s:find("config_watch.open(conn, 'device'", 1, true) == nil, + 'device service should open cfg/device through config_watch.open') +end + +function tests.test_device_generation_is_direct_scope_not_parked_scoped_work() + local s = read('../src/services/device/service.lua') + assert_false(s:find('park_generation_until_parent_closes', 1, true) ~= nil, + 'device generation still uses parked shell') + assert_false(s:find("local scoped_work", 1, true) ~= nil, + 'device service should not use scoped_work for generation ownership') + assert_false(s:find("kind = 'generation_done'", 1, true) ~= nil, + 'device generation should not report synthetic generation_done completions') +end + +function tests.test_device_priority_path_is_admission_sensitive_not_global() + local s = read('../src/services/device/service.lua') + assert_false(s:find('return priority_event.sources_op', 1, true) == nil, + 'device service should still use priority_event for admission-sensitive cases') + assert_false(s:find('has_actions and has_config', 1, true) == nil, + 'device priority path should be gated to action admission plus config') + assert_false(s:find('device.next_event.admission_sensitive', 1, true) == nil, + 'device priority path should be labelled as admission-sensitive') +end + + +function tests.test_device_components_report_through_service_event_ports() + local svc = read('../src/services/device/service.lua') + assert_false(svc:find("devicecode.support.service_events", 1, true) == nil, + 'device service should create a service event port') + assert_false(svc:find('events_port = service_events.port', 1, true) == nil, + 'device service should expose a stamped event port to child reporters') + + local action_manager = read('../src/services/device/action_manager.lua') + assert_false(action_manager:find('service_events.reporter_for', 1, true) == nil, + 'action completions should use service_events.reporter_for') + assert_false(action_manager:find('report = function (ev)', 1, true) ~= nil, + 'action manager should not install a parent callback-shaped report body') + + local observer_manager = read('../src/services/device/observer_manager.lua') + assert_false(observer_manager:find('service_events.reporter_for', 1, true) == nil, + 'observer completions should use service_events.reporter_for') + assert_false(observer_manager:find('report = function (ev)', 1, true) ~= nil, + 'observer manager should not install a parent callback-shaped report body') +end + +function tests.test_device_uses_terminate_vocabulary_for_owned_resources() + for _, path in ipairs(files) do + local s = read(path) + assert_false(s:find('close_source', 1, true) ~= nil, path .. ' still uses close_source') + assert_false(s:find('on_cancel =', 1, true) ~= nil, path .. ' still passes ignored scoped_work on_cancel') + assert_false(s:find('owned:close', 1, true) ~= nil, path .. ' still calls owned:close') + assert_false(s:find('state.model:close', 1, true) ~= nil, path .. ' still terminates the model through close vocabulary') + end +end + +function tests.test_device_action_worker_uses_current_scope_spawn_for_timer() + local s = read('../src/services/device/action_worker.lua') + assert_false(s:find('scope:spawn(function', 1, true) ~= nil, + 'action_worker timeout helper should use fibers.spawn into the current scope') + assert_false(s:find('fibers.spawn(function', 1, true) == nil, + 'action_worker should use fibers.spawn for its current-scope timer helper') +end + +return tests diff --git a/tests/unit/device/test_wired_provider_curation.lua b/tests/unit/device/test_wired_provider_curation.lua new file mode 100644 index 00000000..94e04d7a --- /dev/null +++ b/tests/unit/device/test_wired_provider_curation.lua @@ -0,0 +1,85 @@ +local projection = require 'services.device.projection' +local catalogue = require 'services.device.catalogue' + +local tests = {} +local function assert_not_nil(v,msg) if v == nil then error(msg or 'expected non-nil',2) end end +local function assert_eq(a,b,msg) if a ~= b then error(msg or ('expected '..tostring(b)..', got '..tostring(a)),2) end end + +function tests.test_device_projects_physical_assembly_not_wired_observation_capability() + local cat = catalogue.build({ + assembly = { + product = 'big-box', + components = { + ['switch-main'] = { kind = 'switch', role = 'wired-fabric' }, + }, + surfaces = { + ['lan-1'] = { + component = 'switch-main', + observed_surface = 'GE2', + exposure = 'external', + }, + }, + }, + components = { + ['switch-main'] = { + kind = 'switch', + module = 'switch', + class = 'host', + role = 'switch-fabric', + member = 'switch-main', + facts = { + wired_observation_status = { 'raw', 'host', 'wired', 'provider', 'switch-main', 'status' }, + }, + }, + }, + }) + local snap = { generation = 7, catalogue = cat, components = cat.components } + local assembly = projection.assembly_payload(snap, 123) + assert_eq(assembly.kind, 'device.assembly') + assert_eq(assembly.product, 'big-box') + assert_eq(assembly.generation, 7) + assert_eq(assembly.surfaces['lan-1'].component, 'switch-main') + assert_eq(assembly.surfaces['lan-1'].observed_surface, 'GE2') + + local payloads = projection.component_payloads('switch-main', cat.components['switch-main'], 123) + assert_eq(payloads.wired_observation, nil, 'Device must not publish state/wired payloads') + assert_not_nil(payloads.component, 'component payload should still be projected') + assert_not_nil(payloads.component.observed, 'component retains observed facts locally') +end + + +function tests.test_device_assembly_rejects_provider_surface_aliases() + local ok, err = pcall(function () + catalogue.build({ + assembly = { + surfaces = { + ['lan-1'] = { + provider = 'switch-main', + provider_surface_id = 'GE2', + }, + }, + }, + }) + end) + assert_eq(ok, false) + assert_not_nil(tostring(err):find('component and observed_surface only', 1, true)) +end + +function tests.test_device_assembly_rejects_link_surface_alias() + local ok, err = pcall(function () + catalogue.build({ + assembly = { + links = { + ['cm5-switch'] = { + a = { component = 'cm5-local-wired', surface = 'eth0' }, + b = { component = 'switch-main', observed_surface = 'GE8' }, + }, + }, + }, + }) + end) + assert_eq(ok, false) + assert_not_nil(tostring(err):find('component and observed_surface only', 1, true)) +end + +return tests diff --git a/tests/unit/devicecode/blob_source_spec.lua b/tests/unit/devicecode/blob_source_spec.lua new file mode 100644 index 00000000..18b681ab --- /dev/null +++ b/tests/unit/devicecode/blob_source_spec.lua @@ -0,0 +1,158 @@ +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local runfibers = require 'tests.support.run_fibers' +local blob_source = require 'devicecode.blob_source' + +local T = {} + +function T.string_source_reads_chunks_and_reaches_eof() + runfibers.run(function() + local src = blob_source.from_string('abcdef') + + local c1, e1 = fibers.perform(src:read_chunk_op(2)) + assert(c1 == 'ab' and e1 == nil) + + local c2, e2 = fibers.perform(src:read_chunk_op(3)) + assert(c2 == 'cde' and e2 == nil) + + local c3, e3 = fibers.perform(src:read_chunk_op(3)) + assert(c3 == 'f' and e3 == nil) + + local c4, e4 = fibers.perform(src:read_chunk_op(3)) + assert(c4 == nil and e4 == nil) + end) +end + +function T.string_source_rejects_invalid_max_bytes() + runfibers.run(function() + local src = blob_source.from_string('abc') + local chunk, err = fibers.perform(src:read_chunk_op(0)) + assert(chunk == nil) + assert(tostring(err):match('invalid max_bytes')) + end) +end + +function T.memory_sink_accumulates_written_chunks() + runfibers.run(function() + local sink = blob_source.to_memory() + local ok1, err1 = fibers.perform(sink:write_chunk_op('ab')) + local ok2, err2 = fibers.perform(sink:write_chunk_op('cd')) + assert(ok1 == true and err1 == nil) + assert(ok2 == true and err2 == nil) + assert(sink:result() == 'abcd') + end) +end + +function T.copy_op_copies_string_source_to_memory_sink() + runfibers.run(function() + local src = blob_source.from_string('hello world') + local sink = blob_source.to_memory() + + local st, rep, bytes = fibers.perform(blob_source.copy_op(src, sink, { chunk_size = 4 })) + assert(st == 'ok', tostring(bytes or rep)) + assert(bytes == 11) + assert(sink:result() == 'hello world') + end) +end + +function T.copy_op_does_not_close_when_disabled() + runfibers.run(function() + local src_closed = 0 + local sink_closed = 0 + local src = { + remaining = true, + read_chunk_op = function(self, n) + return op.guard(function() + if self.remaining then + self.remaining = false + return op.always('xy', nil) + end + return op.always(nil, nil) + end) + end, + terminate = function(_) + src_closed = src_closed + 1 + return true, nil + end, + } + local sink = { + chunks = {}, + write_chunk_op = function(self, chunk) + return op.guard(function() + self.chunks[#self.chunks + 1] = chunk + return op.always(true, nil) + end) + end, + terminate = function(_) + sink_closed = sink_closed + 1 + return true, nil + end, + } + + local st, rep, bytes = fibers.perform(blob_source.copy_op(src, sink, { + close_source = false, + close_sink = false, + })) + assert(st == 'ok', tostring(bytes or rep)) + assert(bytes == 2) + assert(src_closed == 0) + assert(sink_closed == 0) + end) +end + +function T.copy_op_closes_on_timeout_losing_arm() + runfibers.run(function() + local src_closed = 0 + local sink_closed = 0 + local src = { + read_chunk_op = function() + return sleep.sleep_op(0.2):wrap(function() return 'late', nil end) + end, + terminate = function(_) + src_closed = src_closed + 1 + return true, nil + end, + } + local sink = { + write_chunk_op = function(self, chunk) + return op.always(true, nil) + end, + terminate = function(_) + sink_closed = sink_closed + 1 + return true, nil + end, + } + + local which = fibers.perform(fibers.named_choice{ + copy = blob_source.copy_op(src, sink):wrap(function(...) return 'copy', ... end), + timeout = sleep.sleep_op(0.02):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout') + + fibers.perform(sleep.sleep_op(0.02)) + + assert(src_closed == 1) + assert(sink_closed == 1) + end) +end + +function T.copy_op_fails_when_sink_write_fails() + runfibers.run(function() + local src = blob_source.from_string('boom') + local sink = { + write_chunk_op = function(self, chunk) + return op.always(nil, 'sink failed') + end, + terminate = function() + return true, nil + end, + } + + local st, rep, primary = fibers.perform(blob_source.copy_op(src, sink)) + assert(st == 'failed') + assert(tostring(primary):match('sink failed')) + end) +end + +return T diff --git a/tests/unit/devicecode/exec_direct_usage_spec.lua b/tests/unit/devicecode/exec_direct_usage_spec.lua new file mode 100644 index 00000000..120d35e1 --- /dev/null +++ b/tests/unit/devicecode/exec_direct_usage_spec.lua @@ -0,0 +1,43 @@ +local T = {} + +local function read_file(path) + local f = assert(io.open(path, 'r')) + local s = f:read('*a') + f:close() + return s +end + +local function list_src_lua_files() + local roots = { '../src', 'src' } + for _, root in ipairs(roots) do + local p = io.popen(('find %s -type f -name "*.lua" | sort 2>/dev/null'):format(root)) + if p then + local files = {} + for line in p:lines() do files[#files + 1] = line end + p:close() + if #files > 0 then return files end + end + end + error('could not enumerate src lua files') +end + +local function assert_no_source_match(pattern, label) + local violations = {} + for _, path in ipairs(list_src_lua_files()) do + local src = read_file(path) + if src:find(pattern) then + violations[#violations + 1] = path:gsub('^%.%./', '') + end + end + assert(#violations == 0, label .. ': ' .. table.concat(violations, ', ')) +end + +function T.source_exec_usage_is_explicit_without_policy_wrapper() + assert_no_source_match('devicecode%.support%.exec', 'top-level exec policy wrapper must not be used') + assert_no_source_match('os%.execute', 'os.execute must not be used in src') + assert_no_source_match('io%.popen', 'io.popen must not be used in src') + assert_no_source_match('exec%.command%([\'\"]', 'exec.command vararg literals must use explicit table specs') + assert_no_source_match('exec%.command%(unpack', 'exec.command(unpack(argv)) must build an explicit table spec') +end + +return T diff --git a/tests/unit/devicecode/service_base_spec.lua b/tests/unit/devicecode/service_base_spec.lua new file mode 100644 index 00000000..f9dbf592 --- /dev/null +++ b/tests/unit/devicecode/service_base_spec.lua @@ -0,0 +1,272 @@ +local busmod = require 'bus' +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' + +local runfibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' +local service_base = require 'devicecode.service_base' + +local T = {} + +local function wait_payload(conn, topic, timeout) + return probe.wait_payload(conn, topic, { timeout = timeout or 0.5 }) +end + +local function expect_no_message(conn, topic, timeout) + timeout = timeout or 0.05 + local sub = conn:subscribe(topic) + + local which = fibers.perform(op.named_choice{ + msg = sub:recv_op():wrap(function() return 'msg' end), + timeout = sleep.sleep_op(timeout):wrap(function() return 'timeout' end), + }) + + sub:unsubscribe() + assert(which == 'timeout', 'unexpected retained/publication on ' .. table.concat(topic, '/')) +end + +function T.new_service_publishes_meta_announce_and_status() + runfibers.run(function() + local bus = busmod.new() + local conn = bus:connect() + local reader = bus:connect() + + local svc = service_base.new(conn, { + name = 'alpha', + env = 'test', + meta = { role = 'worker' }, + announce = { boot = 'cold' }, + }) + + local meta = wait_payload(reader, { 'svc', 'alpha', 'meta' }, 0.5) + assert(meta.service == 'alpha') + assert(meta.env == 'test') + assert(meta.role == 'worker') + assert(type(meta.run_id) == 'string' and meta.run_id ~= '') + + local ann = wait_payload(reader, { 'svc', 'alpha', 'announce' }, 0.5) + assert(ann.service == 'alpha') + assert(ann.boot == 'cold') + assert(ann.run_id == meta.run_id) + + local status_payload = svc:running({ phase = 'steady' }) + assert(status_payload.state == 'running') + assert(status_payload.phase == 'steady') + assert(status_payload.run_id == meta.run_id) + + local status = wait_payload(reader, { 'svc', 'alpha', 'status' }, 0.5) + assert(status.state == 'running') + assert(status.phase == 'steady') + assert(status.run_id == meta.run_id) + end) +end + + +function T.lifecycle_status_retains_only_on_semantic_change() + runfibers.run(function() + local bus = busmod.new() + local conn = bus:connect() + local reader = bus:connect() + local svc = service_base.new(conn, { name = 'idem', env = 'test' }) + local sub = reader:subscribe({ 'svc', 'idem', 'status' }) + + svc:running({ ready = false, phase = 'steady' }) + local first_which, first_msg = fibers.perform(op.named_choice{ + msg = sub:recv_op():wrap(function(msg) return msg end), + timeout = sleep.sleep_op(0.5):wrap(function() return nil end), + }) + assert(first_which == 'msg' and first_msg and first_msg.payload and first_msg.payload.phase == 'steady') + + svc:running({ ready = false, phase = 'steady' }) + local which = fibers.perform(op.named_choice{ + msg = sub:recv_op():wrap(function() return 'msg' end), + timeout = sleep.sleep_op(0.05):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout', 'unchanged lifecycle status should not be retained again') + + svc:running({ ready = true, phase = 'steady' }) + local changed_which, changed_msg = fibers.perform(op.named_choice{ + msg = sub:recv_op():wrap(function(msg) return msg end), + timeout = sleep.sleep_op(0.5):wrap(function() return nil end), + }) + sub:unsubscribe() + assert(changed_which == 'msg' and changed_msg and changed_msg.payload and changed_msg.payload.ready == true) + end) +end + +function T.obs_helpers_publish_legacy_and_v1_topics() + runfibers.run(function() + local bus = busmod.new() + local conn = bus:connect() + local reader = bus:connect() + + local svc = service_base.new(conn, { name = 'beta', env = 'test' }) + + -- Subscribe BEFORE publishing non-retained events/logs. + local evt_legacy_sub = reader:subscribe({ 'obs', 'event', 'beta', 'tick' }) + local evt_v1_sub = reader:subscribe({ 'obs', 'v1', 'beta', 'event', 'tick' }) + + local log_legacy_sub = reader:subscribe({ 'obs', 'log', 'beta', 'info' }) + local log_v1_sub = reader:subscribe({ 'obs', 'v1', 'beta', 'event', 'log' }) + + local counter_v1_sub = reader:subscribe({ 'obs', 'v1', 'beta', 'counter', 'restarts' }) + + svc:obs_event('tick', { n = 1 }) + svc:obs_state('phase', { value = 'booting' }) + svc:obs_log('info', { message = 'hello' }) + svc:obs_metric('temperature', { c = 42 }) + svc:obs_counter('restarts', { n = 3 }) + + local evt_legacy, err = evt_legacy_sub:recv() + assert(evt_legacy, tostring(err)) + assert(type(evt_legacy.payload) == 'table') + assert(evt_legacy.payload.n == 1) + + local evt_v1, err2 = evt_v1_sub:recv() + assert(evt_v1, tostring(err2)) + assert(type(evt_v1.payload) == 'table') + assert(evt_v1.payload.n == 1) + + local state_legacy = wait_payload(reader, { 'obs', 'state', 'beta', 'phase' }, 0.2) + assert(type(state_legacy) == 'table') + assert(state_legacy.value == 'booting') + + local state_v1 = wait_payload(reader, { 'obs', 'v1', 'beta', 'metric', 'phase' }, 0.2) + assert(type(state_v1) == 'table') + assert(state_v1.value == 'booting') + + local log_legacy, err3 = log_legacy_sub:recv() + assert(log_legacy, tostring(err3)) + assert(type(log_legacy.payload) == 'table') + assert(log_legacy.payload.message == 'hello') + + local log_v1, err4 = log_v1_sub:recv() + assert(log_v1, tostring(err4)) + assert(type(log_v1.payload) == 'table') + assert(log_v1.payload.message == 'hello') + assert(log_v1.payload.level == 'info') + + local metric_v1 = wait_payload(reader, { 'obs', 'v1', 'beta', 'metric', 'temperature' }, 0.2) + assert(type(metric_v1) == 'table') + assert(metric_v1.c == 42) + + local counter_v1, err5 = counter_v1_sub:recv() + assert(counter_v1, tostring(err5)) + assert(counter_v1.payload.n == 3) + + evt_legacy_sub:unsubscribe() + evt_v1_sub:unsubscribe() + log_legacy_sub:unsubscribe() + log_v1_sub:unsubscribe() + counter_v1_sub:unsubscribe() + end) +end + +function T.scope_exit_unretains_tracked_topics() + runfibers.run(function(scope) + local bus = busmod.new() + local reader = bus:connect() + + local child, child_err = scope:child() + assert(child, tostring(child_err)) + + local ok, err = child:spawn(function() + local svc = service_base.new(bus:connect(), { + name = 'gamma', + env = 'test', + meta = { role = 'ephemeral' }, + announce = { mode = 'test' }, + }) + svc:ready({ detail = 'complete' }) + svc:obs_metric('phase', { value = 'ready' }) + end) + assert(ok, tostring(err)) + + local st = fibers.perform(child:join_op()) + assert(st == 'ok' or st == 'cancelled' or st == 'failed') + + expect_no_message(reader, { 'svc', 'gamma', 'meta' }, 0.05) + expect_no_message(reader, { 'svc', 'gamma', 'announce' }, 0.05) + expect_no_message(reader, { 'svc', 'gamma', 'status' }, 0.05) + expect_no_message(reader, { 'obs', 'state', 'gamma', 'status' }, 0.05) + expect_no_message(reader, { 'obs', 'v1', 'gamma', 'metric', 'status' }, 0.05) + expect_no_message(reader, { 'obs', 'v1', 'gamma', 'metric', 'phase' }, 0.05) + end) +end + +function T.wait_service_ready_returns_on_explicit_ready() + runfibers.run(function(scope) + local bus = busmod.new() + local admin = bus:connect() + + local ok, err = scope:spawn(function() + sleep.sleep(0.01) + admin:retain({ 'svc', 'delta', 'status' }, { + state = 'starting', + ts = 1, + at = 't1', + }) + sleep.sleep(0.01) + admin:retain({ 'svc', 'delta', 'status' }, { + state = 'running', + ready = true, + ts = 2, + at = 't2', + }) + end) + assert(ok, tostring(err)) + + local payload, wait_err = service_base.wait_service_ready(bus:connect(), 'delta', { + timeout = 0.5, + }) + assert(payload, tostring(wait_err)) + assert(payload.state == 'running') + assert(payload.ready == true) + end) +end + +function T.wait_service_ready_accepts_running_without_ready_when_requested() + runfibers.run(function(scope) + local bus = busmod.new() + local admin = bus:connect() + + local ok, err = scope:spawn(function() + sleep.sleep(0.01) + admin:retain({ 'svc', 'zeta', 'status' }, { + state = 'starting', + ts = 1, + at = 't1', + }) + sleep.sleep(0.01) + admin:retain({ 'svc', 'zeta', 'status' }, { + state = 'running', + ts = 2, + at = 't2', + }) + end) + assert(ok, tostring(err)) + + local payload, wait_err = service_base.wait_service_ready(bus:connect(), 'zeta', { + timeout = 0.5, + accept_running_without_ready = true, + }) + assert(payload, tostring(wait_err)) + assert(payload.state == 'running') + assert(payload.ready == nil) + end) +end + +function T.wait_service_ready_times_out_cleanly() + runfibers.run(function() + local bus = busmod.new() + + local payload, err = service_base.wait_service_ready(bus:connect(), 'epsilon', { + timeout = 0.05, + }) + assert(payload == nil) + assert(err == 'timeout') + end) +end + +return T diff --git a/tests/unit/devicecode/signal_bridge_spec.lua b/tests/unit/devicecode/signal_bridge_spec.lua new file mode 100644 index 00000000..0b2187c2 --- /dev/null +++ b/tests/unit/devicecode/signal_bridge_spec.lua @@ -0,0 +1,89 @@ +local T = {} + +local function with_fake_posix(fn) + local saved_signal = package.loaded['posix.signal'] + local saved_unistd = package.loaded['posix.unistd'] + local saved_nixio = package.loaded['nixio'] + local saved_sleep = package.loaded['fibers.sleep'] + local saved_mod = package.loaded['devicecode.signal_bridge'] + + local handlers = {} + local killed + + package.loaded['posix.signal'] = { + SIGTERM = 15, + SIGINT = 2, + signal = function(signum, handler) + local old = handlers[signum] or true + handlers[signum] = handler + return old + end, + kill = function(pid, sig) + killed = { pid = pid, sig = sig } + return true + end, + } + package.loaded['posix.unistd'] = { + getpid = function() return 12345 end, + } + package.loaded['nixio'] = nil + package.loaded['fibers.sleep'] = { + sleep = function(_seconds) + error('test watcher slept before cancellation', 0) + end, + } + package.loaded['devicecode.signal_bridge'] = nil + + local ok, a, b = pcall(function() + return fn(handlers, function() return killed end) + end) + + package.loaded['posix.signal'] = saved_signal + package.loaded['posix.unistd'] = saved_unistd + package.loaded['nixio'] = saved_nixio + package.loaded['fibers.sleep'] = saved_sleep + package.loaded['devicecode.signal_bridge'] = saved_mod + + if not ok then error(a, 0) end + return a, b +end + +function T.signal_bridge_cancels_scope_from_watcher_not_handler() + with_fake_posix(function(handlers) + local bridge = require 'devicecode.signal_bridge' + local spawned + local restored + local cancelled + local scope = { + spawn = function(_self, fn) + spawned = fn + return true + end, + finally = function(_self, fn) + restored = fn + return function() end + end, + cancel = function(_self, reason) + cancelled = reason + end, + } + + local ok, backend = bridge.install(scope, { TERM = true }) + assert(ok == true) + assert(backend == 'posix.signal') + assert(type(spawned) == 'function') + assert(type(restored) == 'function') + assert(type(handlers[15]) == 'function') + assert(cancelled == nil) + + handlers[15]() + assert(cancelled == nil, 'handler must not cancel scope directly') + + spawned() + assert(cancelled == 'signal:TERM') + + restored() + end) +end + +return T diff --git a/tests/unit/devicecode/support_contracts_spec.lua b/tests/unit/devicecode/support_contracts_spec.lua new file mode 100644 index 00000000..821c402b --- /dev/null +++ b/tests/unit/devicecode/support_contracts_spec.lua @@ -0,0 +1,25 @@ +local contracts = require 'devicecode.support.contracts' + +local tests = {} +local function assert_true(v, msg) if v ~= true then error(msg or 'expected true', 2) end end +local function assert_raises(fn) + local ok = pcall(fn) + if ok then error('expected error', 2) end +end + +function tests.test_mailbox_contracts_are_shape_based() + local rx = { recv_op = function () end } + local tx = { send_op = function () end } + assert_true(contracts.is_mailbox_rx(rx)) + assert_true(contracts.is_mailbox_tx(tx)) + contracts.require_rx(rx, 'rx') + contracts.require_tx(tx, 'tx') + assert_raises(function () contracts.require_rx({}, 'rx') end) +end + +function tests.test_resource_contracts() + contracts.require_terminate({ terminate = function () end }, 'res') + assert_raises(function () contracts.require_terminate({}, 'res') end) +end + +return tests diff --git a/tests/unit/fabric/test_bridge.lua b/tests/unit/fabric/test_bridge.lua new file mode 100644 index 00000000..b1d19c29 --- /dev/null +++ b/tests/unit/fabric/test_bridge.lua @@ -0,0 +1,333 @@ +-- tests/unit/fabric/test_bridge.lua +-- +-- Contract tests for the callback-free Fabric RPC bridge and local bus adapter. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' + +local bridge = require 'services.fabric.bridge' +local bus_adapter = require 'services.fabric.bus_adapter' +local session = require 'services.fabric.session' +local protocol = require 'services.fabric.protocol' +local queue = require 'devicecode.support.queue' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil value') end end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end + +local function recv_with_timeout(rx, label, timeout) + timeout = timeout or 0.25 + local which, item = fibers.perform(fibers.named_choice{ + item = rx:recv_op(), + timeout = sleep.sleep_op(timeout), + }) + if which == 'timeout' then fail('timed out waiting for ' .. tostring(label or 'item')) end + return item +end + +local function try_recv(rx) + return queue.try_recv_now(rx) +end + +local function ctx(gen, sid) + return session.new_session_context { + link_id = 'link-a', + link_generation = 1, + session_generation = gen or 1, + peer_sid = sid or 'sid-1', + peer_node = 'mcu', + proto = protocol.PROTO, + } +end + +local function peer_session_event(gen, sid) + return { kind = 'peer_session', session = ctx(gen, sid), at = fibers.now() } +end + +local function peer_drop_event(gen, sid, reason) + return { kind = 'peer_session_dropped', session = ctx(gen, sid), reason = reason or 'test_drop', at = fibers.now() } +end + +local function rpc_event(frame, gen, sid) + return { kind = 'session_frame', lane = 'rpc', session = ctx(gen, sid), frame = frame, at = fibers.now() } +end + +local function fake_request() + local r = { resolved = false } + function r:reply(v) self.resolved = 'reply'; self.value = v; return true end + function r:fail(e) self.resolved = 'fail'; self.err = e; return true end + return r +end + +local function start_bridge(scope, opts) + opts = opts or {} + local local_tx, local_rx = mailbox.new(32, { full = 'reject_newest' }) + local session_in_tx, session_in_rx = mailbox.new(32, { full = 'reject_newest' }) + local session_tx, session_rx = mailbox.new(32, { full = 'reject_newest' }) + local bus_tx, bus_rx = mailbox.new(32, { full = 'reject_newest' }) + local state_tx, state_rx = mailbox.new(32, { full = 'reject_newest' }) + local outbound_control_tx, outbound_control_rx = mailbox.new(32, { full = 'reject_newest' }) + local outbound_rpc_tx, outbound_rpc_rx = mailbox.new(32, { full = 'reject_newest' }) + local outbound_bulk_tx, outbound_bulk_rx = mailbox.new(32, { full = 'reject_newest' }) + local outbound = session.new_outbound_gate { + tx_control = outbound_control_tx, + tx_rpc = outbound_rpc_tx, + tx_bulk = outbound_bulk_tx, + } + local done_tx, done_rx = mailbox.new(1, { full = 'reject_newest' }) + local ok_forward, forward_err = scope:spawn(function () + while true do + local ev = fibers.perform(session_in_rx:recv_op()) + if ev == nil then + session_tx:close('test complete') + return + end + if ev.kind == 'peer_session' then + outbound:bind(ev.session) + elseif ev.kind == 'peer_session_dropped' then + outbound:drop(ev.reason or 'test_drop') + end + local ok, err = queue.try_admit_required(session_tx, ev, 'test_session_forward_failed') + if ok ~= true then error(err or 'test_session_forward_failed', 0) end + end + end) + assert_true(ok_forward, forward_err) + + local params = { + link_id = 'link-a', + link_generation = 1, + local_rx = local_rx, + session_rx = session_rx, + outbound = outbound, + bus_tx = bus_tx, + state_tx = state_tx, + import_rules = opts.import_rules or { + { remote_prefix = { 'state', 'self' }, local_prefix = { 'raw', 'member', 'mcu', 'state' } }, + { remote_prefix = { 'event', 'self' }, local_prefix = { 'raw', 'member', 'mcu', 'cap', 'telemetry', 'main', 'event' } }, + }, + export_publish_rules = opts.export_publish_rules or { + { local_prefix = { 'local' }, remote_prefix = { 'remote' } }, + }, + export_retained_rules = opts.export_retained_rules or { + { local_prefix = { 'retained' }, remote_prefix = { 'remote_retained' } }, + }, + outbound_call_rules = opts.outbound_call_rules or { + { local_topic = { 'cap', 'test-local', 'main', 'rpc', 'echo' }, remote_topic = { 'cap', 'test-remote', 'main', 'rpc', 'echo' } }, + }, + inbound_call_rules = opts.inbound_call_rules or { + { remote_topic = { 'cap', 'test-remote', 'main', 'rpc', 'echo' }, local_topic = { 'cap', 'test-local', 'main', 'rpc', 'echo' } }, + }, + call_timeout_s = opts.call_timeout_s or 0.05, + } + + local ok, err = scope:spawn(function () + local result = bridge.run(scope, params) + queue.try_admit_required(done_tx, result, 'bridge_done') + end) + assert_true(ok, err) + + return { + local_tx = local_tx, + session_tx = session_in_tx, + bus_rx = bus_rx, + state_rx = state_rx, + outbound = outbound, + rpc_rx = outbound_rpc_rx, + control_rx = outbound_control_rx, + bulk_rx = outbound_bulk_rx, + done_rx = done_rx, + } +end + +local function close_bridge(h) + h.local_tx:close('test complete') + h.session_tx:close('test complete') + return recv_with_timeout(h.done_rx, 'bridge done') +end + +function tests.test_local_publish_is_mapped_and_sent_through_session_gate() + fibers.run(function (scope) + local h = start_bridge(scope) + assert_true(h.session_tx:send(peer_session_event())) + local state = recv_with_timeout(h.state_rx, 'initial state') + assert_eq(state.kind, 'component_snapshot') + assert_true(h.local_tx:send({ kind = 'publish', topic = { 'local', 'a' }, payload = { v = 1 } })) + local item = recv_with_timeout(h.rpc_rx, 'outbound pub') + assert_eq(item.kind, 'send_frame') + assert_eq(item.frame.type, 'pub') + assert_eq(item.frame.topic[1], 'remote') + assert_eq(item.frame.topic[2], 'a') + assert_eq(item.session.peer_sid, 'sid-1') + close_bridge(h) + end) +end + +function tests.test_remote_retained_publish_emits_bus_command_and_updates_import_count() + fibers.run(function (scope) + local h = start_bridge(scope) + assert_true(h.session_tx:send(peer_session_event())) + assert_true(h.session_tx:send(rpc_event(assert(protocol.pub({ 'state', 'self', 'software' }, { image_id = 'i1' }, true))))) + local cmd = recv_with_timeout(h.bus_rx, 'retain command') + assert_eq(cmd.kind, 'retain') + assert_eq(cmd.topic[1], 'raw') + assert_eq(cmd.topic[4], 'state') + assert_eq(cmd.topic[5], 'software') + assert_eq(cmd.session.peer_sid, 'sid-1') + close_bridge(h) + end) +end + +function tests.test_remote_event_publish_maps_to_telemetry_cap_event_topic() + fibers.run(function (scope) + local h = start_bridge(scope) + assert_true(h.session_tx:send(peer_session_event())) + assert_true(h.session_tx:send(rpc_event(assert(protocol.pub( + { 'event', 'self', 'power', 'charger', 'alert' }, + { kind = 'vin_lo' }, + false + ))))) + local cmd = recv_with_timeout(h.bus_rx, 'event publish command') + assert_eq(cmd.kind, 'publish') + assert_eq(cmd.topic[1], 'raw') + assert_eq(cmd.topic[2], 'member') + assert_eq(cmd.topic[3], 'mcu') + assert_eq(cmd.topic[4], 'cap') + assert_eq(cmd.topic[5], 'telemetry') + assert_eq(cmd.topic[6], 'main') + assert_eq(cmd.topic[7], 'event') + assert_eq(cmd.topic[8], 'power') + assert_eq(cmd.topic[9], 'charger') + assert_eq(cmd.topic[10], 'alert') + assert_eq(cmd.payload.kind, 'vin_lo') + close_bridge(h) + end) +end + +function tests.test_peer_session_change_clears_imported_retained_state_by_bus_command() + fibers.run(function (scope) + local h = start_bridge(scope) + assert_true(h.session_tx:send(peer_session_event(1, 'sid-1'))) + assert_true(h.session_tx:send(rpc_event(assert(protocol.pub({ 'state', 'self', 'updater' }, { state = 'ready' }, true)), 1, 'sid-1'))) + recv_with_timeout(h.bus_rx, 'retain command') + assert_true(h.session_tx:send(peer_session_event(2, 'sid-2'))) + local cmd = recv_with_timeout(h.bus_rx, 'clear command') + assert_eq(cmd.kind, 'unretain') + assert_eq(cmd.topic[5], 'updater') + close_bridge(h) + end) +end + +function tests.test_peer_session_drop_clears_imported_retained_state_by_bus_command() + fibers.run(function (scope) + local h = start_bridge(scope) + assert_true(h.session_tx:send(peer_session_event(1, 'sid-1'))) + assert_true(h.session_tx:send(rpc_event(assert(protocol.pub({ 'state', 'self', 'software' }, { image_id = 'i1' }, true)), 1, 'sid-1'))) + recv_with_timeout(h.bus_rx, 'retain command') + assert_true(h.session_tx:send(peer_drop_event(1, 'sid-1', 'bad_frame_limit'))) + local cmd = recv_with_timeout(h.bus_rx, 'clear command') + assert_eq(cmd.kind, 'unretain') + assert_eq(cmd.topic[5], 'software') + assert_eq(cmd.session.peer_sid, 'sid-1') + close_bridge(h) + end) +end + +function tests.test_stale_peer_drop_does_not_clear_current_imported_state() + fibers.run(function (scope) + local h = start_bridge(scope) + assert_true(h.session_tx:send(peer_session_event(1, 'sid-1'))) + assert_true(h.session_tx:send(rpc_event(assert(protocol.pub({ 'state', 'self', 'software' }, { image_id = 'i1' }, true)), 1, 'sid-1'))) + recv_with_timeout(h.bus_rx, 'retain command') + assert_true(h.session_tx:send(peer_session_event(2, 'sid-2'))) + recv_with_timeout(h.bus_rx, 'clear command') + assert_true(h.session_tx:send(rpc_event(assert(protocol.pub({ 'state', 'self', 'software' }, { image_id = 'i2' }, true)), 2, 'sid-2'))) + recv_with_timeout(h.bus_rx, 'retain command 2') + assert_true(h.session_tx:send(peer_drop_event(1, 'sid-1', 'old_drop'))) + local stale = try_recv(h.bus_rx) + assert_nil(stale) + close_bridge(h) + end) +end + +function tests.test_inbound_call_is_routed_to_bus_command_and_reply_frame_is_sent() + fibers.run(function (scope) + local h = start_bridge(scope) + assert_true(h.session_tx:send(peer_session_event())) + assert_true(h.session_tx:send(rpc_event(assert(protocol.call('call-1', { 'cap', 'test-remote', 'main', 'rpc', 'echo' }, { n = 1 }))))) + local cmd = recv_with_timeout(h.bus_rx, 'bus call command') + assert_eq(cmd.kind, 'call') + assert_eq(cmd.topic[1], 'cap') + assert_eq(cmd.topic[2], 'test-local') + assert_eq(cmd.topic[4], 'rpc') + queue.try_admit_required(cmd.reply_tx, { ok = true, payload = { answer = 42 } }, 'call reply') + local frame = recv_with_timeout(h.rpc_rx, 'reply frame').frame + assert_eq(frame.type, 'reply') + assert_eq(frame.id, 'call-1') + assert_eq(frame.ok, true) + assert_eq(frame.payload.answer, 42) + close_bridge(h) + end) +end + +function tests.test_outbound_call_sends_call_frame_and_routes_remote_reply_to_request() + fibers.run(function (scope) + local h = start_bridge(scope) + assert_true(h.session_tx:send(peer_session_event())) + local req = fake_request() + assert_true(h.local_tx:send({ kind = 'call', id = 'local-1', topic = { 'cap', 'test-local', 'main', 'rpc', 'echo' }, payload = { q = true }, request = req })) + local frame = recv_with_timeout(h.rpc_rx, 'outbound call frame').frame + assert_eq(frame.type, 'call') + assert_eq(frame.id, 'local-1') + assert_eq(frame.topic[1], 'cap') + assert_eq(frame.topic[2], 'test-remote') + assert_eq(frame.topic[4], 'rpc') + assert_true(h.session_tx:send(rpc_event(assert(protocol.reply('local-1', true, { ok = 'yes' }, nil))))) + local deadline = fibers.now() + 0.25 + while req.resolved == false and fibers.now() < deadline do fibers.perform(sleep.sleep_op(0.001)) end + assert_eq(req.resolved, 'reply') + assert_eq(req.value.ok, 'yes') + close_bridge(h) + end) +end + +function tests.test_bus_adapter_remote_commands_use_local_bus_methods() + fibers.run(function (scope) + local command_tx, command_rx = mailbox.new(8, { full = 'reject_newest' }) + local calls = {} + local conn = {} + function conn:publish(topic, payload, opts) calls[#calls + 1] = { kind = 'publish', topic = topic, payload = payload, opts = opts }; return true end + function conn:retain(topic, payload, opts) calls[#calls + 1] = { kind = 'retain', topic = topic, payload = payload, opts = opts }; return true end + function conn:unretain(topic, opts) calls[#calls + 1] = { kind = 'unretain', topic = topic, opts = opts }; return true end + function conn:call_op(topic, payload, opts) calls[#calls + 1] = { kind = 'call', topic = topic, payload = payload, opts = opts }; return fibers.always({ ok = true }, nil) end + + local ok, err = scope:spawn(function () + -- Use the public runtime with no subscriptions and drive its command queue. + local rt = bus_adapter.local_runtime(scope, conn, { link_id = 'link-a', link_generation = 1 }) + command_tx, command_rx = rt.command_tx, nil + fibers.perform(sleep.sleep_op(0.03)) + rt:terminate('done') + end) + assert_true(ok, err) + fibers.perform(sleep.sleep_op(0.001)) + assert_true(command_tx:send({ kind = 'retain', topic = { 'raw' }, payload = { v = 1 }, session = ctx(), origin_kind = 'remote_retain' })) + local deadline = fibers.now() + 0.1 + while #calls == 0 and fibers.now() < deadline do fibers.perform(sleep.sleep_op(0.001)) end + assert_eq(calls[1].kind, 'retain') + assert_eq(calls[1].opts.extra.fabric.session.peer_sid, 'sid-1') + end) +end + + +function tests.test_bridge_local_outbound_calls_propagate_bus_request_abandonment() + local f = assert(io.open('../src/services/fabric/bridge.lua', 'r')) + local src = f:read('*a'); f:close() + if not src:find('cancel_op = cancel_op', 1, true) then fail('bridge scoped work wrapper should accept cancel_op') end + if not src:find('owner:caller_cancel_op()', 1, true) then fail('outbound local bus calls should pass caller_cancel_op') end +end + +return tests diff --git a/tests/unit/fabric/test_config.lua b/tests/unit/fabric/test_config.lua new file mode 100644 index 00000000..36d2be71 --- /dev/null +++ b/tests/unit/fabric/test_config.lua @@ -0,0 +1,262 @@ +local cfg = require 'services.fabric.config' + +local T = {} + +function T.compile_builds_canonical_runtime_plan() + local compiled, err = cfg.compile({ + schema = 'devicecode.config/fabric/1', + local_node = 'host-a', + links = { + { + id = 'uart0', + peer_id = 'peer-main', + transport = { + kind = 'uart', + source = 'uart', + id = 'main', + terminator = '\n', + open_opts = { baud = 115200 }, + }, + session = { + hello_interval_s = 1.5, + identity_claim = { id = 'host-a' }, + auth_claim = nil, + }, + bridge = { + max_pending_calls = 9, + max_inbound_calls = 7, + call_timeout_s = 4.5, + imports = { + { + id = 'import-state', + ['local'] = { 'raw', 'member', 'peer', 'state' }, + ['remote'] = { 'state' }, + }, + }, + exports = { + { + id = 'export-cfg', + ['local'] = { 'cfg', 'member', 'peer' }, + ['remote'] = { 'cfg' }, + publish = true, + retain = true, + }, + }, + rpc = { + outbound = { + { + id = 'cap-out', + ['local'] = { 'cap', 'component', 'peer' }, + ['remote'] = { 'cap', 'component', 'peer' }, + timeout_s = 2.0, + }, + }, + inbound = { + { + id = 'diag-in', + ['local'] = { 'cap', 'diag' }, + ['remote'] = { 'cap', 'diag' }, + }, + }, + }, + }, + transfer = { + chunk_size = 8192, + timeout_s = 22.0, + }, + }, + }, + }) + + assert(compiled ~= nil, tostring(err)) + assert(compiled.service.local_node == 'host-a') + assert(#compiled.links == 1) + + local link = compiled.links[1] + assert(link.link_id == 'uart0') + assert(link.peer_id == 'peer-main') + + assert(link.transport.kind == 'uart') + assert(link.transport.class == 'uart') + assert(link.transport.id == 'main') + assert(link.transport.open_opts.baud == 115200) + + assert(link.session.local_node == 'host-a') + assert(link.session.hello_interval_s == 1.5) + assert(link.session.ping_interval_s == cfg.DEFAULTS.session.ping_interval_s) + assert(link.session.identity_claim.id == 'host-a') + + assert(#link.bridge.import_rules == 1) + assert(#link.bridge.export_publish_rules == 1) + assert(#link.bridge.export_retained_rules == 1) + assert(#link.bridge.outbound_call_rules == 1) + assert(#link.bridge.inbound_call_rules == 1) + assert(link.bridge.outbound_call_rules[1].local_topic[1] == 'cap') + assert(link.bridge.outbound_call_rules[1].remote_topic[1] == 'cap') + assert(link.bridge.outbound_call_rules[1].local_prefix == nil) + assert(link.bridge.inbound_call_rules[1].local_topic[2] == 'diag') + assert(link.bridge.max_pending_calls == 9) + assert(link.bridge.max_inbound_calls == 7) + assert(link.bridge.call_timeout_s == 4.5) + + assert(link.transfer.chunk_size == 8192) + assert(link.transfer.timeout_s == 22.0) + + assert(compiled.routing.by_link_id['uart0'] == link) + assert(compiled.routing.by_peer_id['peer-main'][1] == link) +end + +function T.compile_applies_defaults() + local compiled, err = cfg.compile({ + schema = 'devicecode.config/fabric/1', + links = { + { + id = 'uart0', + peer_id = 'peer-a', + bridge = {}, + transfer = {}, + }, + }, + }) + + assert(compiled ~= nil, tostring(err)) + local link = compiled.links[1] + assert(link.transport.kind == 'uart') + assert(link.transport.source == 'uart') + assert(link.transport.class == 'uart') + assert(link.transport.id == 'uart0') + + assert(link.session.hello_interval_s == cfg.DEFAULTS.session.hello_interval_s) + assert(link.session.ping_interval_s == cfg.DEFAULTS.session.ping_interval_s) + assert(link.session.liveness_timeout_s == cfg.DEFAULTS.session.liveness_timeout_s) + + assert(link.bridge.max_pending_calls == cfg.DEFAULTS.bridge.max_pending_calls) + assert(link.transfer.chunk_size == cfg.DEFAULTS.transfer.chunk_size) + assert(link.transfer.timeout_s == cfg.DEFAULTS.transfer.timeout_s) +end + +function T.compile_rejects_wrong_schema() + local compiled, err = cfg.compile({ + schema = 'nope', + links = {}, + }) + assert(compiled == nil) + assert(tostring(err):match('schema')) +end + +function T.compile_rejects_bridge_connections_compat_path() + local compiled, err = cfg.compile({ + schema = 'devicecode.config/fabric/1', + links = { + { + id = 'uart0', + peer_id = 'peer-a', + bridge = { + connections = { + { direction = 'inout' }, + }, + }, + }, + }, + }) + assert(compiled == nil) + assert(tostring(err):match('bridge has unknown field: connections')) +end + +function T.compile_rejects_duplicate_link_ids_but_allows_shared_peer_id() + local compiled1, err1 = cfg.compile({ + schema = 'devicecode.config/fabric/1', + links = { + { id = 'uart0', peer_id = 'peer-a' }, + { id = 'uart0', peer_id = 'peer-b' }, + }, + }) + assert(compiled1 == nil) + assert(tostring(err1):match('duplicate link id')) + + local compiled2, err2 = cfg.compile({ + schema = 'devicecode.config/fabric/1', + links = { + { id = 'uart0', peer_id = 'peer-a' }, + { id = 'uart1', peer_id = 'peer-a' }, + }, + }) + assert(compiled2 ~= nil, tostring(err2)) + assert(#compiled2.routing.by_peer_id['peer-a'] == 2) +end + +function T.compile_rejects_transfer_targets_as_application_routing() + local compiled, err = cfg.compile({ + schema = 'devicecode.config/fabric/1', + links = { + { + id = 'uart0', + peer_id = 'peer-a', + transfer = { + targets = { + { id = 't1', remote_target = 'staging.a' }, + }, + }, + }, + }, + }) + assert(compiled == nil) + assert(tostring(err):match('transfer has unknown field: targets')) +end + +function T.compile_rejects_invalid_topics_and_shapes() + local compiled, err = cfg.compile({ + schema = 'devicecode.config/fabric/1', + links = { + { + id = 'uart0', + peer_id = 'peer-a', + bridge = { + imports = { + { ['local'] = { 'ok', bad = true }, ['remote'] = { 'state' } }, + }, + }, + }, + }, + }) + assert(compiled == nil) + assert(tostring(err):match('dense topic array')) +end + +function T.compile_rejects_rpc_topic_alias_and_requires_exact_local_remote_topics() + local compiled, err = cfg.compile({ + schema = 'devicecode.config/fabric/1', + links = { + { + id = 'uart0', + peer_id = 'peer-a', + bridge = { + rpc = { + outbound = { + { + topic = { 'cap', 'diag' }, + ['local'] = { 'cap', 'diag' }, + ['remote'] = { 'cap', 'diag' }, + }, + }, + }, + }, + }, + }, + }) + assert(compiled == nil) + assert(tostring(err):match('topic is not supported for rpc rules')) +end + +function T.compile_rejects_profile_field() + local compiled, err = cfg.compile({ + schema = 'devicecode.config/fabric/1', + links = { + { id = 'uart0', peer_id = 'peer-a', profile = 'legacy' }, + }, + }) + assert(compiled == nil) + assert(tostring(err):match('unknown field: profile')) +end + +return T diff --git a/tests/unit/fabric/test_dependencies.lua b/tests/unit/fabric/test_dependencies.lua new file mode 100644 index 00000000..32ad0d3d --- /dev/null +++ b/tests/unit/fabric/test_dependencies.lua @@ -0,0 +1,104 @@ +local deps = require 'services.fabric.dependencies' +local cfg = require 'services.fabric.config' + +local T = {} + +function T.derives_raw_host_transport_dependency_for_hal_link() + local compiled = assert(cfg.compile({ + schema = 'devicecode.config/fabric/1', + links = {{ id = 'uart0', peer_id = 'peer', transport = { source = 'uart', class = 'uart', id = 'main' }, bridge = {}, transfer = {} }}, + })) + local specs = deps.transport_dependencies(compiled) + assert(#specs == 1) + assert(specs[1].key == 'transport:uart0') + assert(specs[1].raw_kind == 'host') + assert(specs[1].source == 'uart') + assert(specs[1].class == 'uart') + assert(specs[1].id == 'main') +end + +function T.override_transport_bypasses_dependency() + local compiled = assert(cfg.compile({ + schema = 'devicecode.config/fabric/1', + links = {{ id = 'uart0', peer_id = 'peer', transport = { source = 'uart', class = 'uart', id = 'main' }, bridge = {}, transfer = {} }}, + })) + local specs = deps.transport_dependencies(compiled, { uart0 = { open_transport_op = function () end } }) + assert(#specs == 0) +end + + +function T.accepts_only_canonical_dependency_failure_from_generation_failure() + local service = require 'services.fabric.service' + local state = { + generation_deps = { + dependency = function (_, key) + if key == 'transport:uart0' then return { key = key } end + return nil + end, + }, + } + local key, failure = service._test.generation_route_failure(state, { + primary = { + kind = 'dependency_failure', + err = 'no_route', + dependency_key = 'transport:uart0', + }, + }) + assert(key == 'transport:uart0') + assert(failure.dependency_key == 'transport:uart0') +end + +function T.rejects_legacy_or_nested_dependency_failure_shapes() + local service = require 'services.fabric.service' + local state = { + generation_deps = { + dependency = function (_, key) + if key == 'transport:uart0' then return { key = key } end + return nil + end, + }, + } + local key = service._test.generation_route_failure(state, { + primary = 'link uart0 failed: no_route [dependency_key=transport:uart0]', + }) + assert(key == nil) + key = service._test.generation_route_failure(state, { + report = { children = { { primary = { dependency_key = 'transport:uart0', err = 'no_route' } } } }, + }) + assert(key == nil) +end + + +function T.hal_transport_open_errors_preserve_dependency_key() + local hal_transport = require 'services.fabric.hal_transport' + local _, err = hal_transport.unwrap_open_transport_reply({ + dependency_key = 'transport:uart0', + source = 'uart', + class = 'uart', + id = 'main', + }, nil, 'no_route') + assert(type(err) == 'table') + assert(err.kind == 'dependency_failure') + assert(err.err == 'no_route') + assert(err.dependency_key == 'transport:uart0') + assert(err.source == 'uart') +end + +function T.default_policy_preserves_structured_dependency_key() + local service = require 'services.fabric.service' + local decision = service.default_policy(nil, { + kind = 'link_done', + status = 'failed', + link_id = 'uart0', + primary = { kind = 'dependency_failure', err = 'no_route', dependency_key = 'transport:uart0' }, + }) + assert(decision.action == 'fail') + assert(type(decision.reason) == 'table') + assert(decision.reason.kind == 'dependency_failure') + assert(decision.reason.err == 'no_route') + assert(decision.reason.dependency_key == 'transport:uart0') + assert(decision.reason.link_id == 'uart0') +end + + +return T diff --git a/tests/unit/fabric/test_fabric.lua b/tests/unit/fabric/test_fabric.lua new file mode 100644 index 00000000..dd8fe270 --- /dev/null +++ b/tests/unit/fabric/test_fabric.lua @@ -0,0 +1,160 @@ +-- tests/unit/fabric/test_fabric.lua + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' + +local fabric = require 'services.fabric' +local protocol = require 'services.fabric.protocol' +local hal_transport = require 'services.fabric.hal_transport' +local queue = require 'devicecode.support.queue' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 3) end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil value') end end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_match(s, pat, msg) if type(s) ~= 'string' or not s:match(pat) then fail(msg or ('expected "' .. tostring(s) .. '" to match ' .. tostring(pat))) end end + +local function closed_rx(reason) + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + tx:close(reason or 'closed') + return rx +end + +function tests.test_public_entrypoint_exports_semantic_modules() + assert_eq(fabric.service, require 'services.fabric.service') + assert_eq(fabric.link, require 'services.fabric.link') + assert_eq(fabric.io, require 'services.fabric.io') + assert_eq(fabric.bridge, require 'services.fabric.bridge') + assert_eq(fabric.bus_adapter, require 'services.fabric.bus_adapter') + assert_eq(fabric.session, require 'services.fabric.session') + assert_eq(fabric.transfer, require 'services.fabric.transfer') + assert_eq(fabric.transfer_client, require 'services.fabric.transfer_client') + assert_eq(fabric.transfer_sender, require 'services.fabric.transfer_sender') + assert_eq(fabric.transfer_receive, require 'services.fabric.transfer_receive') + assert_eq(fabric.state, require 'services.fabric.state') + assert_eq(type(fabric.start), 'function') + assert_eq(type(fabric.run), 'function') + assert_eq(type(fabric.run_link), 'function') +end + + +function tests.test_fabric_service_components_report_through_event_ports() + local function read(path) + local f = assert(io.open(path, 'r')) + local text = f:read('*a') + f:close() + return text + end + + local service = read('../src/services/fabric/service.lua') + assert_true(service:find('devicecode.support.service_events', 1, true) ~= nil) + assert_true(service:find('service_events.reporter_for', 1, true) ~= nil) + assert_true(service:find('snapshot = public_snapshot(self)', 1, true) ~= nil) + assert_true(service:find('snapshot = function', 1, true) == nil) + + local link = read('../src/services/fabric/link.lua') + assert_true(link:find('devicecode.support.service_events', 1, true) ~= nil) + assert_true(link:find('service_events.reporter_for', 1, true) ~= nil) + assert_true(link:find('snapshot = public_snapshot(self)', 1, true) ~= nil) + assert_true(link:find('snapshot = function', 1, true) == nil) +end + +function tests.test_composed_link_wires_session_bridge_transfer_and_writer_without_callbacks() + fibers.run(function () + local local_tx, local_rx = mailbox.new(8, { full = 'reject_newest' }) + local written = {} + local read_count = 0 + + local inbound = { assert(protocol.hello_ack('peer-sid', 'peer-a')) } + local transport = { + read_line_op = function () + read_count = read_count + 1 + local frame = inbound[read_count] + if frame == nil then return fibers.always(nil, 'eof') end + return fibers.always(assert(protocol.encode_line(frame)), nil) + end, + write_line_op = function (_, line) + written[#written + 1] = assert(protocol.decode_line(line)) + return fibers.always(true, nil) + end, + flush_op = function () return fibers.always(true, nil) end, + terminate = function () return true, nil end, + } + + fibers.spawn(function () + assert_true(local_tx:send({ kind = 'publish', topic = { 'local', 'state' }, payload = { ok = true }, retain = true })) + local_tx:close('local done') + end) + + local st, rep, result = fibers.run_scope(function (scope) + return fabric.run_link(scope, { + link_id = 'link-composed', + session = { local_node = 'host-a' }, + open_transport_op = function () + local wrapped, err = hal_transport.wrap_transport(transport) + return fibers.always(wrapped, err) + end, + local_rx = local_rx, + transfer_admission_rx = closed_rx('no transfers'), + bridge = { + export_publish_rules = { + { local_prefix = { 'local' }, remote_prefix = { 'remote' } }, + }, + }, + }) + end) + + assert_eq(st, 'ok') + assert_eq(#rep.extra_errors, 0) + assert_not_nil(result) + assert_eq(result.snapshot.state, 'completed') + assert_eq(result.snapshot.components.reader.status, 'ok') + assert_eq(result.snapshot.components.session.status, 'ok') + assert_eq(result.snapshot.components.writer.status, 'ok') + assert_eq(result.snapshot.components.rpc_bridge.status, 'ok') + assert_eq(result.snapshot.components.transfer_manager.status, 'ok') + assert_eq(written[1].type, 'hello') + assert_eq(written[1].node, 'host-a') + assert_eq(written[2].type, 'pub') + assert_eq(written[2].topic[1], 'remote') + end) +end + +function tests.test_public_fabric_run_uses_supplied_link_runner() + fibers.run(function () + local called = false + local st, _, result = fibers.run_scope(function (scope) + return fabric.run(scope, { + links = { { link_id = 'a', generation = 1 } }, + link_runner = function () + called = true + return { link_id = 'a', snapshot = { state = 'completed' } } + end, + }) + end) + assert_eq(st, 'ok') + assert_true(called) + assert_eq(result.snapshot.links.a.status, 'ok') + end) +end + +function tests.test_composed_link_transport_open_failure_fails_link_scope_before_components_start() + fibers.run(function () + local st, _, primary = fibers.run_scope(function (scope) + return fabric.run_link(scope, { + link_id = 'bad-open', + open_transport_op = function () return fibers.always(nil, 'open failed') end, + }) + end) + assert_eq(st, 'failed') + if type(primary) == 'table' then + assert_eq(primary.err, 'open failed') + else + assert_match(primary, 'open failed') + end + end) +end + +return tests diff --git a/tests/unit/fabric/test_io.lua b/tests/unit/fabric/test_io.lua new file mode 100644 index 00000000..77409965 --- /dev/null +++ b/tests/unit/fabric/test_io.lua @@ -0,0 +1,391 @@ +-- tests/unit/fabric/test_io.lua + +local fibers = require 'fibers' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' + +local io_mod = require 'services.fabric.io' +local link = require 'services.fabric.link' +local queue = require 'devicecode.support.queue' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_true(v, msg) + if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end +end + +local function assert_nil(v, msg) + if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end +end + +local function assert_not_nil(v, msg) + if v == nil then fail(msg or 'expected non-nil value') end +end + +local function assert_eq(a, b, msg) + if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end +end + +local function assert_match(s, pat, msg) + if type(s) ~= 'string' or not s:match(pat) then + fail(msg or ('expected "' .. tostring(s) .. '" to match ' .. tostring(pat))) + end +end + +local function frames_reader(frames) + local i = 0 + + return function () + return op.guard(function () + i = i + 1 + + local frame = frames[i] + if frame == nil then + return op.always(nil, 'eof') + end + + return op.always(frame, nil) + end) + end +end + +local function closed_rx(reason) + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + tx:close(reason or 'closed') + return rx +end + +local function send_frame(tx, frame, label) + return queue.try_admit_required(tx, { + kind = 'send_frame', + frame = frame, + }, label or 'send_frame') +end + +------------------------------------------------------------------------------- +-- Reader forwards frames and returns a result table +------------------------------------------------------------------------------- + +function tests.test_reader_forwards_frames_and_returns_count() + fibers.run(function (scope) + local tx, rx = mailbox.new(8, { full = 'reject_newest' }) + + local result = io_mod.run_reader(scope, { + read_frame_op = frames_reader { 'a', 'b' }, + downstream_tx = tx, + }) + + assert_eq(result.role, 'reader') + assert_eq(result.frames_read, 2) + assert_eq(result.reason, 'eof') + + local ev1 = fibers.perform(rx:recv_op()) + local ev2 = fibers.perform(rx:recv_op()) + + assert_eq(ev1.kind, 'frame_received') + assert_eq(ev1.frame, 'a') + + assert_eq(ev2.kind, 'frame_received') + assert_eq(ev2.frame, 'b') + end) +end + +------------------------------------------------------------------------------- +-- Reader may deliberately block on downstream handoff +------------------------------------------------------------------------------- + +function tests.test_reader_blocks_on_downstream_handoff() + fibers.run(function (scope) + local downstream_tx, downstream_rx = mailbox.new(0, { full = 'block' }) + local done_tx, done_rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = scope:spawn(function () + local result = io_mod.run_reader(scope, { + read_frame_op = frames_reader { 'blocked-frame' }, + downstream_tx = downstream_tx, + }) + + queue.try_admit_required(done_tx, result, 'reader_done') + end) + + assert_true(ok, err) + + fibers.perform(sleep.sleep_op(0.001)) + + local done_now, done_err = queue.try_recv_now(done_rx) + assert_nil(done_now) + assert_eq(done_err, 'not_ready') + + local ev = fibers.perform(downstream_rx:recv_op()) + assert_not_nil(ev) + assert_eq(ev.kind, 'frame_received') + assert_eq(ev.frame, 'blocked-frame') + + local result = fibers.perform(done_rx:recv_op()) + assert_eq(result.role, 'reader') + assert_eq(result.frames_read, 1) + assert_eq(result.reason, 'eof') + end) +end + +------------------------------------------------------------------------------- +-- Reader read errors fail the owning scope +------------------------------------------------------------------------------- + +function tests.test_reader_read_error_fails_scope() + fibers.run(function () + local tx = mailbox.new(4, { full = 'reject_newest' }) + + local st, _, primary = fibers.run_scope(function (scope) + return io_mod.run_reader(scope, { + read_frame_op = function () + return op.always(nil, 'transport exploded') + end, + downstream_tx = tx, + }) + end) + + assert_eq(st, 'failed') + assert_match(primary, 'reader read failed') + assert_match(primary, 'transport exploded') + end) +end + +------------------------------------------------------------------------------- +-- Lane writer writes queued frames and flushes on close +------------------------------------------------------------------------------- + +function tests.test_lane_writer_writes_frames_and_flushes_on_close() + fibers.run(function (scope) + local rpc_tx, rpc_rx = mailbox.new(8, { full = 'reject_newest' }) + local written = {} + local flushes = 0 + + send_frame(rpc_tx, 'a', 'rpc_a') + send_frame(rpc_tx, 'b', 'rpc_b') + rpc_tx:close('rpc_done') + + local result = io_mod.run_lane_writer(scope, { + control_rx = closed_rx('control_done'), + rpc_rx = rpc_rx, + bulk_rx = closed_rx('bulk_done'), + + write_frame_op = function (frame) + return op.guard(function () + written[#written + 1] = frame + return op.always(true, nil) + end) + end, + + flush_op = function () + return op.guard(function () + flushes = flushes + 1 + return op.always(true, nil) + end) + end, + }) + + assert_eq(result.role, 'writer') + assert_eq(result.frames_written, 2) + assert_eq(result.reason, 'closed') + assert_eq(result.lanes.rpc, 2) + + assert_eq(written[1], 'a') + assert_eq(written[2], 'b') + assert_eq(flushes, 1) + end) +end + +------------------------------------------------------------------------------- +-- Lane writer can flush each frame when asked +------------------------------------------------------------------------------- + +function tests.test_lane_writer_can_flush_each_frame() + fibers.run(function (scope) + local rpc_tx, rpc_rx = mailbox.new(8, { full = 'reject_newest' }) + local flushes = 0 + + send_frame(rpc_tx, 'a', 'rpc_a') + send_frame(rpc_tx, 'b', 'rpc_b') + rpc_tx:close('rpc_done') + + local result = io_mod.run_lane_writer(scope, { + control_rx = closed_rx('control_done'), + rpc_rx = rpc_rx, + bulk_rx = closed_rx('bulk_done'), + flush_each = true, + + write_frame_op = function () + return op.always(true, nil) + end, + + flush_op = function () + return op.guard(function () + flushes = flushes + 1 + return op.always(true, nil) + end) + end, + }) + + assert_eq(result.frames_written, 2) + assert_eq(flushes, 3) + end) +end + +------------------------------------------------------------------------------- +-- Lane writer write errors fail the owning scope +------------------------------------------------------------------------------- + +function tests.test_lane_writer_write_error_fails_scope() + fibers.run(function () + local rpc_tx, rpc_rx = mailbox.new(8, { full = 'reject_newest' }) + send_frame(rpc_tx, 'bad-frame', 'rpc_bad') + rpc_tx:close('rpc_done') + + local st, _, primary = fibers.run_scope(function (scope) + return io_mod.run_lane_writer(scope, { + control_rx = closed_rx('control_done'), + rpc_rx = rpc_rx, + bulk_rx = closed_rx('bulk_done'), + + write_frame_op = function () + return op.always(nil, 'write exploded') + end, + }) + end) + + assert_eq(st, 'failed') + assert_match(primary, 'writer write failed') + assert_match(primary, 'write exploded') + end) +end + +------------------------------------------------------------------------------- +-- Reader and lane writer owners fit the link supervisor +------------------------------------------------------------------------------- + +function tests.test_reader_and_writer_components_fit_link_supervisor() + fibers.run(function () + local frame_tx, frame_rx = mailbox.new(8, { full = 'reject_newest' }) + local rpc_tx, rpc_rx = mailbox.new(8, { full = 'reject_newest' }) + local written = {} + + send_frame(rpc_tx, 'out-a', 'outbound') + rpc_tx:close('rpc_done') + + local st, _, result = fibers.run_scope(function (scope) + return link.run(scope, { + link_id = 'link-io', + + components = { + { + name = 'reader', + run = function (component_scope) + return io_mod.run_reader(component_scope, { + read_frame_op = frames_reader { 'in-a' }, + downstream_tx = frame_tx, + }) + end, + }, + + { + name = 'writer', + run = function (component_scope) + return io_mod.run_lane_writer(component_scope, { + control_rx = closed_rx('control_done'), + rpc_rx = rpc_rx, + bulk_rx = closed_rx('bulk_done'), + + write_frame_op = function (frame) + return op.guard(function () + written[#written + 1] = frame + return op.always(true, nil) + end) + end, + }) + end, + }, + }, + }) + end) + + assert_eq(st, 'ok') + assert_not_nil(result) + + local snap = result.snapshot + assert_eq(snap.state, 'completed') + assert_eq(snap.components.reader.status, 'ok') + assert_eq(snap.components.writer.status, 'ok') + + assert_eq(snap.components.reader.result.frames_read, 1) + assert_eq(snap.components.writer.result.frames_written, 1) + assert_eq(written[1], 'out-a') + + local ev = fibers.perform(frame_rx:recv_op()) + assert_eq(ev.kind, 'frame_received') + assert_eq(ev.frame, 'in-a') + end) +end + +------------------------------------------------------------------------------- +-- Lane writer treats nil,nil write results as an invalid ambiguous failure +------------------------------------------------------------------------------- + +function tests.test_lane_writer_rejects_ambiguous_nil_nil_write_result() + fibers.run(function () + local rpc_tx, rpc_rx = mailbox.new(8, { full = 'reject_newest' }) + send_frame(rpc_tx, 'ambiguous-frame', 'rpc_ambiguous') + rpc_tx:close('rpc_done') + + local st, _, primary = fibers.run_scope(function (scope) + return io_mod.run_lane_writer(scope, { + control_rx = closed_rx('control_done'), + rpc_rx = rpc_rx, + bulk_rx = closed_rx('bulk_done'), + + write_frame_op = function () + return op.always() + end, + }) + end) + + assert_eq(st, 'failed') + assert_match(primary, 'writer write failed') + assert_match(primary, 'write_failed') + end) +end + +------------------------------------------------------------------------------- +-- Lane writer treats nil,nil flush results as an invalid ambiguous failure +------------------------------------------------------------------------------- + +function tests.test_lane_writer_rejects_ambiguous_nil_nil_flush_result() + fibers.run(function () + local st, _, primary = fibers.run_scope(function (scope) + return io_mod.run_lane_writer(scope, { + control_rx = closed_rx('control_done'), + rpc_rx = closed_rx('rpc_done'), + bulk_rx = closed_rx('bulk_done'), + + write_frame_op = function () + return op.always(true, nil) + end, + + flush_op = function () + return op.always() + end, + }) + end) + + assert_eq(st, 'failed') + assert_match(primary, 'writer flush failed') + assert_match(primary, 'flush_failed') + end) +end + +return tests diff --git a/tests/unit/fabric/test_link.lua b/tests/unit/fabric/test_link.lua new file mode 100644 index 00000000..55aff4a0 --- /dev/null +++ b/tests/unit/fabric/test_link.lua @@ -0,0 +1,780 @@ +-- tests/services/fabric/test_link.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' + +local protocol = require 'services.fabric.protocol' +local queue = require 'devicecode.support.queue' + +local link = require 'services.fabric.link' +local session = require 'services.fabric.session' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_true(v, msg) + if v ~= true then + fail(msg or ('expected true, got ' .. tostring(v))) + end +end + +local function assert_nil(v, msg) + if v ~= nil then + fail(msg or ('expected nil, got ' .. tostring(v))) + end +end + +local function assert_not_nil(v, msg) + if v == nil then + fail(msg or 'expected non-nil value') + end +end + +local function assert_eq(a, b, msg) + if a ~= b then + fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) + end +end + +local function assert_match(s, pat, msg) + if type(s) ~= 'string' or not s:match(pat) then + fail(msg or ('expected "' .. tostring(s) .. '" to match ' .. tostring(pat))) + end +end + + +local function make_outbound_gate(outbound_tx) + return session.new_outbound_gate { + tx_control = outbound_tx, + tx_rpc = outbound_tx, + tx_bulk = outbound_tx, + } +end + +local function run_link_in_child(root_scope, params) + return fibers.run_scope(function (scope) + return link.run(scope, params) + end) +end + + +------------------------------------------------------------------------------- +-- Session control establishes on hello and replies with hello_ack +------------------------------------------------------------------------------- + +function tests.test_session_control_establishes_from_hello_and_sends_ack() + fibers.run(function (scope) + local control_tx, control_rx = mailbox.new(8, { full = 'reject_newest' }) + local out_tx, out_rx = mailbox.new(8, { full = 'reject_newest' }) + local rpc_tx, _rpc_rx = mailbox.new(8, { full = 'reject_newest' }) + local transfer_tx, _transfer_rx = mailbox.new(8, { full = 'reject_newest' }) + local done_tx, done_rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = scope:spawn(function () + local result = session.run(scope, { + link_id = 'link-session', + peer_id = 'peer-a', + local_node = 'local-a', + local_sid = 'local-sid', + frame_rx = control_rx, + tx_control = out_tx, + outbound = make_outbound_gate(out_tx), + rpc_tx = rpc_tx, + transfer_tx = transfer_tx, + hello_interval_s = 10, + ping_interval_s = 10, + liveness_timeout_s = 20, + }) + queue.try_admit_required(done_tx, result, 'session_done') + end) + assert_true(ok, err) + + local initial = fibers.perform(out_rx:recv_op()) + assert_eq(initial.frame.type, 'hello') + assert_eq(initial.frame.sid, 'local-sid') + assert_eq(initial.frame.node, 'local-a') + + queue.try_admit_required(control_tx, { + kind = 'frame_received', + frame = assert(protocol.hello('peer-sid', 'peer-node')), + }, 'remote_hello') + + local ack = fibers.perform(out_rx:recv_op()) + assert_eq(ack.frame.type, 'hello_ack') + assert_eq(ack.frame.sid, 'local-sid') + + control_tx:close('test complete') + + local result = fibers.perform(done_rx:recv_op()) + assert_eq(result.role, 'session') + assert_eq(result.snapshot.established, true) + assert_eq(result.snapshot.peer_sid, 'peer-sid') + assert_eq(result.snapshot.peer_node, 'peer-node') + assert_eq(result.snapshot.link_generation, 1) + end) +end + +------------------------------------------------------------------------------- +-- Session liveness resets to hello after missing peer traffic +------------------------------------------------------------------------------- + +function tests.test_session_liveness_timeout_resets_to_hello() + fibers.run(function (scope) + local control_tx, control_rx = mailbox.new(8, { full = 'reject_newest' }) + local out_tx, out_rx = mailbox.new(8, { full = 'reject_newest' }) + local rpc_tx, session_event_rx = mailbox.new(8, { full = 'reject_newest' }) + local transfer_tx, transfer_event_rx = mailbox.new(8, { full = 'reject_newest' }) + local done_tx, done_rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = scope:spawn(function () + local result = session.run(scope, { + link_id = 'link-liveness', + peer_id = 'peer-a', + local_sid = 'local-sid', + frame_rx = control_rx, + tx_control = out_tx, + outbound = make_outbound_gate(out_tx), + rpc_tx = rpc_tx, + transfer_tx = transfer_tx, + + -- This test is about the liveness reset, not ping emission. + -- Keep ping safely beyond the liveness deadline so scheduler + -- jitter cannot turn this into a ping/liveness ordering test. + hello_interval_s = 10, + ping_interval_s = 10, + liveness_timeout_s = 0.02, + }) + queue.try_admit_required(done_tx, result, 'session_done') + end) + assert_true(ok, err) + + assert_eq(fibers.perform(out_rx:recv_op()).frame.type, 'hello') + queue.try_admit_required(control_tx, { + kind = 'frame_received', + frame = assert(protocol.hello('peer-sid', 'peer-node')), + }, 'remote_hello') + assert_eq(fibers.perform(out_rx:recv_op()).frame.type, 'hello_ack') + + local established = fibers.perform(session_event_rx:recv_op()) + assert_eq(established.kind, 'peer_session') + assert_eq(established.session.peer_sid, 'peer-sid') + local transfer_established = fibers.perform(transfer_event_rx:recv_op()) + assert_eq(transfer_established.kind, 'peer_session') + assert_eq(transfer_established.session.peer_sid, 'peer-sid') + + local hello_after_timeout = fibers.perform(out_rx:recv_op()) + assert_eq(hello_after_timeout.frame.type, 'hello') + assert_eq(hello_after_timeout.frame.sid == 'local-sid', false) + + local dropped = fibers.perform(session_event_rx:recv_op()) + assert_eq(dropped.kind, 'peer_session_dropped') + assert_eq(dropped.session.peer_sid, 'peer-sid') + assert_eq(dropped.reason, 'liveness_timeout') + local transfer_dropped = fibers.perform(transfer_event_rx:recv_op()) + assert_eq(transfer_dropped.kind, 'peer_session_dropped') + assert_eq(transfer_dropped.session.peer_sid, 'peer-sid') + assert_eq(transfer_dropped.reason, 'liveness_timeout') + + control_tx:close('test complete') + local result = fibers.perform(done_rx:recv_op()) + assert_eq(result.snapshot.established, false) + assert_eq(result.snapshot.phase, 'hello') + assert_eq(result.snapshot.why, 'liveness_timeout') + end) +end + +function tests.test_session_ping_is_emitted_before_liveness_deadline() + fibers.run(function (scope) + local control_tx, control_rx = mailbox.new(8, { full = 'reject_newest' }) + local out_tx, out_rx = mailbox.new(8, { full = 'reject_newest' }) + local rpc_tx, session_event_rx = mailbox.new(8, { full = 'reject_newest' }) + local transfer_tx, transfer_event_rx = mailbox.new(8, { full = 'reject_newest' }) + local done_tx, done_rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = scope:spawn(function () + local result = session.run(scope, { + link_id = 'link-ping', + peer_id = 'peer-a', + local_sid = 'local-sid', + frame_rx = control_rx, + tx_control = out_tx, + outbound = make_outbound_gate(out_tx), + rpc_tx = rpc_tx, + transfer_tx = transfer_tx, + hello_interval_s = 10, + ping_interval_s = 0.01, + liveness_timeout_s = 1.0, + }) + queue.try_admit_required(done_tx, result, 'session_done') + end) + assert_true(ok, err) + + assert_eq(fibers.perform(out_rx:recv_op()).frame.type, 'hello') + queue.try_admit_required(control_tx, { + kind = 'frame_received', + frame = assert(protocol.hello('peer-sid', 'peer-node')), + }, 'remote_hello') + assert_eq(fibers.perform(out_rx:recv_op()).frame.type, 'hello_ack') + + local established = fibers.perform(session_event_rx:recv_op()) + assert_eq(established.kind, 'peer_session') + local transfer_established = fibers.perform(transfer_event_rx:recv_op()) + assert_eq(transfer_established.kind, 'peer_session') + + local ping = fibers.perform(out_rx:recv_op()) + assert_eq(ping.frame.type, 'ping') + assert_eq(ping.frame.sid, 'local-sid') + + control_tx:close('test complete') + local result = fibers.perform(done_rx:recv_op()) + assert_eq(result.role, 'session') + end) +end + + +function tests.test_session_control_processes_ready_control_before_timer_work() + fibers.run(function (scope) + local control_tx, control_rx = mailbox.new(8, { full = 'reject_newest' }) + local out_tx, out_rx = mailbox.new(8, { full = 'reject_newest' }) + local rpc_tx, _rpc_rx = mailbox.new(8, { full = 'reject_newest' }) + local transfer_tx, _transfer_rx = mailbox.new(8, { full = 'reject_newest' }) + local done_tx, done_rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = scope:spawn(function () + local result = session.run(scope, { + link_id = 'link-control-before-timer', + peer_id = 'peer-a', + local_sid = 'local-sid', + frame_rx = control_rx, + tx_control = out_tx, + outbound = make_outbound_gate(out_tx), + rpc_tx = rpc_tx, + transfer_tx = transfer_tx, + hello_interval_s = 10, + ping_interval_s = 10, + liveness_timeout_s = 0.05, + }) + queue.try_admit_required(done_tx, result, 'session_done') + end) + assert_true(ok, err) + + assert_eq(fibers.perform(out_rx:recv_op()).frame.type, 'hello') + queue.try_admit_required(control_tx, { + kind = 'frame_received', + frame = assert(protocol.hello('peer-sid', 'peer-node')), + }, 'remote_hello') + assert_eq(fibers.perform(out_rx:recv_op()).frame.type, 'hello_ack') + + -- Move close to the original liveness deadline. If the ready ping is not + -- selected ahead of timer work, the session will reset when the old + -- liveness deadline fires. If the ping is processed first, the deadline is + -- refreshed and the session should still be established at shutdown. + fibers.perform(sleep.sleep_op(0.04)) + + queue.try_admit_required(control_tx, { + kind = 'frame_received', + frame = assert(protocol.ping('peer-sid')), + }, 'remote_ping') + assert_eq(fibers.perform(out_rx:recv_op()).frame.type, 'pong') + + -- Let the previous liveness deadline pass; the refreshed deadline should + -- still be in the future. + fibers.perform(sleep.sleep_op(0.015)) + control_tx:close('test complete') + local result = fibers.perform(done_rx:recv_op()) + assert_eq(result.snapshot.established, true) + assert_eq(result.snapshot.peer_sid, 'peer-sid') + end) +end + +------------------------------------------------------------------------------- +-- Successful components complete the link +------------------------------------------------------------------------------- + +function tests.test_successful_components_complete_link() + fibers.run(function (root_scope) + local st, rep, result = run_link_in_child(root_scope, { + link_id = 'link-a', + + components = { + { + name = 'reader', + run = function () + return { + role = 'reader', + } + end, + }, + + { + name = 'writer', + run = function () + return { + role = 'writer', + } + end, + }, + }, + }) + + assert_eq(st, 'ok') + assert_eq(#rep.extra_errors, 0) + assert_not_nil(result) + + local snap = result.snapshot + + assert_eq(snap.link_id, 'link-a') + assert_eq(snap.state, 'completed') + assert_eq(snap.completed, 2) + assert_eq(snap.total, 2) + + assert_eq(snap.components.reader.status, 'ok') + assert_eq(snap.components.reader.result.role, 'reader') + + assert_eq(snap.components.writer.status, 'ok') + assert_eq(snap.components.writer.result.role, 'writer') + end) +end + +------------------------------------------------------------------------------- +-- Component failure is interpreted by default policy +------------------------------------------------------------------------------- + +function tests.test_component_failure_fails_link_by_default_policy() + fibers.run(function (root_scope) + local st, rep, primary = run_link_in_child(root_scope, { + link_id = 'link-b', + + components = { + { + name = 'reader', + run = function () + error('reader exploded', 0) + end, + }, + + { + name = 'writer', + run = function () + fibers.perform(sleep.sleep_op(1)) + return { + role = 'writer', + } + end, + }, + }, + }) + + assert_eq(st, 'failed') + assert_match(primary, 'component reader failed') + assert_match(primary, 'reader exploded') + + assert_not_nil(rep) + end) +end + +------------------------------------------------------------------------------- +-- Component failure can be treated as data by explicit policy +------------------------------------------------------------------------------- + +function tests.test_component_failure_is_data_when_policy_allows_it() + fibers.run(function (root_scope) + local seen_failed = false + + local st, rep, result = run_link_in_child(root_scope, { + link_id = 'link-c', + + policy = function (_, ev) + if ev.kind == 'component_done' and ev.component == 'reader' and ev.status == 'failed' then + seen_failed = true + end + + return { + action = 'continue', + } + end, + + components = { + { + name = 'reader', + run = function () + error('reader failed as data', 0) + end, + }, + + { + name = 'writer', + run = function () + return { + role = 'writer', + } + end, + }, + }, + }) + + assert_eq(st, 'ok') + assert_eq(#rep.extra_errors, 0) + assert_true(seen_failed) + + local snap = result.snapshot + + assert_eq(snap.state, 'completed') + assert_eq(snap.completed, 2) + + assert_eq(snap.components.reader.status, 'failed') + assert_match(snap.components.reader.primary, 'reader failed as data') + + assert_eq(snap.components.writer.status, 'ok') + end) +end + +------------------------------------------------------------------------------- +-- Unexpected component cancellation fails the link by default +------------------------------------------------------------------------------- + +function tests.test_component_cancellation_fails_link_by_default_policy() + fibers.run(function (root_scope) + local st, _, primary = run_link_in_child(root_scope, { + link_id = 'link-d', + + components = { + { + name = 'session', + run = function (component_scope) + component_scope:cancel('session stopped') + fibers.perform(sleep.sleep_op(1)) + return { + unreachable = true, + } + end, + }, + }, + }) + + assert_eq(st, 'failed') + assert_match(primary, 'component session cancelled unexpectedly') + assert_match(primary, 'session stopped') + end) +end + +------------------------------------------------------------------------------- +-- Link component run must return one result table +------------------------------------------------------------------------------- + +function tests.test_component_must_return_result_table() + fibers.run(function (root_scope) + local st, _, primary = run_link_in_child(root_scope, { + link_id = 'link-e', + + components = { + { + name = 'bad_component', + run = function () + return 'not a table' + end, + }, + }, + }) + + assert_eq(st, 'failed') + assert_match(primary, 'component bad_component failed') + assert_match(primary, 'worker must return one result table') + end) +end + +------------------------------------------------------------------------------- +-- Link start validates duplicate component names before work begins +------------------------------------------------------------------------------- + +function tests.test_duplicate_component_names_are_rejected() + fibers.run(function (root_scope) + local st, _, primary = run_link_in_child(root_scope, { + link_id = 'link-f', + + components = { + { + name = 'reader', + run = function () + return {} + end, + }, + + { + name = 'reader', + run = function () + return {} + end, + }, + }, + }) + + assert_eq(st, 'failed') + assert_match(primary, 'duplicate component name') + end) +end + +------------------------------------------------------------------------------- +-- Link result does not expose a live model owned by the completed scope +------------------------------------------------------------------------------- + +function tests.test_link_result_exposes_snapshot_not_live_model() + fibers.run(function (root_scope) + local st, _, result = run_link_in_child(root_scope, { + link_id = 'link-g', + + components = { + { + name = 'reader', + run = function () + return { + role = 'reader', + } + end, + }, + }, + }) + + assert_eq(st, 'ok') + assert_not_nil(result.snapshot) + assert_nil(result.model) + end) +end + + +------------------------------------------------------------------------------- +-- Complete policy may not finish while components are still live +------------------------------------------------------------------------------- + +function tests.test_complete_policy_action_is_rejected() + fibers.run(function (root_scope) + local st, _, primary = run_link_in_child(root_scope, { + link_id = 'link-complete-early', + + policy = function (_, ev) + if ev.kind == 'component_done' and ev.component == 'fast' then + return { + action = 'complete', + reason = 'not_all_done', + } + end + + return { + action = 'continue', + } + end, + + components = { + { + name = 'fast', + run = function () + return { + role = 'fast', + } + end, + }, + + { + name = 'slow', + run = function () + fibers.perform(sleep.sleep_op(10)) + return { + role = 'slow', + } + end, + }, + }, + }) + + assert_eq(st, 'failed') + assert_match(primary, 'unknown policy action: complete') + end) +end + + +------------------------------------------------------------------------------- +-- Cancel policy state is not regressed by later component completions +------------------------------------------------------------------------------- + +function tests.test_cancel_policy_state_does_not_regress_after_later_completion() + fibers.run(function (root_scope) + local st, _, result = run_link_in_child(root_scope, { + link_id = 'link-cancel-regression', + + policy = function (_, ev) + if ev.kind == 'component_done' and ev.component == 'fast' then + return { + action = 'cancel', + reason = 'stop after fast', + } + end + + return { action = 'continue' } + end, + + components = { + { + name = 'fast', + run = function () + return { role = 'fast' } + end, + }, + + { + name = 'slow', + run = function () + fibers.perform(sleep.sleep_op(10)) + return { role = 'slow' } + end, + }, + }, + }) + + assert_eq(st, 'ok') + assert_eq(result.snapshot.state, 'cancelling') + assert_eq(result.snapshot.reason, 'stop after fast') + assert_eq(result.snapshot.completed, 2) + assert_eq(result.snapshot.components.fast.status, 'ok') + assert_eq(result.snapshot.components.slow.status, 'cancelled') + end) +end + +------------------------------------------------------------------------------- +-- Component result mutation after reporting does not mutate public snapshots +------------------------------------------------------------------------------- + +function tests.test_completion_result_mutation_after_reporting_does_not_mutate_snapshot() + fibers.run(function (root_scope) + local returned = { role = 'reader', value = 1 } + + local st, _, result = run_link_in_child(root_scope, { + link_id = 'link-result-copy', + + components = { + { + name = 'reader', + run = function () + return returned + end, + }, + }, + }) + + assert_eq(st, 'ok') + assert_eq(result.snapshot.components.reader.result.value, 1) + + returned.value = 99 + returned.role = 'mutated' + + assert_eq(result.snapshot.components.reader.result.value, 1) + assert_eq(result.snapshot.components.reader.result.role, 'reader') + end) +end + + +------------------------------------------------------------------------------- +-- Component workers receive narrowed link capabilities, not the link coordinator +------------------------------------------------------------------------------- + +function tests.test_component_worker_receives_narrowed_link_capability() + fibers.run(function (root_scope) + local saw_caps = false + + local st, _, result = run_link_in_child(root_scope, { + link_id = 'link-cap-test', + + components = { + { + name = 'reader', + run = function (_, caps) + saw_caps = true + assert_eq(caps.link_id, 'link-cap-test') + assert_eq(caps.component, 'reader') + assert_eq(caps._model, nil) + assert_eq(caps._components, nil) + assert_eq(caps._done_tx, nil) + assert_eq(type(caps.snapshot), 'table') + return { role = 'reader' } + end, + }, + }, + }) + + assert_eq(st, 'ok') + assert_true(saw_caps) + assert_eq(result.snapshot.components.reader.status, 'ok') + end) +end + + +------------------------------------------------------------------------------- +-- Foreign component completions are ignored even when link_generation matches +------------------------------------------------------------------------------- + +function tests.test_component_completion_with_wrong_link_id_is_ignored() + local scoped_work = require 'devicecode.support.scoped_work' + local old_start = scoped_work.start + + local ok_test, err_test = pcall(function () + scoped_work.start = function (spec) + if spec + and spec.identity + and spec.identity.kind == 'component_done' + and spec.identity.component == 'reader' + then + local orig_report = spec.report + local wrapped = {} + for k, v in pairs(spec) do wrapped[k] = v end + + wrapped.report = function (ev) + local foreign = {} + for k, v in pairs(ev) do foreign[k] = v end + foreign.link_id = 'wrong-link-id' + foreign.status = 'failed' + foreign.primary = 'foreign completion should be ignored' + + local ok, err = orig_report(foreign) + if ok ~= true then return ok, err end + + return orig_report(ev) + end + + return old_start(wrapped) + end + + return old_start(spec) + end + + fibers.run(function (root_scope) + local st, _, result = run_link_in_child(root_scope, { + link_id = 'real-link-id', + + components = { + { + name = 'reader', + run = function () + return { role = 'reader' } + end, + }, + }, + }) + + assert_eq(st, 'ok') + assert_eq(result.snapshot.link_id, 'real-link-id') + assert_eq(result.snapshot.components.reader.status, 'ok') + assert_eq(result.snapshot.components.reader.primary, nil) + end) + end) + + scoped_work.start = old_start + + if not ok_test then + error(err_test, 0) + end +end + +return tests diff --git a/tests/unit/fabric/test_model.lua b/tests/unit/fabric/test_model.lua new file mode 100644 index 00000000..b9e281c1 --- /dev/null +++ b/tests/unit/fabric/test_model.lua @@ -0,0 +1,262 @@ +-- tests/unit/fabric/test_model.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' + +local model_mod = require 'services.fabric.model' +local queue = require 'devicecode.support.queue' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_true(v, msg) + if v ~= true then + fail(msg or ('expected true, got ' .. tostring(v))) + end +end + +local function assert_false(v, msg) + if v ~= false then + fail(msg or ('expected false, got ' .. tostring(v))) + end +end + +local function assert_nil(v, msg) + if v ~= nil then + fail(msg or ('expected nil, got ' .. tostring(v))) + end +end + +local function assert_not_nil(v, msg) + if v == nil then + fail(msg or 'expected non-nil value') + end +end + +local function assert_eq(a, b, msg) + if a ~= b then + fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) + end +end + +------------------------------------------------------------------------------- +-- snapshot returns a copy +------------------------------------------------------------------------------- + +function tests.test_snapshot_returns_copy() + fibers.run(function () + local m = model_mod.new({ state = 'idle', count = 1 }) + + local snap = m:snapshot() + snap.state = 'mutated' + snap.count = 99 + + local snap2 = m:snapshot() + + assert_eq(snap2.state, 'idle') + assert_eq(snap2.count, 1) + end) +end + +------------------------------------------------------------------------------- +-- set_snapshot does not signal when materially unchanged +------------------------------------------------------------------------------- + +function tests.test_set_snapshot_unchanged_does_not_increment_version() + fibers.run(function () + local m = model_mod.new({ state = 'idle', count = 1 }) + + local v0 = m:version() + + local changed, v = m:set_snapshot({ state = 'idle', count = 1 }) + + assert_false(changed) + assert_eq(v, v0) + assert_eq(m:version(), v0) + end) +end + +------------------------------------------------------------------------------- +-- set_snapshot signals on material change +------------------------------------------------------------------------------- + +function tests.test_set_snapshot_changed_increments_version() + fibers.run(function () + local m = model_mod.new({ state = 'idle', count = 1 }) + + local v0 = m:version() + + local changed, v1 = m:set_snapshot({ state = 'ready', count = 1 }) + + assert_true(changed) + assert_eq(v1, v0 + 1) + assert_eq(m:version(), v0 + 1) + + local snap = m:snapshot() + assert_eq(snap.state, 'ready') + assert_eq(snap.count, 1) + end) +end + +------------------------------------------------------------------------------- +-- changed_op returns a versioned snapshot when already stale +------------------------------------------------------------------------------- + +function tests.test_changed_op_returns_immediately_when_version_is_stale() + fibers.run(function () + local m = model_mod.new({ state = 'idle' }) + + local seen = m:version() + + m:set_snapshot({ state = 'ready' }) + + local version, snap, err = fibers.perform(m:changed_op(seen)) + + assert_eq(version, seen + 1) + assert_eq(snap.state, 'ready') + assert_nil(err) + end) +end + +------------------------------------------------------------------------------- +-- changed_op waits and returns versioned snapshot +------------------------------------------------------------------------------- + +function tests.test_changed_op_waits_for_change() + fibers.run(function (scope) + local m = model_mod.new({ state = 'idle' }) + local seen = m:version() + + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = scope:spawn(function () + local version, snap, cerr = fibers.perform(m:changed_op(seen)) + queue.assert_admit_required(tx, { + version = version, + snap = snap, + err = cerr, + }, 'model_change_result') + end) + + assert_true(ok, err) + + -- Let the waiter reach changed_op. + fibers.perform(sleep.sleep_op(0.001)) + + m:set_snapshot({ state = 'ready' }) + + local result = fibers.perform(rx:recv_op()) + + assert_not_nil(result) + assert_eq(result.version, seen + 1) + assert_eq(result.snap.state, 'ready') + assert_nil(result.err) + end) +end + +------------------------------------------------------------------------------- +-- terminate wakes up-to-date observers with a reason +------------------------------------------------------------------------------- + +function tests.test_terminate_wakes_observer_with_reason() + fibers.run(function (scope) + local m = model_mod.new({ state = 'idle' }) + local seen = m:version() + + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = scope:spawn(function () + local version, snap, cerr = fibers.perform(m:changed_op(seen)) + queue.assert_admit_required(tx, { + version = version, + snap = snap, + err = cerr, + }, 'model_terminate_result') + end) + + assert_true(ok, err) + + fibers.perform(sleep.sleep_op(0.001)) + + m:terminate('model stopped') + + local result = fibers.perform(rx:recv_op()) + + assert_not_nil(result) + assert_nil(result.version) + assert_nil(result.snap) + assert_eq(result.err, 'model stopped') + end) +end + +------------------------------------------------------------------------------- +-- terminate is idempotent and preserves first reason +------------------------------------------------------------------------------- + +function tests.test_close_is_idempotent_and_preserves_first_reason() + fibers.run(function () + local m = model_mod.new({ state = 'idle' }) + + assert_true(m:terminate('first')) + assert_true(m:terminate('second')) + + assert_true(m:is_closed()) + assert_eq(m:why(), 'first') + end) +end + +------------------------------------------------------------------------------- +-- set_snapshot after terminate fails without changing version +------------------------------------------------------------------------------- + +function tests.test_set_snapshot_after_terminate_fails_without_incrementing_version() + fibers.run(function () + local m = model_mod.new({ state = 'idle' }) + + local v0 = m:version() + + m:terminate('closed_for_test') + + local changed, err = m:set_snapshot({ state = 'ready' }) + + assert_nil(changed) + assert_eq(err, 'closed_for_test') + assert_eq(m:version(), v0) + + local snap = m:snapshot() + assert_eq(snap.state, 'idle') + end) +end + +------------------------------------------------------------------------------- +-- stale observer after close still sees unseen snapshot first +------------------------------------------------------------------------------- + +function tests.test_stale_observer_after_close_sees_unseen_snapshot_first() + fibers.run(function () + local m = model_mod.new({ state = 'idle' }) + + local seen = m:version() + + m:set_snapshot({ state = 'ready' }) + m:terminate('closed_after_change') + + local version, snap, err = fibers.perform(m:changed_op(seen)) + + assert_eq(version, seen + 1) + assert_eq(snap.state, 'ready') + assert_nil(err) + + local version2, snap2, err2 = fibers.perform(m:changed_op(version)) + + assert_nil(version2) + assert_nil(snap2) + assert_eq(err2, 'closed_after_change') + end) +end + +return tests diff --git a/tests/unit/fabric/test_protocol.lua b/tests/unit/fabric/test_protocol.lua new file mode 100644 index 00000000..13cd3a77 --- /dev/null +++ b/tests/unit/fabric/test_protocol.lua @@ -0,0 +1,534 @@ +-- tests/services/fabric/test_protocol.lua + +local protocol = require 'services.fabric.protocol' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_eq(a, b, msg) + if a ~= b then + fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) + end +end + +local function assert_nil(v, msg) + if v ~= nil then + fail(msg or ('expected nil, got ' .. tostring(v))) + end +end + +local function assert_not_nil(v, msg) + if v == nil then + fail(msg or 'expected non-nil value') + end +end + +function tests.test_classifies_reference_frame_types() + assert_eq(protocol.classify({ type = 'hello' }), 'session_control') + assert_eq(protocol.classify({ type = 'pub' }), 'rpc') + assert_eq(protocol.classify({ type = 'xfer_chunk' }), 'transfer_bulk') + + assert_eq(protocol.dispatch_lane({ type = 'hello' }), 'session_control') + assert_eq(protocol.dispatch_lane({ type = 'reply' }), 'rpc') + assert_eq(protocol.dispatch_lane({ type = 'xfer_chunk' }), 'transfer') +end + +function tests.test_validate_rejects_unknown_type() + local ok, err = protocol.validate({ type = 'wat' }) + + assert_nil(ok) + assert_eq(err, 'invalid_frame_type') +end + +function tests.test_validate_rejects_non_string_frame_keys() + local ok, err = protocol.validate({ + type = 'ping', + [1] = 'bad', + }) + + assert_nil(ok) + assert_eq(err, 'invalid_frame_type') +end + +function tests.test_hello_requires_proto_sid_and_uses_node_wire_field() + local ok, err = protocol.validate({ type = 'hello' }) + + assert_nil(ok) + assert_eq(err, 'missing_proto') + + ok, err = protocol.validate({ + type = 'hello', + proto = protocol.PROTO, + node = 'node-a', + }) + + assert_nil(ok) + assert_eq(err, 'missing_sid') + + ok, err = protocol.validate({ + type = 'hello', + proto = protocol.PROTO, + sid = 'sid-1', + node = 'node-a', + }) + + assert_not_nil(ok) + assert_nil(err) + + ok, err = protocol.validate({ + type = 'hello', + proto = protocol.PROTO, + sid = 'sid-1', + node_id = 'node-a', + }) + + assert_nil(ok) + assert_eq(err, 'unknown_frame_field: node_id') +end + + +function tests.test_proto_field_is_reserved_for_session_hello_frames() + local ok, err = protocol.validate({ + type = 'pub', + proto = protocol.PROTO, + topic = { 'state', 'self' }, + payload = {}, + retain = true, + }) + + assert_nil(ok) + assert_eq(err, 'unknown_frame_field: proto') +end + +function tests.test_ping_and_pong_require_sid() + local ok, err = protocol.validate({ type = 'ping' }) + assert_nil(ok) + assert_eq(err, 'missing_sid') + + ok, err = protocol.validate({ type = 'pong', sid = 'sid-1' }) + assert_not_nil(ok) + assert_nil(err) +end + +function tests.test_pub_requires_dense_scalar_topic_and_boolean_retain() + local ok, err = protocol.validate({ + type = 'pub', + topic = { 'raw', 'member', 'a' }, + payload = { value = 1 }, + retain = true, + }) + + assert_not_nil(ok) + assert_nil(err) + + ok, err = protocol.validate({ + type = 'pub', + topic = { 'raw', {}, 'a' }, + retain = true, + }) + + assert_nil(ok) + assert_eq(err, 'invalid_topic') + + ok, err = protocol.validate({ + type = 'pub', + topic = { 'raw', 'member', 'a' }, + }) + + assert_nil(ok) + assert_eq(err, 'missing_retain') +end + +function tests.test_call_and_reply_validation() + local ok, err = protocol.validate({ + type = 'call', + id = 'call-1', + topic = { 'cap', 'x', 'main', 'rpc', 'do' }, + payload = { n = 1 }, + }) + + assert_not_nil(ok) + assert_nil(err) + + ok, err = protocol.validate({ + type = 'reply', + id = 'call-1', + ok = false, + err = 'nope', + }) + + assert_not_nil(ok) + assert_nil(err) + + ok, err = protocol.validate({ + type = 'reply', + id = 'call-1', + ok = false, + err = {}, + }) + + assert_nil(ok) + assert_eq(err, 'invalid_reply_err') +end + + +function tests.test_rejects_legacy_checksum_field() + local ok, err = protocol.validate({ + type = 'xfer_commit', + xfer_id = 'x1', + size = 3, + digest_alg = protocol.DIGEST_ALG, + digest = protocol.digest_hex('abc'), + checksum = 'legacy', + }) + + assert_nil(ok) + assert_eq(err, 'unknown_frame_field: checksum') +end + +function tests.test_transfer_control_validation() + local ok, err = protocol.validate({ + type = 'xfer_begin', + xfer_id = 'x1', + target = 'slot-a', + size = 123, + digest_alg = protocol.DIGEST_ALG, + digest = '1a2b3c4d', + }) + + assert_not_nil(ok) + assert_nil(err) + + ok, err = protocol.validate({ + type = 'xfer_begin', + xfer_id = 'x1', + target = 'slot-a', + size = 123, + digest_alg = 'sha256', + digest = 'abc', + }) + + assert_nil(ok) + assert_eq(err, 'unsupported_digest_alg') + + ok, err = protocol.validate({ + type = 'xfer_need', + xfer_id = 'x1', + next = -1, + }) + + assert_nil(ok) + assert_eq(err, 'invalid_next') +end + +function tests.test_encode_decode_regular_frame_roundtrip() + local frame = { + type = 'pub', + topic = { 'raw', 'member', 'node-a', 'state', 'temperature' }, + payload = { value = 20 }, + retain = true, + } + + local line, err = protocol.encode_line(frame) + + assert_not_nil(line) + assert_nil(err) + + local decoded, derr = protocol.decode_line(line) + + assert_not_nil(decoded) + assert_nil(derr) + assert_eq(decoded.type, 'pub') + assert_eq(decoded.topic[1], 'raw') + assert_eq(decoded.topic[3], 'node-a') + assert_eq(decoded.payload.value, 20) + assert_eq(decoded.retain, true) +end + +function tests.test_encode_decode_xfer_chunk_roundtrip_preserves_bytes() + local frame = { + type = 'xfer_chunk', + xfer_id = 'xfer-1', + offset = 5, + data = 'abc\0def', + chunk_digest = protocol.chunk_digest('abc\0def'), + } + + local line, err = protocol.encode_line(frame) + + assert_not_nil(line) + assert_nil(err) + + local decoded, derr = protocol.decode_line(line) + + assert_not_nil(decoded) + assert_nil(derr) + assert_eq(decoded.type, 'xfer_chunk') + assert_eq(decoded.xfer_id, 'xfer-1') + assert_eq(decoded.offset, 5) + assert_eq(decoded.data, 'abc\0def') + assert_eq(decoded.chunk_digest, protocol.chunk_digest('abc\0def')) +end + +function tests.test_semantic_xfer_chunk_digest_is_verified() + local ok, err = protocol.validate({ + type = 'xfer_chunk', + xfer_id = 'xfer-1', + offset = 0, + data = 'abc', + chunk_digest = '00000000', + }) + + assert_nil(ok) + assert_eq(err, 'chunk_digest_mismatch') + + ok, err = protocol.validate({ + type = 'xfer_chunk', + xfer_id = 'xfer-1', + offset = 0, + data = 'abc', + chunk_digest = protocol.chunk_digest('abc'), + }) + + assert_not_nil(ok) + assert_nil(err) +end + +function tests.test_encode_line_rejects_semantic_xfer_chunk_digest_mismatch() + local line, err = protocol.encode_line({ + type = 'xfer_chunk', + xfer_id = 'xfer-1', + offset = 0, + data = 'abc', + chunk_digest = '00000000', + }) + + assert_nil(line) + assert_eq(err, 'chunk_digest_mismatch') +end + + +function tests.test_xfer_chunk_requires_chunk_digest_and_strict_unpadded_b64url() + local ok, err = protocol.validate({ + type = 'xfer_chunk', + xfer_id = 'xfer-1', + offset = 0, + data = 'abc', + }) + + assert_nil(ok) + assert_eq(err, 'missing_chunk_digest') + + local line = '{"type":"xfer_chunk","xfer_id":"xfer-1","offset":0,"data":"YQ==","chunk_digest":"' .. protocol.chunk_digest('a') .. '"}' + local frame, derr = protocol.decode_line(line) + assert_nil(frame) + assert_eq(derr, 'invalid_chunk_encoding: invalid_base64url_unpadded') +end + +function tests.test_decode_line_preserves_xfer_chunk_digest_mismatch_for_receiver_retry() + local line = '{"type":"xfer_chunk","xfer_id":"xfer-1","offset":0,"data":"YQ","chunk_digest":"00000000"}' + local frame, err = protocol.decode_line(line) + + assert_not_nil(frame) + assert_nil(err) + assert_eq(frame.type, 'xfer_chunk') + assert_eq(frame.xfer_id, 'xfer-1') + assert_eq(frame.offset, 0) + assert_eq(frame.data, 'a') + assert_eq(frame.chunk_digest, '00000000') + + local ok, verr = protocol.validate(frame) + assert_nil(ok) + assert_eq(verr, 'chunk_digest_mismatch') +end + +function tests.test_decode_line_rejects_non_json() + local frame, err = protocol.decode_line('{') + + assert_nil(frame) + assert_not_nil(err) +end + +function tests.test_constructors_validate_before_returning() + local frame, err = protocol.xfer_need('x1', 10) + + assert_not_nil(frame) + assert_nil(err) + assert_eq(frame.type, 'xfer_need') + assert_eq(frame.next, 10) + + frame, err = protocol.xfer_need('x1', -10) + + assert_nil(frame) + assert_eq(err, 'invalid_next') +end + +function tests.test_hello_accepts_reserved_identity_and_auth_objects() + local frame = assert(protocol.hello('sid-1', 'cm5', { id = 'claim' }, { scheme = 'reserved' })) + local line, err = protocol.encode_line(frame) + assert_not_nil(line) + assert_nil(err) + + local decoded, derr = protocol.decode_line(line) + assert_not_nil(decoded) + assert_nil(derr) + assert_eq(decoded.proto, protocol.PROTO) + assert_eq(decoded.node, 'cm5') + assert_eq(decoded.identity.id, 'claim') + assert_eq(decoded.auth.scheme, 'reserved') +end + + +function tests.test_decode_line_rejects_non_canonical_base64url_chunk_data() + -- "AB" decodes to the same byte as canonical "AA" with non-zero discarded + -- pad bits. fabric-jsonl/1 accepts only the canonical unpadded form. + local line = '{"type":"xfer_chunk","xfer_id":"xfer-1","offset":0,"data":"AB","chunk_digest":"' .. protocol.chunk_digest('\0') .. '"}' + local frame, err = protocol.decode_line(line) + + assert_nil(frame) + assert_eq(err, 'invalid_chunk_encoding: non_canonical_base64url') +end + + +function tests.test_protocol_digest_helpers_use_xxhash32_seed_zero_vectors() + assert_eq(protocol.DIGEST_ALG, 'xxhash32') + assert_eq(protocol.digest_hex(''), '02cc5d05') + assert_eq(protocol.digest_hex('abc'), '32d153ff') + assert_eq(protocol.chunk_digest('abc\0def'), '7955e6e5') +end + +function tests.test_unknown_top_level_fields_are_rejected_but_identity_auth_are_extensible() + local ok, err = protocol.validate({ + type = 'hello', + proto = protocol.PROTO, + sid = 'sid-1', + node = 'mcu', + identity = { id = 'claimed-peer', future = { ignored = true } }, + auth = { scheme = 'reserved', future = { proof = nil } }, + }) + assert_not_nil(ok) + assert_nil(err) + + ok, err = protocol.validate({ + type = 'hello', + proto = protocol.PROTO, + sid = 'sid-1', + node = 'mcu', + future = true, + }) + assert_nil(ok) + assert_eq(err, 'unknown_frame_field: future') + + local line = '{"type":"reply","id":"call-1","ok":true,"future":true}' + local frame, derr = protocol.decode_line(line) + assert_nil(frame) + assert_eq(derr, 'unknown_frame_field: future') +end + +function tests.test_transfer_digests_are_strict_xxhash32_hex() + local ok, err = protocol.validate({ + type = 'xfer_begin', + xfer_id = 'xfer-1', + target = 'updater/main', + size = 3, + digest_alg = 'sha256', + digest = protocol.digest_hex('abc'), + }) + assert_nil(ok) + assert_eq(err, 'unsupported_digest_alg') + + ok, err = protocol.validate({ + type = 'xfer_commit', + xfer_id = 'xfer-1', + size = 3, + digest_alg = protocol.DIGEST_ALG, + digest = 'ABCDEF12', + }) + assert_nil(ok) + assert_eq(err, 'invalid_digest') +end + + +function tests.test_contract_constants_match_fabric_jsonl_v1() + assert_eq(protocol.PROTO, 'fabric-jsonl/1') + assert_eq(protocol.DIGEST_ALG, 'xxhash32') + assert_eq(protocol.DEFAULT_CHUNK_SIZE, 2048) + + assert_eq(protocol.classify('hello'), 'session_control') + assert_eq(protocol.classify('hello_ack'), 'session_control') + assert_eq(protocol.classify('ping'), 'session_control') + assert_eq(protocol.classify('pong'), 'session_control') + + assert_eq(protocol.classify('pub'), 'rpc') + assert_eq(protocol.classify('unretain'), 'rpc') + assert_eq(protocol.classify('call'), 'rpc') + assert_eq(protocol.classify('reply'), 'rpc') + + assert_eq(protocol.classify('xfer_begin'), 'transfer_control') + assert_eq(protocol.classify('xfer_ready'), 'transfer_control') + assert_eq(protocol.classify('xfer_need'), 'transfer_control') + assert_eq(protocol.classify('xfer_commit'), 'transfer_control') + assert_eq(protocol.classify('xfer_done'), 'transfer_control') + assert_eq(protocol.classify('xfer_abort'), 'transfer_control') + assert_eq(protocol.classify('xfer_chunk'), 'transfer_bulk') +end + +function tests.test_unversioned_hello_is_rejected() + local ok, err = protocol.validate({ + type = 'hello', + sid = 'sid-1', + node = 'cm5', + }) + + assert_nil(ok) + assert_eq(err, 'missing_proto') + + ok, err = protocol.validate({ + type = 'hello_ack', + sid = 'sid-2', + node = 'mcu', + }) + + assert_nil(ok) + assert_eq(err, 'missing_proto') +end + +function tests.test_legacy_checksum_field_is_rejected_on_begin_and_commit() + local ok, err = protocol.validate({ + type = 'xfer_begin', + xfer_id = 'x1', + target = 'updater/main', + size = 3, + digest_alg = protocol.DIGEST_ALG, + digest = protocol.digest_hex('abc'), + checksum = 'legacy', + }) + + assert_nil(ok) + assert_eq(err, 'unknown_frame_field: checksum') + + ok, err = protocol.validate({ + type = 'xfer_commit', + xfer_id = 'x1', + size = 3, + digest_alg = protocol.DIGEST_ALG, + digest = protocol.digest_hex('abc'), + checksum = 'legacy', + }) + + assert_nil(ok) + assert_eq(err, 'unknown_frame_field: checksum') +end + +function tests.test_legacy_chunk_without_digest_is_rejected_on_decode() + local line = '{"type":"xfer_chunk","xfer_id":"xfer-1","offset":0,"data":"YQ"}' + local frame, err = protocol.decode_line(line) + + assert_nil(frame) + assert_eq(err, 'missing_chunk_digest') +end + +return tests diff --git a/tests/unit/fabric/test_service.lua b/tests/unit/fabric/test_service.lua new file mode 100644 index 00000000..5733b186 --- /dev/null +++ b/tests/unit/fabric/test_service.lua @@ -0,0 +1,565 @@ +-- tests/unit/fabric/test_service.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local busmod = require 'bus' + +local service = require 'services.fabric.service' +local topics = require 'services.fabric.topics' +local cfg_mod = require 'services.fabric.config' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_true(v, msg) + if v ~= true then + fail(msg or ('expected true, got ' .. tostring(v))) + end +end + +local function assert_not_nil(v, msg) + if v == nil then + fail(msg or 'expected non-nil value') + end +end + +local function assert_nil(v, msg) + if v ~= nil then + fail(msg or ('expected nil, got ' .. tostring(v))) + end +end + +local function assert_eq(a, b, msg) + if a ~= b then + fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) + end +end + +local function assert_match(s, pat, msg) + if type(s) ~= 'string' or not s:match(pat) then + fail(msg or ('expected "' .. tostring(s) .. '" to match ' .. tostring(pat))) + end +end + +local function run_service(params) + return fibers.run_scope(function (scope) + return service.run(scope, params) + end) +end + +local function one_component(name, role) + return { + name = name, + + run = function () + return { + role = role or name, + } + end, + } +end + +------------------------------------------------------------------------------- +-- Successful static links complete the service +------------------------------------------------------------------------------- + +function tests.test_successful_links_complete_service() + fibers.run(function () + local st, rep, result = run_service { + service_id = 'fabric-test', + + links = { + { + link_id = 'link-a', + components = { + one_component('reader'), + }, + }, + + { + link_id = 'link-b', + components = { + one_component('writer'), + }, + }, + }, + } + + assert_eq(st, 'ok') + assert_eq(#rep.extra_errors, 0) + assert_not_nil(result) + + local snap = result.snapshot + + assert_eq(snap.service_id, 'fabric-test') + assert_eq(snap.state, 'completed') + assert_eq(snap.completed, 2) + assert_eq(snap.total, 2) + + assert_eq(snap.links['link-a'].status, 'ok') + assert_eq(snap.links['link-b'].status, 'ok') + assert_eq(snap.links['link-a'].snapshot.link_id, 'link-a') + assert_eq(snap.links['link-b'].snapshot.link_id, 'link-b') + end) +end + +------------------------------------------------------------------------------- +-- Link failure is interpreted by the default service policy +------------------------------------------------------------------------------- + +function tests.test_link_failure_fails_service_by_default_policy() + fibers.run(function () + local st, rep, primary = run_service { + service_id = 'fabric-test', + + links = { + { + link_id = 'bad-link', + components = { + { + name = 'reader', + run = function () + error('reader exploded', 0) + end, + }, + }, + }, + + { + link_id = 'slow-link', + components = { + { + name = 'slow', + run = function () + fibers.perform(sleep.sleep_op(10)) + return { + role = 'slow', + } + end, + }, + }, + }, + }, + } + + assert_eq(st, 'failed') + assert_not_nil(rep) + assert_match(primary, 'link bad%-link failed') + assert_match(primary, 'component reader failed') + assert_match(primary, 'reader exploded') + end) +end + +------------------------------------------------------------------------------- +-- Link failure can be treated as data by explicit service policy +------------------------------------------------------------------------------- + +function tests.test_link_failure_is_data_when_policy_allows_it() + fibers.run(function () + local seen_failed = false + + local st, rep, result = run_service { + service_id = 'fabric-test', + + policy = function (_, ev) + if ev.kind == 'link_done' + and ev.link_id == 'bad-link' + and ev.status == 'failed' + then + seen_failed = true + end + + return { + action = 'continue', + } + end, + + links = { + { + link_id = 'bad-link', + components = { + { + name = 'reader', + run = function () + error('reader failed as data', 0) + end, + }, + }, + }, + + { + link_id = 'good-link', + components = { + one_component('writer'), + }, + }, + }, + } + + assert_eq(st, 'ok') + assert_eq(#rep.extra_errors, 0) + assert_true(seen_failed) + + local snap = result.snapshot + + assert_eq(snap.state, 'completed') + assert_eq(snap.completed, 2) + assert_eq(snap.links['bad-link'].status, 'failed') + assert_match(snap.links['bad-link'].primary, 'reader failed as data') + assert_eq(snap.links['good-link'].status, 'ok') + end) +end + +------------------------------------------------------------------------------- +-- Link cancellation is unexpected by default +------------------------------------------------------------------------------- + +function tests.test_link_cancellation_fails_service_by_default_policy() + fibers.run(function () + local st, _, primary = run_service { + service_id = 'fabric-test', + + links = { + { + link_id = 'cancelled-link', + components = { + { + name = 'session', + + run = function (component_scope) + component_scope:cancel('session stopped') + fibers.perform(sleep.sleep_op(10)) + + return { + unreachable = true, + } + end, + }, + }, + }, + }, + } + + assert_eq(st, 'failed') + assert_match(primary, 'link cancelled%-link failed') + assert_match(primary, 'component session cancelled unexpectedly') + assert_match(primary, 'session stopped') + end) +end + +------------------------------------------------------------------------------- +-- Service validates duplicate link ids before work begins +------------------------------------------------------------------------------- + +function tests.test_duplicate_link_ids_are_rejected() + fibers.run(function () + local st, _, primary = run_service { + service_id = 'fabric-test', + + links = { + { + link_id = 'dup', + components = { + one_component('a'), + }, + }, + + { + link_id = 'dup', + components = { + one_component('b'), + }, + }, + }, + } + + assert_eq(st, 'failed') + assert_match(primary, 'duplicate link id') + end) +end + +------------------------------------------------------------------------------- +-- Service result exposes a snapshot, not a live model +------------------------------------------------------------------------------- + +function tests.test_service_result_exposes_snapshot_not_live_model() + fibers.run(function () + local st, _, result = run_service { + service_id = 'fabric-test', + + links = { + { + link_id = 'link-a', + components = { + one_component('reader'), + }, + }, + }, + } + + assert_eq(st, 'ok') + assert_not_nil(result.snapshot) + assert_eq(result.model, nil) + end) +end + + +------------------------------------------------------------------------------- +-- Complete policy may not finish while links are still live +------------------------------------------------------------------------------- + +function tests.test_complete_policy_before_all_links_fails_service() + fibers.run(function () + local st, _, primary = run_service { + service_id = 'fabric-test', + + policy = function (_, ev) + if ev.kind == 'link_done' and ev.link_id == 'fast-link' then + return { + action = 'complete', + reason = 'not_all_done', + } + end + + return { + action = 'continue', + } + end, + + links = { + { + link_id = 'fast-link', + components = { + one_component('fast'), + }, + }, + + { + link_id = 'slow-link', + components = { + { + name = 'slow', + run = function () + fibers.perform(sleep.sleep_op(10)) + return { + role = 'slow', + } + end, + }, + }, + }, + }, + } + + assert_eq(st, 'failed') + assert_match(primary, 'complete policy before all links completed') + end) +end + + +------------------------------------------------------------------------------- +-- Cancel policy state is not regressed by later link completions +------------------------------------------------------------------------------- + +function tests.test_cancel_policy_state_does_not_regress_after_later_completion() + fibers.run(function () + local st, _, result = run_service { + service_id = 'fabric-cancel-regression', + + link_runner = function (link_scope, spec) + if spec.link_id == 'slow-link' then + fibers.perform(sleep.sleep_op(10)) + end + + return { + link_id = spec.link_id, + role = spec.link_id, + } + end, + + policy = function (_, ev) + if ev.kind == 'link_done' and ev.link_id == 'fast-link' then + return { + action = 'cancel', + reason = 'stop after fast link', + } + end + + return { action = 'continue' } + end, + + links = { + { link_id = 'fast-link' }, + { link_id = 'slow-link' }, + }, + } + + assert_eq(st, 'ok') + assert_eq(result.snapshot.state, 'cancelling') + assert_eq(result.snapshot.reason, 'stop after fast link') + assert_eq(result.snapshot.completed, 2) + assert_eq(result.snapshot.links['fast-link'].status, 'ok') + assert_eq(result.snapshot.links['slow-link'].status, 'cancelled') + end) +end + + +------------------------------------------------------------------------------- +-- Link runners receive narrowed service capabilities, not the service coordinator +------------------------------------------------------------------------------- + +function tests.test_link_runner_receives_narrowed_service_capability() + fibers.run(function () + local saw_caps = false + + local st, _, result = run_service { + service_id = 'fabric-service-cap-test', + + link_runner = function (_, spec, caps) + saw_caps = true + assert_eq(spec.link_id, 'link-cap-a') + assert_eq(caps.service_id, 'fabric-service-cap-test') + assert_eq(caps._model, nil) + assert_eq(caps._links, nil) + assert_eq(caps._done_tx, nil) + assert_eq(type(caps.snapshot), 'table') + return { role = 'link-runner' } + end, + + links = { + { link_id = 'link-cap-a' }, + }, + } + + assert_eq(st, 'ok') + assert_true(saw_caps) + assert_eq(result.snapshot.links['link-cap-a'].status, 'ok') + end) +end + + + +------------------------------------------------------------------------------- +-- Public start shell: config-driven generation lifecycle +------------------------------------------------------------------------------- + +local function start_shell_in_scope(scope, conn, opts) + local ok, err = scope:spawn(function () + service.start(conn, opts) + end) + assert_true(ok, tostring(err)) +end + +local function minimal_compiled_config(link_id) + return { + schema = cfg_mod.SCHEMA, + local_node = 'host-a', + links = { + { + id = link_id or 'link-a', + peer_id = 'peer-a', + bridge = {}, + }, + }, + } +end + +function tests.test_start_shell_loads_retained_config_and_starts_generation() + fibers.run(function () + local bus = busmod.new() + local conn = bus:connect() + local seen = false + local saw_transfer_rx = false + local done_tx, done_rx = require('fibers.mailbox').new(1, { full = 'reject_newest' }) + + conn:retain(topics.cfg(), minimal_compiled_config('link-started')) + + local st, _, primary = fibers.run_scope(function (scope) + start_shell_in_scope(scope, conn, { + link_runner = function (_, spec) + seen = true + saw_transfer_rx = spec.transfer_admission_rx ~= nil + done_tx:send(true) + return { role = 'test-link' } + end, + }) + + local ok = fibers.perform(done_rx:recv_op()) + assert_true(ok) + scope:cancel('test complete') + end) + + assert_eq(st, 'cancelled', tostring(primary)) + assert_true(seen) + assert_eq(saw_transfer_rx, false) + end) +end + +function tests.test_start_shell_replaces_generation_on_config_change_after_old_generation_stops() + fibers.run(function () + local bus = busmod.new() + local conn = bus:connect() + local mailbox = require 'fibers.mailbox' + local cond = require 'fibers.cond' + local ready_tx, ready_rx = mailbox.new(4, { full = 'reject_newest' }) + local old_finalised = cond.new() + local old_final_status + local seen = {} + + conn:retain(topics.cfg(), minimal_compiled_config('link-old')) + + local st, _, primary = fibers.run_scope(function (scope) + start_shell_in_scope(scope, conn, { + link_runner = function (link_scope, spec) + seen[#seen + 1] = spec.link_id + ready_tx:send(spec.link_id) + + if spec.link_id == 'link-old' then + link_scope:finally(function (_, status, primary) + old_final_status = { status = status, primary = primary } + old_finalised:signal() + end) + fibers.perform(sleep.sleep_op(10)) + end + + return { role = 'generation-link', link_id = spec.link_id } + end, + }) + + assert_eq(fibers.perform(ready_rx:recv_op()), 'link-old') + + conn:retain(topics.cfg(), minimal_compiled_config('link-new')) + + local which, value = fibers.perform(fibers.named_choice{ + old_done = old_finalised:wait_op(), + new_started = ready_rx:recv_op(), + timeout = sleep.sleep_op(0.5), + }) + assert_eq(which, 'old_done') + assert_not_nil(old_final_status) + assert_eq(old_final_status.status, 'cancelled') + + assert_eq(fibers.perform(ready_rx:recv_op()), 'link-new') + + scope:cancel('test complete') + end) + + assert_eq(st, 'cancelled', tostring(primary)) + assert_eq(seen[1], 'link-old') + assert_eq(seen[2], 'link-new') + end) +end + + +function tests.test_public_transfer_manager_uses_scoped_request_cancellation() + local f = assert(io.open('../src/services/fabric/service.lua', 'r')) + local src = f:read('*a'); f:close() + if not src:find("devicecode.support.request_owner", 1, true) then fail('fabric service should use request_owner for public transfer requests') end + if not src:find("kind = 'public_transfer_request_done'", 1, true) then fail('public transfer requests should be admitted as scoped work') end + if not src:find('cancel_op = owner:caller_cancel_op()', 1, true) then fail('public transfer requests should propagate caller cancellation') end +end + +return tests diff --git a/tests/unit/fabric/test_session.lua b/tests/unit/fabric/test_session.lua new file mode 100644 index 00000000..d37b64d8 --- /dev/null +++ b/tests/unit/fabric/test_session.lua @@ -0,0 +1,349 @@ +-- tests/unit/fabric/test_session.lua +-- +-- Focused tests for fabric.session as the sole promoter of raw wire frames into +-- session-context-tagged semantic events. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' + +local session = require 'services.fabric.session' +local protocol = require 'services.fabric.protocol' +local queue = require 'devicecode.support.queue' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_true(v, msg) + if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end +end + +local function assert_nil(v, msg) + if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end +end + +local function assert_not_nil(v, msg) + if v == nil then fail(msg or 'expected non-nil value') end +end + +local function assert_eq(a, b, msg) + if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end +end + +local function recv_with_timeout(rx, label, timeout) + timeout = timeout or 0.25 + local which, item = fibers.perform(fibers.named_choice{ + item = rx:recv_op(), + timeout = sleep.sleep_op(timeout), + }) + if which == 'timeout' then + fail('timed out waiting for ' .. tostring(label or 'item')) + end + return item +end + +local function start_session(scope, opts) + opts = opts or {} + local frame_tx, frame_rx = mailbox.new(16, { full = 'reject_newest' }) + local control_tx, control_rx = mailbox.new(16, { full = 'reject_newest' }) + local rpc_tx, rpc_rx = mailbox.new(16, { full = 'reject_newest' }) + local transfer_tx, transfer_rx = mailbox.new(16, { full = 'reject_newest' }) + local outbound = session.new_outbound_gate { tx_control = control_tx, tx_rpc = control_tx, tx_bulk = control_tx } + local done_tx, done_rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = scope:spawn(function () + local result = session.run(scope, { + link_id = opts.link_id or 'link-a', + peer_id = opts.peer_id or 'peer-a', + local_node = opts.local_node or 'cm5', + local_sid = opts.local_sid or 'cm5-sid', + identity_claim = opts.identity_claim, + auth_claim = opts.auth_claim, + frame_rx = frame_rx, + tx_control = control_tx, + outbound = outbound, + rpc_tx = rpc_tx, + transfer_tx = transfer_tx, + hello_interval_s = opts.hello_interval_s or 10, + ping_interval_s = opts.ping_interval_s or 10, + liveness_timeout_s = opts.liveness_timeout_s or 20, + bad_frame_limit = opts.bad_frame_limit, + bad_frame_window_s = opts.bad_frame_window_s, + }) + queue.try_admit_required(done_tx, result, 'session_done') + end) + assert_true(ok, err) + + return { + frame_tx = frame_tx, + control_rx = control_rx, + rpc_rx = rpc_rx, + transfer_rx = transfer_rx, + done_rx = done_rx, + } +end + +local function admit_frame(tx, frame) + local ok, err = queue.try_admit_required(tx, { + kind = 'frame_received', + frame = frame, + }, 'test_frame_admit_failed') + assert_true(ok, err) +end + +local function admit_wire_error(tx, err) + local ok, admit_err = queue.try_admit_required(tx, { + kind = 'wire_error', + err = err or 'decode_failed: bad json', + at = fibers.now(), + }, 'test_wire_error_admit_failed') + assert_true(ok, admit_err) +end + +function tests.test_session_establishes_and_emits_lifecycle_to_both_lanes() + fibers.run(function (scope) + local h = start_session(scope, { + identity_claim = { id = 'cm5' }, + }) + + local hello = recv_with_timeout(h.control_rx, 'initial hello') + assert_eq(hello.frame.type, 'hello') + assert_eq(hello.frame.proto, protocol.PROTO) + assert_eq(hello.frame.node, 'cm5') + assert_eq(hello.frame.identity.id, 'cm5') + + admit_frame(h.frame_tx, assert(protocol.hello_ack('mcu-sid', 'mcu'))) + + local rpc_ev = recv_with_timeout(h.rpc_rx, 'rpc peer session') + local xfer_ev = recv_with_timeout(h.transfer_rx, 'transfer peer session') + assert_eq(rpc_ev.kind, 'peer_session') + assert_eq(xfer_ev.kind, 'peer_session') + assert_eq(rpc_ev.session.session_generation, 1) + assert_eq(rpc_ev.session.peer_sid, 'mcu-sid') + assert_eq(rpc_ev.session.peer_node, 'mcu') + assert_eq(rpc_ev.session.proto, protocol.PROTO) + + h.frame_tx:close('done') + local done = recv_with_timeout(h.done_rx, 'session done') + assert_eq(done.role, 'session') + assert_eq(done.snapshot.session_generation, 1) + end) +end + +function tests.test_non_session_frames_are_dropped_until_session_is_established() + fibers.run(function (scope) + local h = start_session(scope) + recv_with_timeout(h.control_rx, 'initial hello') + + admit_frame(h.frame_tx, assert(protocol.pub({ 'state', 'self' }, { ok = true }, true))) + + local ev = queue.try_recv_now(h.rpc_rx) + assert_nil(ev, 'rpc frame should not be emitted before session establishment') + + admit_frame(h.frame_tx, assert(protocol.hello_ack('mcu-sid', 'mcu'))) + recv_with_timeout(h.rpc_rx, 'peer session') + + admit_frame(h.frame_tx, assert(protocol.pub({ 'state', 'self' }, { ok = true }, true))) + local routed = recv_with_timeout(h.rpc_rx, 'session-tagged rpc frame') + assert_eq(routed.kind, 'session_frame') + assert_eq(routed.lane, 'rpc') + assert_eq(routed.session.session_generation, 1) + assert_eq(routed.session.peer_sid, 'mcu-sid') + assert_eq(routed.frame.type, 'pub') + + h.frame_tx:close('done') + recv_with_timeout(h.done_rx, 'session done') + end) +end + +function tests.test_transfer_frames_are_tagged_for_transfer_lane_only() + fibers.run(function (scope) + local h = start_session(scope) + recv_with_timeout(h.control_rx, 'initial hello') + admit_frame(h.frame_tx, assert(protocol.hello_ack('mcu-sid', 'mcu'))) + recv_with_timeout(h.rpc_rx, 'rpc peer session') + recv_with_timeout(h.transfer_rx, 'transfer peer session') + + admit_frame(h.frame_tx, assert(protocol.xfer_ready('xfer-1'))) + local ev = recv_with_timeout(h.transfer_rx, 'transfer frame') + assert_eq(ev.kind, 'session_frame') + assert_eq(ev.lane, 'transfer') + assert_eq(ev.session.session_generation, 1) + assert_eq(ev.frame.type, 'xfer_ready') + + local rpc_ev = queue.try_recv_now(h.rpc_rx) + assert_nil(rpc_ev, 'transfer frame should not be emitted to rpc lane') + + h.frame_tx:close('done') + recv_with_timeout(h.done_rx, 'session done') + end) +end + +function tests.test_new_peer_sid_drops_old_generation_and_starts_next_generation() + fibers.run(function (scope) + local h = start_session(scope) + recv_with_timeout(h.control_rx, 'initial hello') + + admit_frame(h.frame_tx, assert(protocol.hello_ack('sid-1', 'mcu'))) + local first = recv_with_timeout(h.rpc_rx, 'first peer session') + assert_eq(first.kind, 'peer_session') + assert_eq(first.session.session_generation, 1) + + admit_frame(h.frame_tx, assert(protocol.hello('sid-2', 'mcu'))) + local drop = recv_with_timeout(h.rpc_rx, 'drop old peer session') + local next_ev = recv_with_timeout(h.rpc_rx, 'next peer session') + + assert_eq(drop.kind, 'peer_session_dropped') + assert_eq(drop.session.session_generation, 1) + assert_eq(drop.session.peer_sid, 'sid-1') + assert_eq(drop.reason, 'peer_sid_changed') + + assert_eq(next_ev.kind, 'peer_session') + assert_eq(next_ev.session.session_generation, 2) + assert_eq(next_ev.session.peer_sid, 'sid-2') + + local ack = recv_with_timeout(h.control_rx, 'hello ack for new sid') + assert_eq(ack.frame.type, 'hello_ack') + + h.frame_tx:close('done') + recv_with_timeout(h.done_rx, 'session done') + end) +end + + + +function tests.test_wire_errors_below_limit_are_counted_without_dropping_session() + fibers.run(function (scope) + local h = start_session(scope, { bad_frame_limit = 3, bad_frame_window_s = 10 }) + recv_with_timeout(h.control_rx, 'initial hello') + admit_frame(h.frame_tx, assert(protocol.hello_ack('mcu-sid', 'mcu'))) + recv_with_timeout(h.rpc_rx, 'peer session') + recv_with_timeout(h.transfer_rx, 'transfer peer session') + + admit_wire_error(h.frame_tx, 'decode_failed: truncated line') + local stale = queue.try_recv_now(h.rpc_rx) + assert_nil(stale, 'single bad frame should not drop session') + + admit_frame(h.frame_tx, assert(protocol.pub({ 'state', 'self' }, { ok = true }, true))) + local routed = recv_with_timeout(h.rpc_rx, 'rpc frame after tolerated bad frame') + assert_eq(routed.kind, 'session_frame') + assert_eq(routed.frame.type, 'pub') + + h.frame_tx:close('done') + local done = recv_with_timeout(h.done_rx, 'session done') + assert_eq(done.snapshot.wire_errors, 1) + assert_eq(done.snapshot.last_wire_error, 'decode_failed: truncated line') + end) +end + +function tests.test_bad_frame_limit_drops_current_peer_session() + fibers.run(function (scope) + local h = start_session(scope, { bad_frame_limit = 2, bad_frame_window_s = 10 }) + recv_with_timeout(h.control_rx, 'initial hello') + admit_frame(h.frame_tx, assert(protocol.hello_ack('mcu-sid', 'mcu'))) + recv_with_timeout(h.rpc_rx, 'peer session') + recv_with_timeout(h.transfer_rx, 'transfer peer session') + + admit_wire_error(h.frame_tx, 'decode_failed: first') + assert_nil(queue.try_recv_now(h.rpc_rx), 'first bad frame should be tolerated') + admit_wire_error(h.frame_tx, 'decode_failed: second') + + local drop = recv_with_timeout(h.rpc_rx, 'peer session drop') + assert_eq(drop.kind, 'peer_session_dropped') + assert_eq(drop.reason, 'bad_frame_limit') + assert_eq(drop.session.peer_sid, 'mcu-sid') + + local hello = recv_with_timeout(h.control_rx, 'hello after bad-frame reset') + assert_eq(hello.frame.type, 'hello') + assert_eq(hello.frame.sid, 'cm5-sid') + + admit_frame(h.frame_tx, assert(protocol.pub({ 'state', 'self' }, { ok = true }, true))) + assert_nil(queue.try_recv_now(h.rpc_rx), 'rpc frame should be dropped after bad-frame session reset') + + h.frame_tx:close('done') + recv_with_timeout(h.done_rx, 'session done') + end) +end + +function tests.test_outbound_gate_accepts_only_current_session_context() + fibers.run(function () + local control_tx, control_rx = mailbox.new(4, { full = 'reject_newest' }) + local rpc_tx, rpc_rx = mailbox.new(4, { full = 'reject_newest' }) + local bulk_tx, bulk_rx = mailbox.new(4, { full = 'reject_newest' }) + local gate = session.new_outbound_gate { + tx_control = control_tx, + tx_rpc = rpc_tx, + tx_bulk = bulk_tx, + } + + local ctx = { session_generation = 1, peer_sid = 'sid-1' } + local rpc_frame = assert(protocol.pub({ 'state', 'self' }, { ok = true }, true)) + local bulk_frame = assert(protocol.xfer_chunk('xfer-1', 0, 'abc', protocol.chunk_digest('abc'))) + + local ok, err = gate:send_rpc_frame_now(ctx, rpc_frame, 'test_rpc_send') + assert_nil(ok) + assert_not_nil(err) + + gate:bind(ctx) + + ok, err = gate:send_rpc_frame_now(ctx, rpc_frame, 'test_rpc_send') + assert_true(ok, err) + local item = recv_with_timeout(rpc_rx, 'gated rpc frame') + assert_eq(item.kind, 'send_frame') + assert_eq(item.lane, 'rpc') + assert_eq(item.frame.type, 'pub') + assert_eq(item.session.session_generation, 1) + assert_eq(item.session.peer_sid, 'sid-1') + assert_nil(queue.try_recv_now(control_rx)) + assert_nil(queue.try_recv_now(bulk_rx)) + + ok, err = gate:send_rpc_frame_now(ctx, bulk_frame, 'test_rpc_wrong_lane') + assert_nil(ok) + assert_not_nil(err) + + ok, err = gate:send_transfer_bulk_frame_now(ctx, bulk_frame, 'test_bulk_send') + assert_true(ok, err) + item = recv_with_timeout(bulk_rx, 'gated bulk frame') + assert_eq(item.lane, 'bulk') + assert_eq(item.frame.type, 'xfer_chunk') + + ok, err = gate:send_rpc_frame_now({ session_generation = 2, peer_sid = 'sid-1' }, rpc_frame, 'test_rpc_send') + assert_nil(ok) + assert_not_nil(err) + + gate:drop('session_dropped') + ok, err = gate:send_rpc_frame_now(ctx, rpc_frame, 'test_rpc_send') + assert_nil(ok) + assert_not_nil(err) + end) +end + + +function tests.test_session_context_deep_copies_identity_and_auth_claims() + local identity = { id = 'peer', nested = { role = 'mcu' } } + local auth = { scheme = 'reserved', nested = { proof = 'claim' } } + local ctx = session.new_session_context({ + link_id = 'link-a', + link_generation = 1, + session_generation = 1, + peer_sid = 'sid-1', + identity_claim = identity, + auth_claim = auth, + }) + + identity.nested.role = 'mutated' + auth.nested.proof = 'mutated' + assert_eq(ctx.identity_claim.nested.role, 'mcu') + assert_eq(ctx.auth_claim.nested.proof, 'claim') + + local copy = session.copy_context(ctx) + copy.identity_claim.nested.role = 'copy-mutated' + copy.auth_claim.nested.proof = 'copy-mutated' + assert_eq(ctx.identity_claim.nested.role, 'mcu') + assert_eq(ctx.auth_claim.nested.proof, 'claim') +end + +return tests diff --git a/tests/unit/fabric/test_state.lua b/tests/unit/fabric/test_state.lua new file mode 100644 index 00000000..5661c972 --- /dev/null +++ b/tests/unit/fabric/test_state.lua @@ -0,0 +1,62 @@ +-- tests/unit/fabric/test_state.lua + +local fibers = require 'fibers' +local busmod = require 'bus' + +local state = require 'services.fabric.state' +local topics = require 'services.fabric.topics' +local probe = require 'tests.support.bus_probe' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil value') end end + +function tests.test_projector_retains_transfer_view_with_correlation() + fibers.run(function (scope) + local bus = busmod.new() + local conn = bus:connect() + local tx, rx = state.new_queue(8) + assert_true(scope:spawn(function () + return state.run_projector(scope, { conn = conn, state_rx = rx }) + end)) + + assert_true(tx:send(state.component_snapshot_event('link-a', 1, 'transfer', { + manager_id = 'link-a:transfer', + last = { + request_id = 'req-1', + request_generation = 1, + direction = 'send', + status = 'ok', + result = { + xfer_id = 'xfer-1', + request_id = 'req-1', + target = 'updater/main', + job_id = 'job-1', + component = 'mcu', + digest_alg = 'xxhash32', + digest = '12345678', + size = 5, + sent_bytes = 5, + retransmits = 1, + }, + }, + }))) + + local payload = probe.wait_retained_payload(conn, topics.state_transfer('xfer-1'), { timeout = 0.3 }) + assert_eq(payload.kind, 'fabric.transfer') + assert_eq(payload.link_id, 'link-a') + assert_eq(payload.xfer_id, 'xfer-1') + assert_eq(payload.state, 'ok') + assert_eq(payload.sent_bytes, 5) + assert_eq(payload.retransmits, 1) + assert_eq(payload.correlation.job_id, 'job-1') + assert_eq(payload.correlation.component, 'mcu') + + tx:close('done') + end) +end + +return tests diff --git a/tests/unit/fabric/test_topics.lua b/tests/unit/fabric/test_topics.lua new file mode 100644 index 00000000..15cf1d37 --- /dev/null +++ b/tests/unit/fabric/test_topics.lua @@ -0,0 +1,174 @@ +-- tests/services/fabric/test_topics.lua + +local topics = require 'services.fabric.topics' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_eq(a, b, msg) + if a ~= b then + fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) + end +end + +local function assert_nil(v, msg) + if v ~= nil then + fail(msg or ('expected nil, got ' .. tostring(v))) + end +end + +local function assert_not_nil(v, msg) + if v == nil then + fail(msg or 'expected non-nil value') + end +end + +local function assert_topic(t, expected) + assert_eq(#t, #expected, 'topic length mismatch') + + for i = 1, #expected do + assert_eq(t[i], expected[i], 'topic token mismatch at ' .. tostring(i)) + end +end + +function tests.test_service_topics() + assert_topic(topics.svc_status(), { 'svc', 'fabric', 'status' }) + assert_topic(topics.svc_meta(), { 'svc', 'fabric', 'meta' }) + assert_topic(topics.cfg(), { 'cfg', 'fabric' }) +end + +function tests.test_state_link_topics() + assert_topic( + topics.state_root(), + { 'state', 'fabric' } + ) + + assert_topic( + topics.state_link('link-a', 'session', 'status'), + { 'state', 'fabric', 'link', 'link-a', 'session', 'status' } + ) + + assert_topic( + topics.state_link_component('link-a', 'reader'), + { 'state', 'fabric', 'link', 'link-a', 'component', 'reader' } + ) +end + +function tests.test_topic_helpers_return_fresh_tables() + local a = topics.svc_status() + local b = topics.svc_status() + + a[1] = 'changed' + + assert_topic(b, { 'svc', 'fabric', 'status' }) +end + +function tests.test_validate_accepts_dense_scalar_topics() + local ok, err = topics.validate({ 'raw', 'member', 7, 'status' }) + + assert_not_nil(ok) + assert_nil(err) +end + +function tests.test_validate_rejects_bad_topics() + local ok, err = topics.validate('not-topic') + + assert_nil(ok) + assert_eq(err, 'topic_must_be_table') + + ok, err = topics.validate({ 'raw', {}, 'bad' }) + + assert_nil(ok) + assert_eq(err, 'invalid_topic_token') + + local sparse = { 'raw', 'member' } + sparse[4] = 'bad' + + ok, err = topics.validate(sparse) + + assert_nil(ok) + assert_eq(err, 'topic_must_be_dense_array') +end + +function tests.test_topic_key_is_collision_safe_and_type_aware() + local a = topics.key({ 'a/b' }) + local b = topics.key({ 'a', 'b' }) + local c = topics.key({ 12 }) + local d = topics.key({ '12' }) + + if a == b then + fail('single-token and two-token topic keys collided') + end + + if c == d then + fail('numeric and string topic keys collided') + end +end + + +function tests.test_topic_mapping_helpers_replace_prefix_and_return_matching_rule() + local rules = { + { + id = 'state-import', + local_prefix = { 'raw', 'member', 'node-a', 'state' }, + remote_prefix = { 'state' }, + }, + } + + local remote, rule = topics.map_local_to_remote(rules, { + 'raw', 'member', 'node-a', 'state', 'battery', 'voltage' + }) + + assert_not_nil(remote) + assert_eq(rule, rules[1]) + assert_topic(remote, { 'state', 'battery', 'voltage' }) + + local local_topic = topics.map_remote_to_local(rules, remote) + assert_topic(local_topic, { 'raw', 'member', 'node-a', 'state', 'battery', 'voltage' }) +end + +function tests.test_topic_mapping_helpers_support_exact_topic_rules() + local rule = { + topic = { 'cap', 'diag', 'rpc', 'read' }, + local_prefix = { 'cap', 'diag', 'rpc', 'read' }, + remote_prefix = { 'rpc', 'diag.read' }, + } + + local mapped, matched = topics.map_local_to_remote_rule(rule, { 'cap', 'diag', 'rpc', 'read' }) + assert_not_nil(mapped) + assert_eq(matched, rule) + assert_topic(mapped, { 'rpc', 'diag.read' }) + + local none = topics.map_local_to_remote_rule(rule, { 'cap', 'diag', 'rpc', 'write' }) + assert_nil(none) +end + + +function tests.test_topic_mapping_helpers_support_exact_rpc_call_rules() + local rules = { + { + id = 'diag-read', + local_topic = { 'cap', 'diag', 'rpc', 'read' }, + remote_topic = { 'rpc', 'diag.read' }, + }, + } + + local remote, rule = topics.map_local_call_to_remote(rules, { 'cap', 'diag', 'rpc', 'read' }) + assert_not_nil(remote) + assert_eq(rule, rules[1]) + assert_topic(remote, { 'rpc', 'diag.read' }) + + local local_topic = topics.map_remote_call_to_local(rules, remote) + assert_topic(local_topic, { 'cap', 'diag', 'rpc', 'read' }) + + local not_remote = topics.map_local_call_to_remote(rules, { 'cap', 'diag', 'rpc', 'read', 'extra' }) + assert_nil(not_remote) + + local not_local = topics.map_remote_call_to_local(rules, { 'rpc', 'diag.read', 'extra' }) + assert_nil(not_local) +end + +return tests diff --git a/tests/unit/fabric/test_transfer.lua b/tests/unit/fabric/test_transfer.lua new file mode 100644 index 00000000..d0798c23 --- /dev/null +++ b/tests/unit/fabric/test_transfer.lua @@ -0,0 +1,438 @@ +-- tests/unit/fabric/test_transfer.lua +-- +-- Tests for the transfer manager after removing injected attempt callbacks. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' + +local transfer = require 'services.fabric.transfer' +local session = require 'services.fabric.session' +local protocol = require 'services.fabric.protocol' +local queue = require 'devicecode.support.queue' +local resource = require 'devicecode.support.resource' +local blob = require 'devicecode.blob_source' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil value') end end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end + +local function recv_with_timeout(rx, label, timeout) + timeout = timeout or 0.25 + local which, item = fibers.perform(fibers.named_choice{ + item = rx:recv_op(), + timeout = sleep.sleep_op(timeout), + }) + if which == 'timeout' then fail('timed out waiting for ' .. tostring(label or 'item')) end + return item +end + +local function wait_transfer_last_status(h, wanted, label, timeout) + timeout = timeout or 0.25 + local deadline = fibers.now() + timeout + while fibers.now() < deadline do + local remaining = deadline - fibers.now() + local which, ev = fibers.perform(fibers.named_choice{ + event = h.state_rx:recv_op(), + timeout = sleep.sleep_op(remaining), + }) + if which == 'timeout' then break end + local snap = ev and ev.snapshot + local last = snap and snap.last + if last and last.status == wanted then + return snap + end + end + fail('timed out waiting for transfer last status ' .. tostring(wanted) .. ' for ' .. tostring(label or 'transfer')) +end + +local function ctx(gen, sid) + return session.new_session_context { + link_id = 'link-a', + link_generation = 1, + session_generation = gen or 1, + peer_sid = sid or 'sid-1', + peer_node = 'mcu', + proto = protocol.PROTO, + } +end + +local function peer_session_event(gen, sid) + return { kind = 'peer_session', session = ctx(gen, sid), at = fibers.now() } +end + +local function transfer_frame_event(frame, gen, sid) + return { kind = 'session_frame', lane = 'transfer', session = ctx(gen, sid), frame = frame, at = fibers.now() } +end + +local function slot_request(id) + local r = { request_id = id or 'req-1', request_generation = 1, xfer_id = 'xfer-1', result = nil } + function r:reply(v) self.result = { ok = true, value = v }; return true end + function r:fail(e) self.result = { ok = false, err = e }; return true end + return r +end + +local function start_manager(scope, opts) + opts = opts or {} + local admission_tx, admission_rx = mailbox.new(8, { full = 'reject_newest' }) + local session_tx, session_rx = mailbox.new(32, { full = 'reject_newest' }) + local state_tx, state_rx = mailbox.new(32, { full = 'reject_newest' }) + local outbound_control_tx, outbound_control_rx = mailbox.new(32, { full = 'reject_newest' }) + local outbound_rpc_tx, outbound_rpc_rx = mailbox.new(32, { full = 'reject_newest' }) + local outbound_bulk_tx, outbound_bulk_rx = mailbox.new(32, { full = 'reject_newest' }) + local outbound = session.new_outbound_gate { + tx_control = outbound_control_tx, + tx_rpc = outbound_rpc_tx, + tx_bulk = outbound_bulk_tx, + } + local done_tx, done_rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = scope:spawn(function () + local result = transfer.run(scope, { + manager_id = 'transfer-test', + link_id = 'link-a', + link_generation = 1, + admission_rx = admission_rx, + session_rx = session_rx, + outbound = outbound, + state_tx = state_tx, + chunk_size = opts.chunk_size or 3, + timeout_s = opts.timeout_s or 0.05, + retry_limit = opts.retry_limit, + receive_targets = opts.receive_targets, + }) + queue.try_admit_required(done_tx, result, 'transfer_done') + end) + assert_true(ok, err) + + return { + admission_tx = admission_tx, + session_tx = session_tx, + state_rx = state_rx, + control_rx = outbound_control_rx, + rpc_rx = outbound_rpc_rx, + bulk_rx = outbound_bulk_rx, + outbound = outbound, + done_rx = done_rx, + } +end + +function tests.test_reducer_requires_session_context_for_claims() + local state = transfer.new_state { manager_id = 'm' } + local ok, err = pcall(function () + transfer.claim_slot(state, { request_id = 'r1', request_generation = 1 }) + end) + assert_eq(ok, false) + assert_not_nil(err) + + local accepted = transfer.claim_slot(state, { + request_id = 'r1', + request_generation = 1, + session = ctx(), + xfer_id = 'xfer-1', + }) + assert_eq(accepted, true) + assert_eq(transfer.snapshot(state).active.session.peer_sid, 'sid-1') +end + +function tests.test_slot_admission_without_session_waits_for_peer_session() + fibers.run(function (scope) + local h = start_manager(scope) + local req = slot_request('req-wait-session') + assert_true(h.admission_tx:send(req)) + fibers.perform(sleep.sleep_op(0.02)) + assert_nil(req.result, 'slot request should wait while no peer session exists') + + local c = ctx() + h.outbound:bind(c) + assert_true(h.session_tx:send(peer_session_event())) + local deadline = fibers.now() + 0.1 + while req.result == nil and fibers.now() < deadline do fibers.perform(sleep.sleep_op(0.001)) end + assert_eq(req.result.ok, true) + assert_not_nil(req.result.value.lease) + assert_true(req.result.value.lease:release('not used')) + + h.admission_tx:close('done') + h.session_tx:close('done') + local done = recv_with_timeout(h.done_rx, 'manager done') + assert_eq(done.snapshot.stats.deferred_no_session, 1) + end) +end + +function tests.test_slot_admission_grants_lease_and_release_clears_slot() + fibers.run(function (scope) + local h = start_manager(scope) + local c = ctx() + h.outbound:bind(c) + assert_true(h.session_tx:send(peer_session_event())) + local req = slot_request('req-lease') + assert_true(h.admission_tx:send(req)) + local deadline = fibers.now() + 0.1 + while req.result == nil and fibers.now() < deadline do fibers.perform(sleep.sleep_op(0.001)) end + assert_eq(req.result.ok, true) + assert_not_nil(req.result.value.lease) + assert_true(req.result.value.lease:release('not used')) + h.admission_tx:close('done') + h.session_tx:close('done') + local done = recv_with_timeout(h.done_rx, 'manager done') + assert_eq(done.snapshot.active, nil) + assert_eq(done.snapshot.stats.released, 1) + end) +end + +function tests.test_real_sender_attempt_uses_session_bound_outbound_gate() + fibers.run(function (scope) + local h = start_manager(scope, { chunk_size = 3 }) + local c = ctx() + h.outbound:bind(c) + assert_true(h.session_tx:send(peer_session_event())) + + local req = slot_request('req-send') + assert_true(h.admission_tx:send(req)) + local deadline = fibers.now() + 0.1 + while req.result == nil and fibers.now() < deadline do fibers.perform(sleep.sleep_op(0.001)) end + local lease = req.result.value.lease + assert_not_nil(lease) + + local source_owner = resource.owned(blob.from_string('abcdef')) + local attempt, err = transfer.start_attempt(scope, lease, { + request_id = 'req-send', + request_generation = 1, + source_owner = source_owner, + target = 'updater/main', + xfer_id = 'xfer-1', + size = 6, + digest_alg = protocol.DIGEST_ALG, + digest = protocol.digest_hex('abcdef'), + }) + assert_not_nil(attempt, err) + + local begin = recv_with_timeout(h.control_rx, 'xfer begin').frame + assert_eq(begin.type, 'xfer_begin') + assert_eq(begin.target, 'updater/main') + + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_ready('xfer-1'))))) + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_need('xfer-1', 0))))) + local chunk1 = recv_with_timeout(h.bulk_rx, 'chunk 1').frame + assert_eq(chunk1.type, 'xfer_chunk') + assert_eq(chunk1.offset, 0) + assert_eq(chunk1.data, 'abc') + + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_need('xfer-1', 3))))) + local chunk2 = recv_with_timeout(h.bulk_rx, 'chunk 2').frame + assert_eq(chunk2.offset, 3) + assert_eq(chunk2.data, 'def') + + -- The receiver acknowledges the final accepted chunk with next == size. + -- The sender must wait for this before sending xfer_commit so that + -- control commit cannot overtake the final bulk frame. + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_need('xfer-1', 6))))) + local commit = recv_with_timeout(h.control_rx, 'commit').frame + assert_eq(commit.type, 'xfer_commit') + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_done('xfer-1'))))) + + local ev = recv_with_timeout({ recv_op = function () return attempt:outcome_op() end }, 'attempt outcome') + assert_eq(ev.status, 'ok') + assert_eq(ev.result.sent_bytes, 6) + + h.admission_tx:close('done') + h.session_tx:close('done') + recv_with_timeout(h.done_rx, 'manager done') + end) +end + +function tests.test_session_drop_poisoned_lease_cannot_start_attempt() + fibers.run(function (scope) + local h = start_manager(scope) + local c = ctx() + h.outbound:bind(c) + assert_true(h.session_tx:send(peer_session_event())) + local req = slot_request('req-drop') + assert_true(h.admission_tx:send(req)) + local deadline = fibers.now() + 0.1 + while req.result == nil and fibers.now() < deadline do fibers.perform(sleep.sleep_op(0.001)) end + local lease = req.result.value.lease + assert_true(h.session_tx:send({ kind = 'peer_session_dropped', session = c, reason = 'drop' })) + fibers.perform(sleep.sleep_op(0.01)) + local h2, err = transfer.start_attempt(scope, lease, {}) + assert_nil(h2) + assert_not_nil(err) + h.admission_tx:close('done') + h.session_tx:close('done') + recv_with_timeout(h.done_rx, 'manager done') + end) +end + +function tests.test_manager_receives_inbound_transfer_for_registered_target() + fibers.run(function (scope) + local received = {} + local committed = false + local target = {} + function target:open_sink_op(req) + assert_eq(req.target, 'updater/main') + assert_eq(req.size, 6) + local sink = {} + function sink:append_op(chunk) + received[#received + 1] = chunk + return fibers.always(true, nil) + end + function sink:commit_op(req2) + committed = true + assert_eq(req2.digest, protocol.digest_hex('abcdef')) + return fibers.always({ staged = true }, nil) + end + function sink:abort(_) return true, nil end + return fibers.always(sink, nil) + end + + local h = start_manager(scope, { + chunk_size = 3, + receive_targets = { ['updater/main'] = target }, + }) + local c = ctx() + h.outbound:bind(c) + assert_true(h.session_tx:send(peer_session_event())) + + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_begin( + 'xfer-in-1', 'updater/main', 6, protocol.DIGEST_ALG, protocol.digest_hex('abcdef'), + { kind = 'firmware', component = 'mcu' } + ))))) + + local ready = recv_with_timeout(h.control_rx, 'receive ready').frame + assert_eq(ready.type, 'xfer_ready') + assert_eq(ready.xfer_id, 'xfer-in-1') + + local need0 = recv_with_timeout(h.control_rx, 'receive need 0').frame + assert_eq(need0.type, 'xfer_need') + assert_eq(need0.next, 0) + + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_chunk( + 'xfer-in-1', 0, 'abc', protocol.chunk_digest('abc') + ))))) + local need3 = recv_with_timeout(h.control_rx, 'receive need 3').frame + assert_eq(need3.type, 'xfer_need') + assert_eq(need3.next, 3) + + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_chunk( + 'xfer-in-1', 3, 'def', protocol.chunk_digest('def') + ))))) + local need6 = recv_with_timeout(h.control_rx, 'receive need 6').frame + assert_eq(need6.type, 'xfer_need') + assert_eq(need6.next, 6) + + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_commit( + 'xfer-in-1', 6, protocol.DIGEST_ALG, protocol.digest_hex('abcdef') + ))))) + + local done = recv_with_timeout(h.control_rx, 'receive done').frame + assert_eq(done.type, 'xfer_done') + assert_eq(table.concat(received), 'abcdef') + assert_true(committed) + + local completed = wait_transfer_last_status(h, 'ok', 'inbound receive') + assert_eq(completed.last.result.received_bytes, 6) + + h.admission_tx:close('done') + h.session_tx:close('done') + local result = recv_with_timeout(h.done_rx, 'manager done') + assert_eq(result.snapshot.last.status, 'ok') + assert_eq(result.snapshot.last.result.received_bytes, 6) + end) +end + +function tests.test_manager_reasks_same_offset_after_bad_chunk_digest_and_accepts_retry() + fibers.run(function (scope) + local received = {} + local target = {} + function target:open_sink_op(req) + assert_eq(req.target, 'updater/main') + local sink = {} + function sink:append_op(chunk) + received[#received + 1] = chunk + return fibers.always(true, nil) + end + function sink:commit_op(_) + return fibers.always({ staged = true }, nil) + end + function sink:abort(_) return true, nil end + return fibers.always(sink, nil) + end + + local h = start_manager(scope, { + chunk_size = 3, + retry_limit = 1, + receive_targets = { ['updater/main'] = target }, + }) + local c = ctx() + h.outbound:bind(c) + assert_true(h.session_tx:send(peer_session_event())) + + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_begin( + 'xfer-in-retry', 'updater/main', 3, protocol.DIGEST_ALG, protocol.digest_hex('abc'), nil + ))))) + + assert_eq(recv_with_timeout(h.control_rx, 'ready').frame.type, 'xfer_ready') + local need0 = recv_with_timeout(h.control_rx, 'need 0').frame + assert_eq(need0.type, 'xfer_need') + assert_eq(need0.next, 0) + + -- A bad digest for the expected offset should not advance the sink. + assert_true(h.session_tx:send(transfer_frame_event({ + type = 'xfer_chunk', + xfer_id = 'xfer-in-retry', + offset = 0, + data = 'abc', + chunk_digest = '00000000', + }))) + local retry_need = recv_with_timeout(h.control_rx, 'retry need 0').frame + assert_eq(retry_need.type, 'xfer_need') + assert_eq(retry_need.next, 0) + assert_eq(#received, 0) + + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_chunk( + 'xfer-in-retry', 0, 'abc', protocol.chunk_digest('abc') + ))))) + local need3 = recv_with_timeout(h.control_rx, 'need 3').frame + assert_eq(need3.type, 'xfer_need') + assert_eq(need3.next, 3) + + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_commit( + 'xfer-in-retry', 3, protocol.DIGEST_ALG, protocol.digest_hex('abc') + ))))) + local done = recv_with_timeout(h.control_rx, 'done').frame + assert_eq(done.type, 'xfer_done') + assert_eq(table.concat(received), 'abc') + + local completed = wait_transfer_last_status(h, 'ok', 'inbound retry') + assert_eq(completed.last.result.chunk_retries, 1) + + h.admission_tx:close('done') + h.session_tx:close('done') + recv_with_timeout(h.done_rx, 'manager done') + end) +end + +function tests.test_manager_aborts_inbound_transfer_for_unknown_target() + fibers.run(function (scope) + local h = start_manager(scope) + local c = ctx() + h.outbound:bind(c) + assert_true(h.session_tx:send(peer_session_event())) + assert_true(h.session_tx:send(transfer_frame_event(assert(protocol.xfer_begin( + 'xfer-unknown', 'updater/main', 0, protocol.DIGEST_ALG, protocol.digest_hex(''), nil + ))))) + local abort = recv_with_timeout(h.control_rx, 'unsupported target abort').frame + assert_eq(abort.type, 'xfer_abort') + assert_eq(abort.xfer_id, 'xfer-unknown') + assert_eq(abort.err, 'unsupported_target') + h.admission_tx:close('done') + h.session_tx:close('done') + recv_with_timeout(h.done_rx, 'manager done') + end) +end + +return tests diff --git a/tests/unit/fabric/test_transfer_sender.lua b/tests/unit/fabric/test_transfer_sender.lua new file mode 100644 index 00000000..f0391f6e --- /dev/null +++ b/tests/unit/fabric/test_transfer_sender.lua @@ -0,0 +1,470 @@ +-- tests/unit/fabric/test_transfer_sender.lua +-- +-- Contract tests for the doctrinal transfer sender worker. + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' +local op = require 'fibers.op' + +local transfer_sender = require 'services.fabric.transfer_sender' +local protocol = require 'services.fabric.protocol' +local blob = require 'devicecode.blob_source' +local queue = require 'devicecode.support.queue' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_true(v, msg) + if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end +end + +local function assert_eq(a, b, msg) + if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end +end + +local function assert_match(s, pat, msg) + if type(s) ~= 'string' or not s:match(pat) then + fail(msg or ('expected "' .. tostring(s) .. '" to match ' .. tostring(pat))) + end +end + +local function recv_with_timeout(rx, label, timeout) + timeout = timeout or 0.25 + + local which, item = fibers.perform(fibers.named_choice { + item = rx:recv_op(), + timeout = sleep.sleep_op(timeout), + }) + + if which == 'timeout' then + fail('timed out waiting for ' .. tostring(label or 'item')) + end + + return item +end + +local function send_frame(tx, frame) + assert_true(tx:send({ frame = frame })) +end + +local function make_sender_caps(control_tx, bulk_tx, frame_rx, opts) + opts = opts or {} + + local function admit(tx, frame, label) + return queue.try_admit_required( + tx, + { kind = 'send_frame', at = fibers.now(), frame = frame }, + label or 'transfer_sender_test_send_failed' + ) + end + + return { + frame_rx = frame_rx, + chunk_size = opts.chunk_size or 3, + timeout_s = opts.timeout_s or 0.05, + begin_retry_s = opts.begin_retry_s, + commit_retry_limit = opts.commit_retry_limit, + + send_control_frame_now = function (frame, label) + return admit(control_tx, frame, label) + end, + + send_bulk_frame_now = function (frame, label) + return admit(bulk_tx, frame, label) + end, + } +end + +local function collect_result(req, driver, opts) + opts = opts or {} + local out = {} + + fibers.run(function () + local st, rep, value = fibers.run_scope(function (scope) + local control_tx, control_rx = mailbox.new(opts.control_queue_len or 32, { full = 'reject_newest' }) + local bulk_tx, bulk_rx = mailbox.new(opts.bulk_queue_len or 32, { full = 'reject_newest' }) + local frame_tx, frame_rx = mailbox.new(opts.frame_queue_len or 32, { full = 'reject_newest' }) + + out.control_rx = control_rx + out.bulk_rx = bulk_rx + out.frame_tx = frame_tx + + local ok, err = scope:spawn(function () + driver({ + control_rx = control_rx, + bulk_rx = bulk_rx, + frame_tx = frame_tx, + }) + end) + assert_true(ok, err) + + return transfer_sender.run( + scope, + req, + make_sender_caps(control_tx, bulk_tx, frame_rx, opts) + ) + end) + + out.status = st + out.report = rep + out.value = value + end) + + return out +end + +local function make_req(overrides) + overrides = overrides or {} + + local data = overrides.data + if data == nil and overrides.source == nil then + data = 'abcdef' + end + + local source = overrides.source + if source == nil and type(data) == 'string' then + source = blob.from_string(data) + end + + local size = overrides.size + if size == nil and type(data) == 'string' then + size = #data + end + + local digest_data = overrides.digest_data + if digest_data == nil and type(data) == 'string' then + digest_data = data + end + if digest_data == nil then + -- Source-backed tests that do not expose their bytes should set digest + -- explicitly if the exact value matters. This default is only to satisfy + -- the sender's xxhash32 request validation. + digest_data = 'abcdef' + end + + local req = { + request_id = overrides.request_id or 'req-1', + target = overrides.target or 'remote.stage.main', + source = source, + data = data, + size = size, + digest_alg = overrides.digest_alg or protocol.DIGEST_ALG, + digest = overrides.digest or protocol.digest_hex(digest_data), + timeout_s = overrides.timeout_s, + xfer_id = overrides.xfer_id, + meta = overrides.meta, + } + + for k, v in pairs(overrides) do + req[k] = v + end + + return req +end + +function tests.test_sender_sends_begin_chunks_commit_and_returns_after_done() + local seen = {} + local req = make_req { data = 'abcdef', size = 6, xfer_id = 'xfer-1' } + + local out = collect_result(req, function (io) + local begin = recv_with_timeout(io.control_rx, 'begin') + assert_eq(begin.frame.type, 'xfer_begin') + assert_eq(begin.frame.xfer_id, 'xfer-1') + assert_eq(begin.frame.target, 'remote.stage.main') + assert_eq(begin.frame.size, 6) + assert_eq(begin.frame.digest_alg, protocol.DIGEST_ALG) + assert_eq(begin.frame.digest, protocol.digest_hex('abcdef')) + seen[#seen + 1] = begin.frame.type + + send_frame(io.frame_tx, assert(protocol.xfer_ready('xfer-1'))) + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-1', 0))) + + local chunk1 = recv_with_timeout(io.bulk_rx, 'chunk1') + assert_eq(chunk1.frame.type, 'xfer_chunk') + assert_eq(chunk1.frame.offset, 0) + assert_eq(chunk1.frame.data, 'abc') + assert_eq(chunk1.frame.chunk_digest, protocol.chunk_digest('abc')) + seen[#seen + 1] = chunk1.frame.type + + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-1', 3))) + + local chunk2 = recv_with_timeout(io.bulk_rx, 'chunk2') + assert_eq(chunk2.frame.offset, 3) + assert_eq(chunk2.frame.data, 'def') + assert_eq(chunk2.frame.chunk_digest, protocol.chunk_digest('def')) + seen[#seen + 1] = chunk2.frame.type + + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-1', 6))) + + local commit = recv_with_timeout(io.control_rx, 'commit') + assert_eq(commit.frame.type, 'xfer_commit') + assert_eq(commit.frame.size, 6) + assert_eq(commit.frame.digest_alg, protocol.DIGEST_ALG) + assert_eq(commit.frame.digest, protocol.digest_hex('abcdef')) + seen[#seen + 1] = commit.frame.type + + send_frame(io.frame_tx, assert(protocol.xfer_done('xfer-1'))) + end) + + assert_eq(out.status, 'ok', tostring(out.value)) + assert_eq(out.value.xfer_id, 'xfer-1') + assert_eq(out.value.sent_bytes, 6) + assert_eq(out.value.target, 'remote.stage.main') + assert_eq(out.value.digest_alg, protocol.DIGEST_ALG) + assert_eq(out.value.digest, protocol.digest_hex('abcdef')) + assert_eq(seen[1], 'xfer_begin') + assert_eq(seen[2], 'xfer_chunk') + assert_eq(seen[3], 'xfer_chunk') + assert_eq(seen[4], 'xfer_commit') +end + +function tests.test_sender_resends_cached_chunk_when_receiver_reasks_same_offset() + local req = make_req { data = 'abcdef', size = 6, xfer_id = 'xfer-retry' } + + local out = collect_result(req, function (io) + recv_with_timeout(io.control_rx, 'begin') + send_frame(io.frame_tx, assert(protocol.xfer_ready('xfer-retry'))) + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-retry', 0))) + + local chunk1 = recv_with_timeout(io.bulk_rx, 'chunk1') + assert_eq(chunk1.frame.offset, 0) + assert_eq(chunk1.frame.data, 'abc') + + -- Same offset means the receiver did not advance. The sender must + -- resend the cached frame without reading from the source again. + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-retry', 0))) + local retry1 = recv_with_timeout(io.bulk_rx, 'chunk1 retry') + assert_eq(retry1.frame.offset, 0) + assert_eq(retry1.frame.data, 'abc') + + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-retry', 3))) + local chunk2 = recv_with_timeout(io.bulk_rx, 'chunk2') + assert_eq(chunk2.frame.offset, 3) + assert_eq(chunk2.frame.data, 'def') + + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-retry', 6))) + local commit = recv_with_timeout(io.control_rx, 'commit') + assert_eq(commit.frame.type, 'xfer_commit') + send_frame(io.frame_tx, assert(protocol.xfer_done('xfer-retry'))) + end) + + assert_eq(out.status, 'ok', tostring(out.value)) + assert_eq(out.value.sent_bytes, 6) + assert_eq(out.value.retransmits, 1) +end + +function tests.test_sender_timeout_sends_abort_and_fails_attempt() + local req = make_req { data = 'abc', size = 3, xfer_id = 'xfer-timeout', timeout_s = 0.01 } + + local out = collect_result(req, function (io) + local begin = recv_with_timeout(io.control_rx, 'begin') + assert_eq(begin.frame.type, 'xfer_begin') + end, { timeout_s = 0.01 }) + + assert_eq(out.status, 'failed') + assert_match(out.value, 'timeout') +end + +function tests.test_sender_remote_abort_fails_without_echoing_abort() + local req = make_req { data = 'abc', size = 3, xfer_id = 'xfer-abort' } + + local out = collect_result(req, function (io) + recv_with_timeout(io.control_rx, 'begin') + send_frame(io.frame_tx, assert(protocol.xfer_abort('xfer-abort', 'remote denied'))) + end) + + assert_eq(out.status, 'failed') + assert_match(out.value, 'remote denied') +end + +function tests.test_sender_unexpected_offset_sends_abort_and_fails() + local req = make_req { data = 'abc', size = 3, xfer_id = 'xfer-offset' } + + local out = collect_result(req, function (io) + recv_with_timeout(io.control_rx, 'begin') + send_frame(io.frame_tx, assert(protocol.xfer_ready('xfer-offset'))) + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-offset', 1))) + end) + + assert_eq(out.status, 'failed') + assert_match(out.value, 'unexpected_offset') +end + +function tests.test_sender_source_read_error_sends_abort_and_fails() + local source = { + read_chunk_op = function () + return op.always(nil, 'read boom') + end, + } + + local req = make_req { + source = source, + data = nil, + size = 3, + xfer_id = 'xfer-read-error', + digest = protocol.digest_hex('abc'), + } + + local out = collect_result(req, function (io) + recv_with_timeout(io.control_rx, 'begin') + send_frame(io.frame_tx, assert(protocol.xfer_ready('xfer-read-error'))) + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-read-error', 0))) + end) + + assert_eq(out.status, 'failed') + assert_match(out.value, 'read boom') +end + +function tests.test_sender_does_not_close_source_itself() + local source = blob.from_string('abc') + local closed = 0 + + function source:close(reason) + closed = closed + 1 + self.close_reason = reason + return true + end + + local req = make_req { + source = source, + data = nil, + size = 3, + xfer_id = 'xfer-no-close', + digest = protocol.digest_hex('abc'), + } + + local out = collect_result(req, function (io) + recv_with_timeout(io.control_rx, 'begin') + send_frame(io.frame_tx, assert(protocol.xfer_ready('xfer-no-close'))) + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-no-close', 0))) + recv_with_timeout(io.bulk_rx, 'chunk') + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-no-close', 3))) + recv_with_timeout(io.control_rx, 'commit') + send_frame(io.frame_tx, assert(protocol.xfer_done('xfer-no-close'))) + end) + + assert_eq(out.status, 'ok', tostring(out.value)) + assert_eq(closed, 0) +end + +function tests.test_sender_accepts_zero_byte_transfer_commit_after_need_zero() + local req = make_req { data = '', size = 0, xfer_id = 'xfer-zero' } + + local out = collect_result(req, function (io) + recv_with_timeout(io.control_rx, 'begin') + send_frame(io.frame_tx, assert(protocol.xfer_ready('xfer-zero'))) + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-zero', 0))) + + local commit = recv_with_timeout(io.control_rx, 'commit') + assert_eq(commit.frame.type, 'xfer_commit') + assert_eq(commit.frame.size, 0) + + send_frame(io.frame_tx, assert(protocol.xfer_done('xfer-zero'))) + end) + + assert_eq(out.status, 'ok', tostring(out.value)) + assert_eq(out.value.sent_bytes, 0) +end + + +function tests.test_sender_does_not_send_first_chunk_implicitly_after_ready() + local req = make_req { data = 'abc', size = 3, xfer_id = 'xfer-demand-driven', timeout_s = 0.03 } + + local out = collect_result(req, function (io) + recv_with_timeout(io.control_rx, 'begin') + send_frame(io.frame_tx, assert(protocol.xfer_ready('xfer-demand-driven'))) + + local which = fibers.perform(fibers.named_choice { + bulk = io.bulk_rx:recv_op(), + timeout = sleep.sleep_op(0.01), + }) + assert_eq(which, 'timeout', 'sender must wait for xfer_need before sending bulk') + end, { timeout_s = 0.03 }) + + assert_eq(out.status, 'failed') + assert_match(out.value, 'timeout') +end + + +function tests.test_sender_ignores_late_retry_need_after_commit() + local req = make_req { data = 'abcdef', size = 6, xfer_id = 'xfer-late-need' } + + local out = collect_result(req, function (io) + recv_with_timeout(io.control_rx, 'begin') + send_frame(io.frame_tx, assert(protocol.xfer_ready('xfer-late-need'))) + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-late-need', 0))) + recv_with_timeout(io.bulk_rx, 'chunk1') + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-late-need', 3))) + recv_with_timeout(io.bulk_rx, 'chunk2') + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-late-need', 6))) + local commit = recv_with_timeout(io.control_rx, 'commit') + assert_eq(commit.frame.type, 'xfer_commit') + + -- A retry xfer_need emitted for earlier active-transfer corruption may + -- arrive after the receiver has already accepted all bytes and after the + -- sender has committed. It is stale and must not fail the attempt. + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-late-need', 0, true, 'bad_json'))) + send_frame(io.frame_tx, assert(protocol.xfer_done('xfer-late-need'))) + end) + + assert_eq(out.status, 'ok', tostring(out.value)) + assert_eq(out.value.sent_bytes, 6) +end + + +function tests.test_sender_retries_begin_while_waiting_for_ready() + local req = make_req { data = 'abc', size = 3, xfer_id = 'xfer-begin-retry' } + + local out = collect_result(req, function (io) + local begin1 = recv_with_timeout(io.control_rx, 'begin1') + assert_eq(begin1.frame.type, 'xfer_begin') + local begin2 = recv_with_timeout(io.control_rx, 'begin retry') + assert_eq(begin2.frame.type, 'xfer_begin') + assert_eq(begin2.frame.xfer_id, 'xfer-begin-retry') + + send_frame(io.frame_tx, assert(protocol.xfer_ready('xfer-begin-retry'))) + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-begin-retry', 0))) + recv_with_timeout(io.bulk_rx, 'chunk') + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-begin-retry', 3))) + recv_with_timeout(io.control_rx, 'commit') + send_frame(io.frame_tx, assert(protocol.xfer_done('xfer-begin-retry'))) + end, { timeout_s = 0.12, begin_retry_s = 0.01 }) + + assert_eq(out.status, 'ok', tostring(out.value)) + assert_eq(out.value.begin_retries >= 1, true, 'expected at least one xfer_begin retry') +end + +function tests.test_sender_resends_commit_on_retry_need_at_eof_while_committing() + local req = make_req { data = 'abcdef', size = 6, xfer_id = 'xfer-commit-retry' } + + local out = collect_result(req, function (io) + recv_with_timeout(io.control_rx, 'begin') + send_frame(io.frame_tx, assert(protocol.xfer_ready('xfer-commit-retry'))) + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-commit-retry', 0))) + recv_with_timeout(io.bulk_rx, 'chunk1') + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-commit-retry', 3))) + recv_with_timeout(io.bulk_rx, 'chunk2') + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-commit-retry', 6))) + local commit1 = recv_with_timeout(io.control_rx, 'commit1') + assert_eq(commit1.frame.type, 'xfer_commit') + + -- The receiver did receive all bytes, but the commit frame was damaged. + -- A retry xfer_need at EOF must retransmit xfer_commit rather than being + -- discarded as stale. + send_frame(io.frame_tx, assert(protocol.xfer_need('xfer-commit-retry', 6, true, 'bad_json'))) + local commit2 = recv_with_timeout(io.control_rx, 'commit retry') + assert_eq(commit2.frame.type, 'xfer_commit') + assert_eq(commit2.frame.xfer_id, 'xfer-commit-retry') + + send_frame(io.frame_tx, assert(protocol.xfer_done('xfer-commit-retry'))) + end) + + assert_eq(out.status, 'ok', tostring(out.value)) + assert_eq(out.value.commit_retries, 1) +end + +return tests diff --git a/tests/unit/gsm/test_apn.lua b/tests/unit/gsm/test_apn.lua new file mode 100644 index 00000000..10099971 --- /dev/null +++ b/tests/unit/gsm/test_apn.lua @@ -0,0 +1,25 @@ +local apn = require 'services.gsm.apn' + +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function ok(v,msg) if not v then fail(msg or 'expected truthy') end end + +function tests.test_custom_apn_is_ranked_before_default() + local ranked, rankings = apn.get_ranked_apns('234', '10', '234100000000000', nil, nil, { + { carrier='Custom', mcc='234', mnc='10', apn='custom.net' }, + }) + ok(rankings[1]) + eq(rankings[1].name, 'custom-1') + eq(ranked['custom-1'].apn, 'custom.net') +end + +function tests.test_connection_string_uses_allowed_auth_when_present() + local s, err = apn.build_connection_string({ apn='internet', user='u', password='p', authtype='1' }, true) + ok(s, err) + ok(s:find('apn=internet', 1, true)) + ok(s:find('allowed-auth=pap', 1, true)) + ok(s:find('allow-roaming=true', 1, true)) +end + +return tests diff --git a/tests/unit/gsm/test_apn_model.lua b/tests/unit/gsm/test_apn_model.lua new file mode 100644 index 00000000..11ea0d2d --- /dev/null +++ b/tests/unit/gsm/test_apn_model.lua @@ -0,0 +1,61 @@ +local model = require 'services.gsm.apn_model' + +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function ok(v,msg) if not v then fail(msg or 'expected truthy') end end + +function tests.test_normalise_record_requires_core_fields() + local rec, err = model.normalise_record({ carrier=' Test ', mcc='234', mnc='10', apn=' internet ' }) + ok(rec, err) + eq(rec.carrier, 'Test') + eq(rec.apn, 'internet') + local bad = model.normalise_record({ carrier='x', mcc='23', mnc='10', apn='internet' }) + if bad then fail('invalid MCC accepted') end +end + +function tests.test_normalise_record_accepts_apn_editor_payload() + local rec, err = model.normalise_record({ + carrier = 'Demo Carrier', + mcc = '234', + mnc = '10', + apn = 'custom-apn', + authtype = 'none', + user = 'demo-user', + password = 'demo-password', + }) + ok(rec, err) + eq(rec.carrier, 'Demo Carrier') + eq(rec.mcc, '234') + eq(rec.mnc, '10') + eq(rec.apn, 'custom-apn') + eq(rec.authtype, 'none') + eq(rec.user, 'demo-user') + eq(rec.password, 'demo-password') +end + +function tests.test_normalise_list_rejects_duplicates() + local list, err = model.normalise_list({ + { carrier='A', mcc='234', mnc='10', apn='internet' }, + { carrier='A', mcc='234', mnc='10', apn='internet' }, + }) + if list then fail('duplicate accepted') end + ok(err and err:find('duplicate', 1, true)) +end + + +function tests.test_redact_list_removes_apn_secrets_but_preserves_presence_flags() + local redacted = model.redact_list({ + { carrier='A', mcc='234', mnc='10', apn='internet', user='alice', password='secret', auth='token' }, + }) + eq(redacted[1].carrier, 'A') + eq(redacted[1].apn, 'internet') + eq(redacted[1].user, nil) + eq(redacted[1].password, nil) + eq(redacted[1].auth, nil) + eq(redacted[1].has_user, true) + eq(redacted[1].has_password, true) + eq(redacted[1].has_auth, true) +end + +return tests diff --git a/tests/unit/gsm/test_apn_store_control_store.lua b/tests/unit/gsm/test_apn_store_control_store.lua new file mode 100644 index 00000000..0622ca05 --- /dev/null +++ b/tests/unit/gsm/test_apn_store_control_store.lua @@ -0,0 +1,47 @@ +local fibers = require 'fibers' +local op = require 'fibers.op' +local runfibers = require 'tests.support.run_fibers' +local store_mod = require 'services.gsm.apn_store_control_store' + +local tests = {} + +local function fake_conn() + local data = {} + return { + data = data, + call_op = function(_, topic, payload) + local method = topic[5] + if method == 'get' then + if data[payload.key] == nil then return op.always({ ok = false, reason = 'not found' }, nil) end + return op.always({ ok = true, reason = data[payload.key] }, nil) + elseif method == 'put' then + data[payload.key] = payload.data + return op.always({ ok = true, reason = nil }, nil) + end + return op.always({ ok = false, reason = 'bad_method' }, nil) + end, + } +end + +function tests.test_save_accepts_void_successful_control_store_put() + runfibers.run(function() + local conn = fake_conn() + local store = store_mod.new(conn) + + local saved, save_err = fibers.perform(store:save_op({ + { carrier = 'O2 - UK', mcc = '234', mnc = '10', apn = 'payandgo.o2.co.uk' }, + })) + assert(saved ~= nil, tostring(save_err)) + assert(saved[1].apn == 'payandgo.o2.co.uk') + assert(conn.data['custom-apns-v1'] ~= nil) + + local loaded, load_err = fibers.perform(store:load_op()) + assert(loaded ~= nil, tostring(load_err)) + assert(loaded[1].carrier == 'O2 - UK') + assert(loaded[1].mcc == '234') + assert(loaded[1].mnc == '10') + assert(loaded[1].apn == 'payandgo.o2.co.uk') + end) +end + +return tests diff --git a/tests/unit/gsm/test_state.lua b/tests/unit/gsm/test_state.lua new file mode 100644 index 00000000..b8365a2c --- /dev/null +++ b/tests/unit/gsm/test_state.lua @@ -0,0 +1,85 @@ +local gsm = require 'services.gsm' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function eq(a, b, msg) + if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end +end + +function tests.test_uplink_state_uses_semantic_registered_state() + eq(gsm._test.uplink_state_for_modem(false, 'locked', 'sim-pin'), 'locked') + eq(gsm._test.uplink_state_for_modem(false, 'locked', nil), 'locked') + eq(gsm._test.uplink_state_for_modem(false, 'registered', 'sim-puk'), 'registered') + eq(gsm._test.uplink_state_for_modem(false, 'registered', nil), 'registered') + eq(gsm._test.uplink_state_for_modem(true, 'locked', 'sim-pin'), 'locked') +end + +function tests.test_uplink_state_reports_sim_absent_when_sim_is_absent() + eq(gsm._test.uplink_state_for_modem(false, 'locked', nil, '--'), 'sim_absent') + eq(gsm._test.uplink_state_for_modem(false, 'registered', 'sim-puk', '--'), 'sim_absent') + eq(gsm._test.uplink_state_for_modem(true, 'locked', 'sim-pin', '--'), 'sim_absent') +end + +function tests.test_sim_payload_carries_lock_and_retry_details() + local payload = gsm._test.build_sim_payload('present', 'sim-puk', { + ['sim-pin'] = 0, + ['sim-puk'] = 10, + }, 'locked') + + eq(payload.present, true) + eq(payload.state, 'locked') + eq(payload.lock, 'sim-puk') + eq(payload.lock_retries['sim-pin'], 0) + eq(payload.lock_retries['sim-puk'], 10) +end + +function tests.test_sim_payload_ignores_non_blocking_pin2_on_connected_modem() + local payload = gsm._test.build_sim_payload('present', 'sim-pin2', { + ['sim-pin'] = 3, + ['sim-pin2'] = 3, + }, 'connected') + + eq(payload.present, true) + eq(payload.state, 'present') + eq(payload.lock, nil) + eq(payload.lock_retries, nil) +end + +function tests.test_sim_payload_treats_connected_modem_as_present_when_sim_is_unknown() + local payload = gsm._test.build_sim_payload(nil, nil, nil, 'connected') + + eq(payload.present, true) + eq(payload.state, 'present') + eq(payload.lock, nil) + eq(payload.lock_retries, nil) +end + +function tests.test_sim_payload_reports_locked_without_lock_detail_when_modem_locked() + local payload = gsm._test.build_sim_payload('present', nil, nil, 'locked') + + eq(payload.present, true) + eq(payload.state, 'locked') + eq(payload.lock, nil) + eq(payload.lock_retries, nil) +end + +function tests.test_sim_payload_preserves_unlocked_presence() + local payload = gsm._test.build_sim_payload('present', nil, nil, 'connected') + + eq(payload.present, true) + eq(payload.state, 'present') + eq(payload.lock, nil) + eq(payload.lock_retries, nil) +end + +function tests.test_sim_payload_reports_absent_explicitly() + local payload = gsm._test.build_sim_payload('--', nil, nil, 'disabled') + + eq(payload.present, false) + eq(payload.state, 'absent') + eq(payload.legacy_state, '--') + eq(payload.lock, nil) +end + +return tests diff --git a/tests/unit/hal/artifact_store_driver_spec.lua b/tests/unit/hal/artifact_store_driver_spec.lua new file mode 100644 index 00000000..24ba15b3 --- /dev/null +++ b/tests/unit/hal/artifact_store_driver_spec.lua @@ -0,0 +1,242 @@ +local fibers = require 'fibers' +local runfibers = require 'tests.support.run_fibers' +local blob_source = require 'devicecode.blob_source' + +local store_mod = require 'services.hal.drivers.artifact_store' + +local T = {} + +local function mk_tmpdir(tag) + local path = ('/tmp/dc-lua-%s-%d-%06d'):format(tag, os.time(), math.random(0, 999999)) + local ok = os.execute(('mkdir -p %q'):format(path)) + assert(ok == true or ok == 0, 'failed to create temp dir: ' .. path) + return path +end + +local function rm_rf(path) + os.execute(('rm -rf %q'):format(path)) +end + +function T.artifact_store_imports_source_opens_and_deletes_transient_artifact() + local base = mk_tmpdir('as-driver-roundtrip') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + + runfibers.run(function() + local store = store_mod.new({ + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, nil) + + local ok_import, artifact = fibers.perform(store:import_source_op( + blob_source.from_string('hello'), + { kind = 'update', target = 'mcu' }, + { policy = 'transient_only' } + )) + assert(ok_import == true, tostring(artifact)) + + local rec = artifact:describe() + assert(rec.durability == 'transient') + assert(rec.state == 'ready') + assert(rec.size == 5) + assert(type(rec.checksum) == 'string') + assert(#rec.checksum == 8) + + local ok_open_src, src = fibers.perform(artifact:open_source_op()) + assert(ok_open_src == true, tostring(src)) + + local chunk, err = fibers.perform(src:read_chunk_op(99)) + assert(err == nil) + assert(chunk == 'hello') + + local eof, eof_err = fibers.perform(src:read_chunk_op(99)) + assert(eof == nil) + assert(eof_err == nil) + + local ok_resolve, resolved = fibers.perform(store:resolve_local_op(artifact:ref())) + assert(ok_resolve == true, tostring(resolved)) + assert(type(resolved.path) == 'string') + + local ok_delete, err_delete = fibers.perform(store:delete_op(artifact:ref())) + assert(ok_delete == true, tostring(err_delete)) + + local ok_open, open_err = fibers.perform(store:open_op(artifact:ref())) + assert(ok_open == false) + assert(open_err == 'not_found') + end) + + rm_rf(base) +end + +function T.artifact_store_honours_durability_policy() + local base = mk_tmpdir('as-driver-policy') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + + runfibers.run(function() + local store = store_mod.new({ + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, nil) + + local ok_art, art_or_err = fibers.perform(store:import_source_op( + blob_source.from_string('a'), + { kind = 'update' }, + { policy = 'prefer_durable' } + )) + assert(ok_art == true, tostring(art_or_err)) + assert(art_or_err:describe().durability == 'transient') + + local ok_sink, sink_or_err = fibers.perform(store:create_sink_op( + { kind = 'update' }, + { policy = 'require_durable' } + )) + assert(ok_sink == false) + assert(sink_or_err == 'durable_disabled') + + local store2 = store_mod.new({ + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = true, + }, nil) + + local ok_art2, art2_or_err = fibers.perform(store2:import_source_op( + blob_source.from_string('b'), + { kind = 'update' }, + { policy = 'require_durable' } + )) + assert(ok_art2 == true, tostring(art2_or_err)) + assert(art2_or_err:describe().durability == 'durable') + end) + + rm_rf(base) +end + +function T.artifact_store_imports_path_and_reports_status() + local base = mk_tmpdir('as-driver-path') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + local import_root = base .. '/incoming' + + os.execute(('mkdir -p %q'):format(import_root)) + + local f = assert(io.open(import_root .. '/fw.bin', 'wb')) + assert(f:write('firmware-bytes')) + assert(f:close()) + + runfibers.run(function() + local store = store_mod.new({ + transient_root = transient_root, + durable_root = durable_root, + import_root = import_root, + durable_enabled = false, + }, nil) + + local ok_import, artifact = fibers.perform(store:import_path_op('fw.bin', { + kind = 'firmware', + target = 'cm5', + }, { + policy = 'prefer_durable', + })) + assert(ok_import == true, tostring(artifact)) + + local rec = artifact:describe() + assert(rec.durability == 'transient') + assert(rec.state == 'ready') + + local ok_status, status = fibers.perform(store:status_op()) + assert(ok_status == true, tostring(status)) + assert(status.kind == 'artifact-store') + assert(status.transient_root == transient_root) + assert(status.durable_root == durable_root) + assert(status.durable_enabled == false) + assert(status.import_root == import_root) + end) + + rm_rf(base) +end + +function T.artifact_sink_close_aborts_open_session_and_cleans_up_partial_artifact() + local base = mk_tmpdir('as-driver-close-abort') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + + runfibers.run(function() + local store = store_mod.new({ + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, nil) + + local ok_sink, sink_or_err = fibers.perform(store:create_sink_op( + { kind = 'update', target = 'mcu' }, + { policy = 'transient_only' } + )) + assert(ok_sink == true, tostring(sink_or_err)) + local sink = sink_or_err + + local ref = sink:status().artifact_ref + + local ok_write, write_err = fibers.perform(sink:write_chunk_op('partial-data')) + assert(ok_write == true, tostring(write_err)) + + local ok_close, close_err = fibers.perform(sink:close_op()) + assert(ok_close == true, tostring(close_err)) + + local snapshot = sink:status() + assert(snapshot.terminal == true) + assert(snapshot.state == 'aborted') + + local ok_open, open_err = fibers.perform(store:open_op(ref)) + assert(ok_open == false) + assert(open_err == 'not_found') + + local ok_resolve, resolve_err = fibers.perform(store:resolve_local_op(ref)) + assert(ok_resolve == false) + assert(resolve_err == 'not_found') + end) + + rm_rf(base) +end + +function T.artifact_sink_commit_is_idempotent_for_same_session() + local base = mk_tmpdir('as-driver-idempotent-commit') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + + runfibers.run(function() + local store = store_mod.new({ + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, nil) + + local ok_sink, sink_or_err = fibers.perform(store:create_sink_op( + { kind = 'update', target = 'mcu' }, + { policy = 'transient_only' } + )) + assert(ok_sink == true, tostring(sink_or_err)) + local sink = sink_or_err + + local ok_write, write_err = fibers.perform(sink:write_chunk_op('payload')) + assert(ok_write == true, tostring(write_err)) + + local art1, err1 = fibers.perform(sink:commit_op()) + assert(art1 ~= nil, tostring(err1)) + + local art2, err2 = fibers.perform(sink:commit_op()) + assert(art2 ~= nil, tostring(err2)) + + assert(art1:ref() == art2:ref()) + assert(art1:describe().checksum == art2:describe().checksum) + + local ok_open, art_or_err = fibers.perform(store:open_op(art1:ref())) + assert(ok_open == true, tostring(art_or_err)) + end) + + rm_rf(base) +end + +return T diff --git a/tests/unit/hal/artifact_store_manager_spec.lua b/tests/unit/hal/artifact_store_manager_spec.lua new file mode 100644 index 00000000..38187dcb --- /dev/null +++ b/tests/unit/hal/artifact_store_manager_spec.lua @@ -0,0 +1,235 @@ +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local channel = require 'fibers.channel' + +local runfibers = require 'tests.support.run_fibers' + +local T = {} + +local function fresh_manager() + package.loaded['services.hal.managers.artifact_store'] = nil + return require 'services.hal.managers.artifact_store' +end + +local function recv_or_fail(ch) + local v, err = fibers.perform(ch:get_op()) + assert(v, tostring(err)) + return v +end + +local function mk_tmpdir(tag) + local path = ('/tmp/dc-lua-%s-%d-%06d'):format(tag, os.time(), math.random(0, 999999)) + local ok = os.execute(('mkdir -p %q'):format(path)) + assert(ok == true or ok == 0, 'failed to create temp dir: ' .. path) + return path +end + +local function rm_rf(path) + os.execute(('rm -rf %q'):format(path)) +end + +function T.apply_config_fails_when_not_started() + local M = fresh_manager() + + runfibers.run(function() + local ok, err = fibers.perform(M.apply_config_op({ + stores = { + { id = 'main', transient_root = '/tmp/a', durable_root = '/tmp/b', durable_enabled = false }, + }, + })) + assert(ok == false) + assert(tostring(err):match('not started')) + end) +end + +function T.start_apply_config_and_stop_round_trip() + local M = fresh_manager() + local base = mk_tmpdir('as-manager-start') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok_cfg, err_cfg = fibers.perform(M.apply_config_op({ + stores = { + { + id = 'main', + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, + }, + })) + assert(ok_cfg == true, tostring(err_cfg)) + + local ev = recv_or_fail(dev_ev_ch) + assert(ev.event_type == 'added') + assert(ev.class == 'artifact-store') + assert(ev.id == 'main') + assert(#ev.capabilities == 1) + assert(ev.capabilities[1].class == 'artifact-store') + assert(ev.capabilities[1].offerings['create-sink'] == true) + + local e1 = recv_or_fail(cap_emit_ch) + local e2 = recv_or_fail(cap_emit_ch) + + local by_mode = { + [e1.mode] = e1, + [e2.mode] = e2, + } + + assert(by_mode.meta ~= nil) + assert(by_mode.state ~= nil) + assert(by_mode.meta.class == 'artifact-store') + assert(by_mode.meta.id == 'main') + assert(by_mode.meta.data.transient_root == transient_root) + assert(by_mode.meta.data.durable_root == durable_root) + assert(by_mode.state.data.state == 'available') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) + + rm_rf(base) +end + +function T.reapply_same_config_is_idempotent() + local M = fresh_manager() + local base = mk_tmpdir('as-manager-same') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local cfg = { + stores = { + { + id = 'main', + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, + }, + } + + local ok1, err1 = fibers.perform(M.apply_config_op(cfg)) + assert(ok1 == true, tostring(err1)) + local added = recv_or_fail(dev_ev_ch) + assert(added.event_type == 'added') + + local ok2, err2 = fibers.perform(M.apply_config_op(cfg)) + assert(ok2 == true, tostring(err2)) + + local which = fibers.perform(require('fibers').named_choice{ + msg = dev_ev_ch:get_op():wrap(function(v) return 'msg', v end), + timeout = sleep.sleep_op(0.05):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout', 'reapplying same config should not emit new device events') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) + + rm_rf(base) +end + +function T.config_change_removes_then_adds_provider() + local M = fresh_manager() + local base = mk_tmpdir('as-manager-change') + local transient_root_a = base .. '/transient-a' + local transient_root_b = base .. '/transient-b' + local durable_root = base .. '/durable' + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok1, err1 = fibers.perform(M.apply_config_op({ + stores = { + { + id = 'main', + transient_root = transient_root_a, + durable_root = durable_root, + durable_enabled = false, + }, + }, + })) + assert(ok1 == true, tostring(err1)) + local first = recv_or_fail(dev_ev_ch) + assert(first.event_type == 'added') + + local ok2, err2 = fibers.perform(M.apply_config_op({ + stores = { + { + id = 'main', + transient_root = transient_root_b, + durable_root = durable_root, + durable_enabled = false, + }, + }, + })) + assert(ok2 == true, tostring(err2)) + + local ev_a = recv_or_fail(dev_ev_ch) + local ev_b = recv_or_fail(dev_ev_ch) + assert(ev_a.event_type == 'removed') + assert(ev_b.event_type == 'added') + assert(ev_b.id == 'main') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) + + rm_rf(base) +end + +function T.invalid_config_is_rejected() + local M = fresh_manager() + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok_cfg, err_cfg = fibers.perform(M.apply_config_op({ + stores = 'nope', + })) + assert(ok_cfg == false) + assert(tostring(err_cfg):match('stores must be a table')) + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.shutdown_op_before_start_is_ok_and_fault_op_is_inert() + local M = fresh_manager() + + runfibers.run(function() + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + + local which = fibers.perform(require('fibers').named_choice{ + fault = M.fault_op():wrap(function(...) return 'fault', ... end), + timeout = sleep.sleep_op(0.05):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout') + end) +end + +return T diff --git a/tests/unit/hal/artifact_store_provider_spec.lua b/tests/unit/hal/artifact_store_provider_spec.lua new file mode 100644 index 00000000..e55d4a6e --- /dev/null +++ b/tests/unit/hal/artifact_store_provider_spec.lua @@ -0,0 +1,335 @@ +local fibers = require 'fibers' +local channel = require 'fibers.channel' + +local runfibers = require 'tests.support.run_fibers' +local core_types = require 'services.hal.types.core' +local cap_args = require 'services.hal.types.capability_args' +local blob_source = require 'devicecode.blob_source' + +local provider_mod = require 'services.hal.drivers.artifact_store_provider' + +local T = {} + +local function mk_tmpdir(tag) + local path = ('/tmp/dc-lua-%s-%d-%06d'):format(tag, os.time(), math.random(0, 999999)) + local ok = os.execute(('mkdir -p %q'):format(path)) + assert(ok == true or ok == 0, 'failed to create temp dir: ' .. path) + return path +end + +local function rm_rf(path) + os.execute(('rm -rf %q'):format(path)) +end + +local function recv_or_fail(ch) + local v, err = fibers.perform(ch:get_op()) + assert(v, tostring(err)) + return v +end + +local function request(driver, verb, opts) + local reply_ch = channel.new(1) + local req, err = core_types.new.ControlRequest(verb, opts or {}, reply_ch) + assert(req, tostring(err)) + + local sent, send_err = fibers.perform(driver.control_ch:put_op(req)) + assert(sent ~= false, tostring(send_err)) + + local reply, recv_err = fibers.perform(reply_ch:get_op()) + assert(reply, tostring(recv_err)) + return reply +end + +function T.capabilities_op_returns_artifact_store_capability() + local base = mk_tmpdir('as-provider-cap') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + local emit_ch = channel.new(8) + + runfibers.run(function() + local driver = provider_mod.new('main', { + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, nil) + + local ok, caps_or_err = fibers.perform(driver:capabilities_op(emit_ch)) + assert(ok == true, tostring(caps_or_err)) + assert(type(caps_or_err) == 'table') + assert(#caps_or_err == 1) + + local cap = caps_or_err[1] + assert(cap.class == 'artifact-store') + assert(cap.id == 'main') + assert(cap.offerings['create-sink'] == true) + assert(cap.offerings['import-path'] == true) + assert(cap.offerings['import-source'] == true) + assert(cap.offerings.open == true) + assert(cap.offerings.delete == true) + assert(cap.offerings.status == true) + end) + + rm_rf(base) +end + +function T.start_op_emits_initial_meta_and_available_state() + local base = mk_tmpdir('as-provider-start') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + local emit_ch = channel.new(8) + + runfibers.run(function(scope) + local driver = provider_mod.new('main', { + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, nil) + + local ok_caps, caps_err = fibers.perform(driver:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_err)) + + local ok_start, start_err = fibers.perform(driver:start_op(scope)) + assert(ok_start == true, tostring(start_err)) + + local e1 = recv_or_fail(emit_ch) + local e2 = recv_or_fail(emit_ch) + + local by_mode = { + [e1.mode] = e1, + [e2.mode] = e2, + } + + assert(by_mode.meta ~= nil) + assert(by_mode.state ~= nil) + + assert(by_mode.meta.class == 'artifact-store') + assert(by_mode.meta.id == 'main') + assert(by_mode.meta.key == 'info') + assert(by_mode.meta.data.transient_root == transient_root) + assert(by_mode.meta.data.durable_root == durable_root) + assert(by_mode.meta.data.durable_enabled == false) + + assert(by_mode.state.key == 'status') + assert(by_mode.state.data.state == 'available') + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + assert(ok_stop == true, tostring(stop_err)) + end) + + rm_rf(base) +end + +function T.create_sink_commit_open_and_delete_round_trip() + local base = mk_tmpdir('as-provider-roundtrip') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + local emit_ch = channel.new(8) + + runfibers.run(function(scope) + local driver = provider_mod.new('main', { + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, nil) + + local ok_caps, caps_err = fibers.perform(driver:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_err)) + + local ok_start, start_err = fibers.perform(driver:start_op(scope)) + assert(ok_start == true, tostring(start_err)) + + local create_opts = assert(cap_args.new.ArtifactStoreCreateSinkOpts( + { kind = 'update', target = 'mcu' }, + 'transient_only' + )) + local create_reply = request(driver, 'create-sink', create_opts) + assert(create_reply.ok == true) + + local sink = create_reply.reason + local ok_write, write_err = fibers.perform(sink:write_chunk_op('hello')) + assert(ok_write == true, tostring(write_err)) + + local artifact, commit_err = fibers.perform(sink:commit_op()) + assert(artifact ~= nil, tostring(commit_err)) + + local open_opts = assert(cap_args.new.ArtifactStoreOpenOpts(artifact:ref())) + local open_reply = request(driver, 'open', open_opts) + assert(open_reply.ok == true) + + local opened = open_reply.reason + local ok_src, src_or_err = fibers.perform(opened:open_source_op()) + assert(ok_src == true, tostring(src_or_err)) + + local chunk, err = fibers.perform(src_or_err:read_chunk_op(99)) + assert(err == nil) + assert(chunk == 'hello') + + local del_opts = assert(cap_args.new.ArtifactStoreDeleteOpts(artifact:ref())) + local del_reply = request(driver, 'delete', del_opts) + assert(del_reply.ok == true) + + local status_reply = request(driver, 'status', assert(cap_args.new.ArtifactStoreStatusOpts())) + assert(status_reply.ok == true) + assert(status_reply.reason.kind == 'artifact-store') + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + assert(ok_stop == true, tostring(stop_err)) + end) + + rm_rf(base) +end + +function T.import_source_request_round_trips_success() + local base = mk_tmpdir('as-provider-import-source') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + local emit_ch = channel.new(8) + + runfibers.run(function(scope) + local driver = provider_mod.new('main', { + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, nil) + + local ok_caps, caps_err = fibers.perform(driver:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_err)) + + local ok_start, start_err = fibers.perform(driver:start_op(scope)) + assert(ok_start == true, tostring(start_err)) + + local import_opts = assert(cap_args.new.ArtifactStoreImportSourceOpts( + blob_source.from_string('firmware'), + { kind = 'firmware', target = 'cm5' }, + 'transient_only' + )) + + local reply = request(driver, 'import-source', import_opts) + assert(reply.ok == true) + + local artifact = reply.reason + local rec = artifact:describe() + assert(rec.state == 'ready') + assert(rec.size == 8) + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + assert(ok_stop == true, tostring(stop_err)) + end) + + rm_rf(base) +end + +function T.shutdown_op_before_start_is_ok_and_fault_op_is_inert() + local base = mk_tmpdir('as-provider-stop') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + + runfibers.run(function() + local driver = provider_mod.new('main', { + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, nil) + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + assert(ok_stop == true, tostring(stop_err)) + + local which = fibers.perform(require('fibers').named_choice{ + fault = driver:fault_op():wrap(function(...) return 'fault', ... end), + timeout = require('fibers.sleep').sleep_op(0.05):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout') + end) + + rm_rf(base) +end + +function T.create_sink_close_aborts_partial_artifact() + local base = mk_tmpdir('as-provider-close') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + local emit_ch = channel.new(8) + + runfibers.run(function(scope) + local driver = provider_mod.new('main', { + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, nil) + + local ok_caps, caps_err = fibers.perform(driver:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_err)) + + local ok_start, start_err = fibers.perform(driver:start_op(scope)) + assert(ok_start == true, tostring(start_err)) + + local create_opts = assert(cap_args.new.ArtifactStoreCreateSinkOpts( + { kind = 'update', target = 'mcu' }, + 'transient_only' + )) + + local create_reply = request(driver, 'create-sink', create_opts) + assert(create_reply.ok == true) + local sink = create_reply.reason + + local ref = sink:status().artifact_ref + + local ok_write, write_err = fibers.perform(sink:write_chunk_op('partial')) + assert(ok_write == true, tostring(write_err)) + + local ok_close, close_err = fibers.perform(sink:close_op()) + assert(ok_close == true, tostring(close_err)) + + local open_opts = assert(cap_args.new.ArtifactStoreOpenOpts(ref)) + local open_reply = request(driver, 'open', open_opts) + assert(open_reply.ok == false) + assert(open_reply.reason == 'not_found') + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + assert(ok_stop == true, tostring(stop_err)) + end) + + rm_rf(base) +end + +function T.commit_is_idempotent_for_same_provider_sink_session() + local base = mk_tmpdir('as-provider-idempotent') + local transient_root = base .. '/transient' + local durable_root = base .. '/durable' + local emit_ch = channel.new(8) + + runfibers.run(function(scope) + local driver = provider_mod.new('main', { + transient_root = transient_root, + durable_root = durable_root, + durable_enabled = false, + }, nil) + + local ok_caps, caps_err = fibers.perform(driver:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_err)) + + local ok_start, start_err = fibers.perform(driver:start_op(scope)) + assert(ok_start == true, tostring(start_err)) + + local create_opts = assert(cap_args.new.ArtifactStoreCreateSinkOpts({ kind = 'update' }, 'transient_only')) + local create_reply = request(driver, 'create-sink', create_opts) + assert(create_reply.ok == true) + local sink = create_reply.reason + + local ok_write, write_err = fibers.perform(sink:write_chunk_op('abc')) + assert(ok_write == true, tostring(write_err)) + + local art1, err1 = fibers.perform(sink:commit_op()) + assert(art1 ~= nil, tostring(err1)) + local art2, err2 = fibers.perform(sink:commit_op()) + assert(art2 ~= nil, tostring(err2)) + assert(art1:ref() == art2:ref()) + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + assert(ok_stop == true, tostring(stop_err)) + end) + + rm_rf(base) +end + +return T diff --git a/tests/unit/hal/cap_sdk_spec.lua b/tests/unit/hal/cap_sdk_spec.lua new file mode 100644 index 00000000..92385921 --- /dev/null +++ b/tests/unit/hal/cap_sdk_spec.lua @@ -0,0 +1,328 @@ +local busmod = require 'bus' +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local runfibers = require 'tests.support.run_fibers' +local fake_hal_mod = require 'tests.support.fake_hal' +local cap_sdk = require 'services.hal.sdk.cap' + +local T = {} + +local function topic_eq(a, b) + if type(a) ~= 'table' or type(b) ~= 'table' or #a ~= #b then + return false + end + for i = 1, #a do + if a[i] ~= b[i] then return false end + end + return true +end + +function T.legacy_cap_listener_retains_sync_wrappers() + runfibers.run(function() + local bus = busmod.new() + fake_hal_mod.new({ + caps = { read_state = true }, + scripted = { + read_state = { + { ok = true, found = true, data = '{"hello":true}' }, + }, + }, + }):start(bus:connect(), { name = 'hal' }) + + local listener = cap_sdk.new_cap_listener(bus:connect(), 'fs', 'config') + assert(type(listener.wait_for_cap) == 'function') + + local ref, err = listener:wait_for_cap({ timeout = 0.5 }) + assert(ref, tostring(err)) + assert(type(ref.call_control) == 'function') + + local opts, opts_err = cap_sdk.args.new.FilesystemReadOpts('services.json') + assert(opts, tostring(opts_err)) + + local reply, call_err = ref:call_control('read', opts) + assert(reply, tostring(call_err)) + assert(reply.ok == true) + assert(reply.reason == '{"hello":true}') + + listener:close() + end) +end + +function T.curated_cap_listener_is_op_only_and_composes() + runfibers.run(function() + local bus = busmod.new() + fake_hal_mod.new({ + caps = { read_state = true }, + scripted = { + read_state = { + { ok = true, found = true, data = '{"x":1}' }, + }, + }, + }):start(bus:connect(), { name = 'hal' }) + + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'fs', 'config') + assert(listener.wait_for_cap == nil) + + local which, ref, err = fibers.perform(fibers.named_choice{ + cap = listener:wait_for_cap_op(), + timeout = sleep.sleep_op(0.5), + }) + assert(which == 'cap', tostring(err)) + assert(ref ~= nil) + assert(ref.call_control == nil) + + local opts, opts_err = cap_sdk.args.new.FilesystemReadOpts('services.json') + assert(opts, tostring(opts_err)) + local reply, call_err = fibers.perform(ref:call_control_op('read', opts)) + assert(reply, tostring(call_err)) + assert(reply.ok == true) + assert(reply.reason == '{"x":1}') + + listener:close() + end) +end + +function T.curated_cap_listener_fails_loudly_without_mailbox_backing() + runfibers.run(function() + local fake_sub = { + unsubscribe = function() end, + } + + local listener = setmetatable({ + conn = {}, + sub = fake_sub, + topic = { 'cap', 'fs', 'config', 'status' }, + mode = 'curated-public', + }, getmetatable(cap_sdk.new_curated_cap_listener(busmod.new():connect(), 'fs', 'config'))) + + local ok, err = pcall(function() + fibers.perform(listener:wait_for_cap_op()) + end) + assert(ok == false) + assert(tostring(err):match('mailbox%-backed subscription')) + end) +end + +function T.curated_cap_listener_reports_closed_subscription() + runfibers.run(function() + local bus = busmod.new() + local conn = bus:connect() + local listener = cap_sdk.new_curated_cap_listener(conn, 'fs', 'config') + listener:close() + + local ref, err = fibers.perform(listener:wait_for_cap_op()) + assert(ref == nil) + assert(type(err) == 'string') + assert(err:match('closed') or err:match('unsubscribed')) + end) +end + +function T.curated_cap_listener_ignores_removed_status() + runfibers.run(function() + local bus = busmod.new() + local admin = bus:connect() + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'fs', 'config') + + admin:retain({ 'cap', 'fs', 'config', 'status' }, { + state = 'removed', + available = false, + }) + + local which = fibers.perform(op.named_choice{ + cap = listener:wait_for_cap_op():wrap(function() + return 'cap' + end), + timeout = sleep.sleep_op(0.05):wrap(function() + return 'timeout' + end), + }) + + assert(which == 'timeout') + listener:close() + end) +end + +function T.curated_cap_listener_ignores_malformed_status_payload() + runfibers.run(function() + local bus = busmod.new() + local admin = bus:connect() + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'fs', 'config') + + admin:retain({ 'cap', 'fs', 'config', 'status' }, { + state = 'added', + available = false, + }) + + local which = fibers.perform(op.named_choice{ + cap = listener:wait_for_cap_op():wrap(function() + return 'cap' + end), + timeout = sleep.sleep_op(0.05):wrap(function() + return 'timeout' + end), + }) + + assert(which == 'timeout') + listener:close() + end) +end + +function T.curated_ref_routes_meta_status_state_event_and_control_topics() + runfibers.run(function() + local seen = {} + + local conn = { + subscribe = function(_, topic, opts) + seen[#seen + 1] = { kind = 'subscribe', topic = topic, opts = opts } + return { topic = topic } + end, + call_op = function(_, topic, args, opts) + seen[#seen + 1] = { kind = 'call', topic = topic, args = args, opts = opts } + return op.always({ ok = true }, nil) + end, + } + + local ref = cap_sdk.new_curated_cap_ref(conn, 'wifi', 'main') + assert(ref.call_control == nil) + + ref:get_meta_sub() + ref:get_status_sub() + ref:get_state_sub('signal') + ref:get_event_sub('changed') + local reply, err = fibers.perform(ref:call_control_op('scan', { passive = true })) + assert(reply, tostring(err)) + + assert(topic_eq(seen[1].topic, { 'cap', 'wifi', 'main', 'meta' })) + assert(topic_eq(seen[2].topic, { 'cap', 'wifi', 'main', 'status' })) + assert(topic_eq(seen[3].topic, { 'cap', 'wifi', 'main', 'state', 'signal' })) + assert(topic_eq(seen[4].topic, { 'cap', 'wifi', 'main', 'event', 'changed' })) + assert(topic_eq(seen[5].topic, { 'cap', 'wifi', 'main', 'rpc', 'scan' })) + end) +end + +function T.raw_host_ref_routes_topics() + runfibers.run(function() + local seen = {} + + local conn = { + subscribe = function(_, topic, opts) + seen[#seen + 1] = { kind = 'subscribe', topic = topic, opts = opts } + return { topic = topic } + end, + call_op = function(_, topic, args, opts) + seen[#seen + 1] = { kind = 'call', topic = topic, args = args, opts = opts } + return op.always({ ok = true }, nil) + end, + } + + local ref = cap_sdk.new_raw_host_cap_ref(conn, 'uart_main', 'uart', 'main') + + ref:get_meta_sub() + ref:get_status_sub() + ref:get_state_sub('baud') + ref:get_event_sub('opened') + local reply, err = fibers.perform(ref:call_control_op('open', { speed = 115200 })) + assert(reply, tostring(err)) + + assert(topic_eq(seen[1].topic, { 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'meta' })) + assert(topic_eq(seen[2].topic, { 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'status' })) + assert(topic_eq(seen[3].topic, { 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'state', 'baud' })) + assert(topic_eq(seen[4].topic, { 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'event', 'opened' })) + assert(topic_eq(seen[5].topic, { 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'rpc', 'open' })) + end) +end + +function T.raw_member_ref_routes_topics() + runfibers.run(function() + local seen = {} + + local conn = { + subscribe = function(_, topic, opts) + seen[#seen + 1] = { kind = 'subscribe', topic = topic, opts = opts } + return { topic = topic } + end, + call_op = function(_, topic, args, opts) + seen[#seen + 1] = { kind = 'call', topic = topic, args = args, opts = opts } + return op.always({ ok = true }, nil) + end, + } + + local ref = cap_sdk.new_raw_member_cap_ref(conn, 'member_a', 'gps', 'main') + + ref:get_meta_sub() + ref:get_status_sub() + ref:get_state_sub('fix') + ref:get_event_sub('updated') + local reply, err = fibers.perform(ref:call_control_op('poll', {})) + assert(reply, tostring(err)) + + assert(topic_eq(seen[1].topic, { 'raw', 'member', 'member_a', 'cap', 'gps', 'main', 'meta' })) + assert(topic_eq(seen[2].topic, { 'raw', 'member', 'member_a', 'cap', 'gps', 'main', 'status' })) + assert(topic_eq(seen[3].topic, { 'raw', 'member', 'member_a', 'cap', 'gps', 'main', 'state', 'fix' })) + assert(topic_eq(seen[4].topic, { 'raw', 'member', 'member_a', 'cap', 'gps', 'main', 'event', 'updated' })) + assert(topic_eq(seen[5].topic, { 'raw', 'member', 'member_a', 'cap', 'gps', 'main', 'rpc', 'poll' })) + end) +end + +function T.raw_host_listener_requires_structured_status_not_legacy_added() + runfibers.run(function() + local bus = busmod.new() + local admin = bus:connect() + local listener = cap_sdk.new_raw_host_cap_listener(bus:connect(), 'uart_main', 'uart', 'main') + + admin:retain({ 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'status' }, 'added') + + local which = fibers.perform(op.named_choice{ + cap = listener:wait_for_cap_op():wrap(function() return 'cap' end), + timeout = sleep.sleep_op(0.05):wrap(function() return 'timeout' end), + }) + + assert(which == 'timeout') + listener:close() + end) +end + +function T.raw_host_listener_ignores_removed_status() + runfibers.run(function() + local bus = busmod.new() + local admin = bus:connect() + local listener = cap_sdk.new_raw_host_cap_listener(bus:connect(), 'uart_main', 'uart', 'main') + + admin:retain({ 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'status' }, { + state = 'removed', + available = false, + }) + + local which = fibers.perform(op.named_choice{ + cap = listener:wait_for_cap_op():wrap(function() return 'cap' end), + timeout = sleep.sleep_op(0.05):wrap(function() return 'timeout' end), + }) + + assert(which == 'timeout') + listener:close() + end) +end + +function T.raw_host_listener_accepts_structured_status() + runfibers.run(function() + local bus = busmod.new() + local admin = bus:connect() + local listener = cap_sdk.new_raw_host_cap_listener(bus:connect(), 'uart_main', 'uart', 'main') + + admin:retain({ 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'status' }, { + state = 'available', + available = true, + }) + + local ref, err = fibers.perform(listener:wait_for_cap_op()) + assert(ref, tostring(err)) + assert(ref.call_control == nil) + assert(ref.class == 'uart') + assert(ref.id == 'main') + assert(ref.source == 'uart_main') + listener:close() + end) +end + +return T diff --git a/tests/unit/hal/common_uci_compat_spec.lua b/tests/unit/hal/common_uci_compat_spec.lua new file mode 100644 index 00000000..32aaaae3 --- /dev/null +++ b/tests/unit/hal/common_uci_compat_spec.lua @@ -0,0 +1,30 @@ +-- tests/unit/hal/common_uci_compat_spec.lua + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end + +local function read_file(path) + local f = assert(io.open(path, 'r')) + local s = f:read('*a') + f:close() + return s +end + +function tests.test_common_uci_preserves_wifi_facing_surface() + local uci = require 'services.hal.backends.common.uci' + if type(uci.ensure_started) ~= 'function' then fail('ensure_started missing') end + if type(uci.new_session) ~= 'function' then fail('new_session missing') end + if type(uci.get_value) ~= 'function' then fail('get_value missing') end + if type(uci.section_exists) ~= 'function' then fail('section_exists missing') end + if type(uci.get_sections) ~= 'function' then fail('get_sections missing') end +end + +function tests.test_common_uci_wrapper_does_not_spawn_into_root_scope() + local s = read_file('../src/services/hal/backends/common/uci.lua') + if s:find('Scope.root', 1, true) then + fail('common UCI compatibility wrapper must not spawn into Scope.root') + end +end + +return tests diff --git a/tests/unit/hal/control_loop_spec.lua b/tests/unit/hal/control_loop_spec.lua new file mode 100644 index 00000000..b3bee05c --- /dev/null +++ b/tests/unit/hal/control_loop_spec.lua @@ -0,0 +1,190 @@ +local fibers = require 'fibers' +local runtime = require 'fibers.runtime' +local op = require 'fibers.op' +local channel = require 'fibers.channel' +local runfibers = require 'tests.support.run_fibers' +local control_loop = require 'services.hal.support.control_loop' +local types = require 'services.hal.types.core' + +local T = {} + +function T.evaluate_request_op_rejects_unknown_verbs() + runfibers.run(function() + local req = assert(types.new.ControlRequest('missing', {}, channel.new())) + local ok, err = fibers.perform(control_loop.evaluate_request_op({}, req)) + assert(ok == false) + assert(tostring(err):match('unsupported verb')) + end) +end + +function T.evaluate_request_op_requires_op_handlers() + runfibers.run(function() + local req = assert(types.new.ControlRequest('bad', {}, channel.new())) + local ok, err = pcall(function() + fibers.perform(control_loop.evaluate_request_op({ + bad = function() return true, 'nope' end, + }, req)) + end) + assert(ok == false) + assert(tostring(err):match('must return an Op')) + end) +end + +function T.run_request_loop_handles_requests_and_replies() + runfibers.run(function(scope) + local ch = channel.new() + + local ok_spawn, err = scope:spawn(function() + control_loop.run_request_loop(ch, { + echo = function(opts) + return op.always(true, { echoed = opts.value }) + end, + }, nil, 'test_loop') + end) + assert(ok_spawn, tostring(err)) + + local reply_ch = channel.new() + local req = assert(types.new.ControlRequest('echo', { value = 42 }, reply_ch)) + + local sent, send_err = fibers.perform(ch:put_op(req)) + assert(sent ~= false, tostring(send_err)) + + local reply, reply_err = fibers.perform(reply_ch:get_op()) + assert(reply, tostring(reply_err)) + assert(reply.ok == true) + assert(type(reply.reason) == 'table') + assert(reply.reason.echoed == 42) + end) +end + +function T.run_request_loop_returns_error_reply_for_unsupported_verbs() + runfibers.run(function(scope) + local ch = channel.new() + + local ok_spawn, err = scope:spawn(function() + control_loop.run_request_loop(ch, {}, nil, 'test_loop') + end) + assert(ok_spawn, tostring(err)) + + local reply_ch = channel.new() + local req = assert(types.new.ControlRequest('nope', {}, reply_ch)) + + local sent, send_err = fibers.perform(ch:put_op(req)) + assert(sent ~= false, tostring(send_err)) + + local reply = fibers.perform(reply_ch:get_op()) + assert(reply ~= nil) + assert(reply.ok == false) + assert(tostring(reply.reason):match('unsupported verb')) + end) +end + +function T.run_request_loop_exits_when_scope_is_cancelled() + runfibers.run(function(scope) + local loop_scope, cerr = scope:child() + assert(loop_scope, tostring(cerr)) + + local ch = channel.new() + + local ok_spawn, err = loop_scope:spawn(function() + control_loop.run_request_loop(ch, { + echo = function(opts) + return op.always(true, opts) + end, + }, nil, 'test_loop') + end) + assert(ok_spawn, tostring(err)) + + loop_scope:cancel('stop test') + + local st, rep, primary = fibers.perform(loop_scope:join_op()) + assert(st == 'cancelled', tostring(primary)) + end) +end + + +function T.run_request_loop_cancels_handler_op_when_request_cancel_op_fires() + runfibers.run(function(scope) + local ch = channel.new() + local reply_ch = channel.new() + local cancel_ch = channel.new() + local entered = channel.new() + local aborted = false + + local ok_spawn, err = scope:spawn(function() + control_loop.run_request_loop(ch, { + slow = function() + entered:put(true) + return op.never():on_abort(function () aborted = true end) + end, + }, nil, 'test_loop') + end) + assert(ok_spawn, tostring(err)) + + local cancel_op = cancel_ch:get_op():wrap(function (reason) return reason or 'caller_abandoned' end) + local req = assert(types.new.ControlRequest('slow', {}, reply_ch, cancel_op)) + assert(fibers.perform(ch:put_op(req)) ~= false) + assert(fibers.perform(entered:get_op()) == true) + assert(fibers.perform(cancel_ch:put_op('caller_abandoned')) ~= false) + + for _ = 1, 4 do runtime.yield() end + assert(aborted == true) + + local got = fibers.perform(reply_ch:get_op():or_else(function () return nil, 'not_ready' end)) + assert(got == nil) + end) +end + + +function T.run_request_loop_detaches_caller_after_admission_when_policy_requests_it() + runfibers.run(function(scope) + local ch = channel.new() + local reply_ch = channel.new() + local cancel_ch = channel.new(1) + local entered = channel.new(1) + local release = channel.new(1) + local aborted = false + local completed = false + local logs = {} + local logger = { + warn = function(_, payload) logs[#logs + 1] = payload end, + info = function(_, payload) logs[#logs + 1] = payload end, + debug = function(_, payload) logs[#logs + 1] = payload end, + } + + local ok_spawn, err = scope:spawn(function() + control_loop.run_request_loop(ch, { + __cancel_policy = { apply = 'detach_after_admission' }, + apply = function() + fibers.perform(entered:put_op(true)) + return release:get_op():wrap(function () completed = true; return true, { applied = true } end) + :on_abort(function () aborted = true end) + end, + }, logger, 'network_config') + end) + assert(ok_spawn, tostring(err)) + + local cancel_op = cancel_ch:get_op():wrap(function (reason) return reason or 'caller_abandoned' end) + local req = assert(types.new.ControlRequest('apply', {}, reply_ch, cancel_op)) + assert(fibers.perform(ch:put_op(req)) ~= false) + assert(fibers.perform(entered:get_op()) == true) + assert(fibers.perform(cancel_ch:put_op('caller_abandoned')) ~= false) + for _ = 1, 4 do runtime.yield() end + assert(aborted == false, 'admitted apply must not be aborted by caller abandonment') + assert(fibers.perform(release:put_op(true)) ~= false) + for _ = 1, 4 do runtime.yield() end + assert(completed == true, 'admitted apply should complete after caller detaches') + local got = fibers.perform(reply_ch:get_op():or_else(function () return nil, 'not_ready' end)) + assert(got == nil, 'detached caller should not receive a late reply') + + local saw_detached = false + for _, rec in ipairs(logs) do + if rec.what == 'network_config_request_detached' and rec.admitted == true then + saw_detached = true + end + end + assert(saw_detached == true, 'expected admitted caller detachment log') + end) +end + +return T diff --git a/tests/unit/hal/control_store_manager_spec.lua b/tests/unit/hal/control_store_manager_spec.lua new file mode 100644 index 00000000..714f27c1 --- /dev/null +++ b/tests/unit/hal/control_store_manager_spec.lua @@ -0,0 +1,230 @@ +local fibers = require 'fibers' +local channel = require 'fibers.channel' +local runfibers = require 'tests.support.run_fibers' + +local T = {} + +local function mk_tmpdir(tag) + local path = ('/tmp/dc-lua-%s-%d-%06d'):format(tag, os.time(), math.random(0, 999999)) + local ok = os.execute(('mkdir -p %q'):format(path)) + assert(ok == true or ok == 0, 'failed to create temp dir: ' .. path) + return path +end + +local function rm_rf(path) + os.execute(('rm -rf %q'):format(path)) +end + +local function fresh_manager() + package.loaded['services.hal.managers.control_store'] = nil + package.loaded['services.hal.drivers.control_store'] = nil + package.loaded['services.hal.drivers.control_store_provider'] = nil + return require('services.hal.managers.control_store') +end + +local function recv_or_fail(ch) + local v, err = fibers.perform(ch:get_op()) + assert(v ~= nil, tostring(err)) + return v +end + +function T.start_apply_config_and_stop_round_trip() + local root = mk_tmpdir('csm-roundtrip') + local M = fresh_manager() + + runfibers.run(function(scope) + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok_cfg, err_cfg = fibers.perform(M.apply_config_op({ + { name = 'main', root = root }, + })) + assert(ok_cfg == true, tostring(err_cfg)) + + local ev = recv_or_fail(dev_ev_ch) + assert(ev.event_type == 'added') + assert(ev.class == 'control-store') + assert(ev.id == 'main') + assert(type(ev.capabilities) == 'table' and #ev.capabilities == 1) + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) + + rm_rf(root) +end + +function T.apply_config_creates_missing_control_store_root() + local base = mk_tmpdir('csm-create-root') + local root = base .. '/nested/control-store' + local M = fresh_manager() + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok_cfg, err_cfg = fibers.perform(M.apply_config_op({ + { name = 'main', root = root }, + })) + assert(ok_cfg == true, tostring(err_cfg)) + + local ev = recv_or_fail(dev_ev_ch) + assert(ev.event_type == 'added') + assert(ev.class == 'control-store') + assert(ev.id == 'main') + + local probe, perr = io.open(root .. '/.probe', 'wb') + assert(probe ~= nil, tostring(perr)) + probe:write('ok') + probe:close() + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) + + rm_rf(base) +end + +function T.apply_config_fails_when_not_started() + local root = mk_tmpdir('csm-not-started') + local M = fresh_manager() + + runfibers.run(function() + local ok, err = fibers.perform(M.apply_config_op({ + { name = 'main', root = root }, + })) + assert(ok == false) + assert(tostring(err):match('not started')) + end) + + rm_rf(root) +end + +function T.cap_emit_channel_receives_initial_meta_and_state() + local root = mk_tmpdir('csm-emit') + local M = fresh_manager() + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok_cfg, err_cfg = fibers.perform(M.apply_config_op({ + { name = 'main', root = root }, + })) + assert(ok_cfg == true, tostring(err_cfg)) + + local e1 = recv_or_fail(cap_emit_ch) + local e2 = recv_or_fail(cap_emit_ch) + + local by_mode = { + [e1.mode] = e1, + [e2.mode] = e2, + } + + assert(by_mode.meta ~= nil) + assert(by_mode.state ~= nil) + assert(by_mode.meta.class == 'control-store') + assert(by_mode.meta.id == 'main') + assert(by_mode.meta.key == 'details') + assert(by_mode.meta.data.root == root) + assert(by_mode.state.key == 'status') + assert(by_mode.state.data.state == 'available') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) + + rm_rf(root) +end + +function T.reapply_same_config_is_idempotent() + local root = mk_tmpdir('csm-idempotent') + local M = fresh_manager() + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true) + + local ok1, err1 = fibers.perform(M.apply_config_op({ { name = 'main', root = root } })) + assert(ok1 == true, tostring(err1)) + local added = recv_or_fail(dev_ev_ch) + assert(added.event_type == 'added') + + local ok2, err2 = fibers.perform(M.apply_config_op({ { name = 'main', root = root } })) + assert(ok2 == true, tostring(err2)) + + local which = fibers.perform(require('fibers').named_choice{ + msg = dev_ev_ch:get_op():wrap(function(v) return 'msg', v end), + timeout = require('fibers.sleep').sleep_op(0.05):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout', 'reapplying same config should not emit new device events') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) + + rm_rf(root) +end + +function T.reconcile_root_change_emits_removed_then_added() + local root1 = mk_tmpdir('csm-root1') + local root2 = mk_tmpdir('csm-root2') + local M = fresh_manager() + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok1, err1 = fibers.perform(M.apply_config_op({ { name = 'main', root = root1 } })) + assert(ok1 == true, tostring(err1)) + local first = recv_or_fail(dev_ev_ch) + assert(first.event_type == 'added') + + local ok2, err2 = fibers.perform(M.apply_config_op({ { name = 'main', root = root2 } })) + assert(ok2 == true, tostring(err2)) + + local ev_a = recv_or_fail(dev_ev_ch) + local ev_b = recv_or_fail(dev_ev_ch) + assert(ev_a.event_type == 'removed') + assert(ev_b.event_type == 'added') + assert(ev_b.id == 'main') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) + + rm_rf(root1) + rm_rf(root2) +end + +function T.shutdown_op_before_start_is_ok_and_fault_op_is_inert() + local M = fresh_manager() + + runfibers.run(function() + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + + local which = fibers.perform(fibers.named_choice{ + fault = M.fault_op():wrap(function(...) return 'fault', ... end), + timeout = require('fibers.sleep').sleep_op(0.05):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout') + end) +end + +return T diff --git a/tests/unit/hal/control_store_provider_spec.lua b/tests/unit/hal/control_store_provider_spec.lua new file mode 100644 index 00000000..94a6a8b9 --- /dev/null +++ b/tests/unit/hal/control_store_provider_spec.lua @@ -0,0 +1,171 @@ +local runfibers = require 'tests.support.run_fibers' +local cap_args = require 'services.hal.types.capability_args' + +local T = {} + +local function mk_tmpdir(tag) + local path = ('/tmp/dc-lua-%s-%d-%06d'):format(tag, os.time(), math.random(0, 999999)) + local ok = os.execute(('mkdir -p %q'):format(path)) + assert(ok == true or ok == 0, 'failed to create temp dir: ' .. path) + return path +end + +local function rm_rf(path) + os.execute(('rm -rf %q'):format(path)) +end + +local function read_file(path) + local f, err = io.open(path, 'rb') + if not f then return nil, err end + local data = f:read('*a') + f:close() + return data +end + +local function fresh_provider(root) + package.loaded['services.hal.drivers.control_store_provider'] = nil + return require('services.hal.drivers.control_store_provider').new(root, nil) +end + +function T.status_op_reports_root_and_kind() + local root = mk_tmpdir('csp-status') + local provider = fresh_provider(root) + + runfibers.run(function() + local ok, payload = require('fibers').perform(provider:status_op()) + assert(ok == true) + assert(type(payload) == 'table') + assert(payload.root == root) + assert(payload.kind == 'control-store') + end) + + rm_rf(root) +end + +function T.put_get_and_list_round_trip() + local root = mk_tmpdir('csp-roundtrip') + local provider = fresh_provider(root) + + runfibers.run(function() + local fibers = require 'fibers' + + local put_opts = assert(cap_args.new.ControlStorePutOpts('alpha', 'hello')) + local ok_put, err_put = fibers.perform(provider:put_op(put_opts)) + assert(ok_put == true, tostring(err_put)) + + local get_opts = assert(cap_args.new.ControlStoreGetOpts('alpha')) + local ok_get, value = fibers.perform(provider:get_op(get_opts)) + assert(ok_get == true, tostring(value)) + assert(value == 'hello') + + local ok_list, keys = fibers.perform(provider:list_op()) + assert(ok_list == true, tostring(keys)) + assert(#keys == 1) + assert(keys[1] == 'alpha') + end) + + rm_rf(root) +end + +function T.list_op_filters_by_prefix_and_is_sorted_unique() + local root = mk_tmpdir('csp-prefix') + local provider = fresh_provider(root) + + runfibers.run(function() + local fibers = require 'fibers' + + for _, pair in ipairs({ + { 'a.one', '1' }, + { 'a.two', '2' }, + { 'b.one', '3' }, + }) do + local ok, err = fibers.perform(provider:put_op(assert(cap_args.new.ControlStorePutOpts(pair[1], pair[2])))) + assert(ok == true, tostring(err)) + end + + local ok, keys = fibers.perform(provider:list_op(assert(cap_args.new.ControlStoreListOpts('a.')))) + assert(ok == true, tostring(keys)) + assert(#keys == 2) + assert(keys[1] == 'a.one') + assert(keys[2] == 'a.two') + end) + + rm_rf(root) +end + +function T.get_op_rejects_invalid_key() + local root = mk_tmpdir('csp-invalid-get') + local provider = fresh_provider(root) + + runfibers.run(function() + local fibers = require 'fibers' + local ok, err = fibers.perform(provider:get_op({ key = '../bad' })) + assert(ok == false) + assert(tostring(err):match('invalid key')) + end) + + rm_rf(root) +end + +function T.put_op_rejects_invalid_data() + local root = mk_tmpdir('csp-invalid-put') + local provider = fresh_provider(root) + + runfibers.run(function() + local fibers = require 'fibers' + local ok, err = fibers.perform(provider:put_op({ key = 'alpha', data = 123 })) + assert(ok == false) + assert(tostring(err):match('data must be a string')) + end) + + rm_rf(root) +end + +function T.get_op_returns_not_found_for_missing_key() + local root = mk_tmpdir('csp-missing') + local provider = fresh_provider(root) + + runfibers.run(function() + local fibers = require 'fibers' + local ok, err = fibers.perform(provider:get_op(assert(cap_args.new.ControlStoreGetOpts('missing')))) + assert(ok == false) + assert(tostring(err):match('not found')) + end) + + rm_rf(root) +end + +function T.delete_op_removes_key_from_index_and_truncates_file() + local root = mk_tmpdir('csp-delete') + local provider = fresh_provider(root) + local key_path = root .. '/alpha' + local index_path = root .. '/.control_store_index' + + runfibers.run(function() + local fibers = require 'fibers' + + local ok_put, err_put = fibers.perform(provider:put_op(assert(cap_args.new.ControlStorePutOpts('alpha', 'payload')))) + assert(ok_put == true, tostring(err_put)) + + local ok_del, err_del = fibers.perform(provider:delete_op(assert(cap_args.new.ControlStoreDeleteOpts('alpha')))) + assert(ok_del == true, tostring(err_del)) + + local ok_get, err_get = fibers.perform(provider:get_op(assert(cap_args.new.ControlStoreGetOpts('alpha')))) + assert(ok_get == false) + assert(tostring(err_get):match('not found')) + + local ok_list, keys = fibers.perform(provider:list_op()) + assert(ok_list == true, tostring(keys)) + assert(#keys == 0) + end) + + local data = assert(read_file(key_path)) + assert(data == '', 'delete currently truncates underlying file') + + local index_data = assert(read_file(index_path)) + assert(index_data == '', 'index should no longer contain deleted key') + + rm_rf(root) +end + +return T diff --git a/tests/unit/hal/hal_compat_spec.lua b/tests/unit/hal/hal_compat_spec.lua new file mode 100644 index 00000000..6b5cd496 --- /dev/null +++ b/tests/unit/hal/hal_compat_spec.lua @@ -0,0 +1,886 @@ +local busmod = require 'bus' +local safe = require 'coxpcall' +local fibers = require 'fibers' +local op = require 'fibers.op' +local channel = require 'fibers.channel' + +local runfibers = require 'tests.support.run_fibers' +local cap_sdk = require 'services.hal.sdk.cap' +local core_types = require 'services.hal.types.core' +local cap_types = require 'services.hal.types.capabilities' + +local T = {} + +local function patch_modules(patches, fn) + local saved = {} + for name, value in pairs(patches) do + saved[name] = package.loaded[name] + package.loaded[name] = value + end + + local old_hal = package.loaded['services.hal'] + package.loaded['services.hal'] = nil + + local ok, a, b = safe.pcall(fn) + + package.loaded['services.hal'] = old_hal + for name, old in pairs(saved) do + package.loaded[name] = old + end + + if not ok then + error(a, 0) + end + + return a, b +end + +local function new_bootstrap_filesystem_manager() + local manager = {} + + function manager.start(_logger, _dev_ev_ch, _cap_emit_ch) + manager.scope = fibers.current_scope() + return '' + end + + function manager.apply_config(_self, cfg) + manager.last_cfg = cfg + return true, nil + end + + function manager.stop(_self) + -- Deliberately returns nothing: this is the legacy normalisation case. + end + + return manager +end + +local function new_capability_manager(opts) + opts = opts or {} + + local manager = { + name = assert(opts.name, 'name required'), + cap_class = assert(opts.cap_class, 'cap_class required'), + cap_id = assert(opts.cap_id, 'cap_id required'), + offerings = opts.offerings or { 'echo' }, + apply_mode = opts.apply_mode or 'legacy', + no_reply = not not opts.no_reply, + reply_fn = opts.reply_fn, + calls = {}, + api_mode = (opts.apply_mode == 'op') and 'op_only' or nil, + } + + local control_ch = channel.new() + local emitted = false + + local function emit_added() + if emitted then return end + emitted = true + + local cap, cap_err = cap_types.new.Capability( + manager.cap_class, + manager.cap_id, + control_ch, + manager.offerings + ) + assert(cap, tostring(cap_err)) + + local dev_ev, dev_err = core_types.new.DeviceEvent( + 'added', + 'testdev', + manager.name, + {}, + { cap } + ) + assert(dev_ev, tostring(dev_err)) + + local sent, send_err = fibers.perform(manager.dev_ev_ch:put_op(dev_ev)) + assert(sent ~= false, tostring(send_err)) + end + + local function control_worker(scope) + while true do + local which, a, b = fibers.perform(fibers.named_choice{ + req = control_ch:get_op(), + stop = scope:not_ok_op(), + }) + + if which == 'stop' then + return + end + + local req = a + manager.calls[#manager.calls + 1] = { + verb = req.verb, + opts = req.opts, + } + + if manager.no_reply then + -- Consume the request and deliberately never reply. + else + local ok, reason + if manager.reply_fn then + ok, reason = manager.reply_fn(req) + else + ok, reason = true, { + manager = manager.name, + verb = req.verb, + echo = req.opts and req.opts.value, + } + end + + local reply, reply_err = core_types.new.Reply(ok, reason) + assert(reply, tostring(reply_err)) + + local sent, send_err = fibers.perform(req.reply_ch:put_op(reply)) + assert(sent ~= false, tostring(send_err)) + end + end + end + + if manager.apply_mode == 'op' then + function manager.start_op(_logger, dev_ev_ch, _cap_emit_ch) + return op.guard(function() + manager.scope = fibers.current_scope() + manager.dev_ev_ch = dev_ev_ch + + local ok, err = manager.scope:spawn(function(s) + control_worker(s) + end) + if not ok then + return op.always(false, tostring(err)) + end + + return op.always(true, nil) + end) + end + + function manager.apply_config_op(cfg) + return op.guard(function() + manager.last_cfg = cfg + manager.apply_calls = (manager.apply_calls or 0) + 1 + emit_added() + return op.always(true, nil) + end) + end + + function manager.shutdown_op(_timeout) + return op.guard(function() + manager.shutdown_calls = (manager.shutdown_calls or 0) + 1 + return op.always(true, nil) + end) + end + + function manager.terminate(reason) + manager.terminate_calls = (manager.terminate_calls or 0) + 1 + manager.terminate_reason = reason + if manager.scope then + manager.scope:cancel(tostring(reason or 'terminated')) + end + return true, nil + end + + function manager.fault_op() + if manager.scope then + return manager.scope:fault_op() + end + return op.never() + end + else + function manager.start(_logger, dev_ev_ch, _cap_emit_ch) + manager.scope = fibers.current_scope() + manager.dev_ev_ch = dev_ev_ch + + local ok, err = manager.scope:spawn(function(s) + control_worker(s) + end) + assert(ok, tostring(err)) + + return '' + end + + function manager.apply_config(_self, cfg) + manager.last_cfg = cfg + manager.apply_calls = (manager.apply_calls or 0) + 1 + emit_added() + return true, nil + end + + function manager.stop(_self) + manager.stop_calls = (manager.stop_calls or 0) + 1 + -- Legacy-compatible nil return. + end + end + + return manager +end + +local function new_hanging_start_manager() + return { + api_mode = 'op_only', + + start_op = function() + return op.never() + end, + + apply_config_op = function() + return op.always(true, nil) + end, + + shutdown_op = function() + return op.always(true, nil) + end, + + terminate = function() + return true, nil + end, + + fault_op = function() + return op.never() + end, + } +end + +local function new_hanging_apply_manager() + local manager = { + api_mode = 'op_only', + started = false, + } + + function manager.start_op(_logger, _dev_ev_ch, _cap_emit_ch) + return op.guard(function() + manager.started = true + manager.scope = fibers.current_scope() + return op.always(true, nil) + end) + end + + function manager.apply_config_op(_cfg) + return op.never() + end + + function manager.shutdown_op() + return op.always(true, nil) + end + + function manager.terminate(reason) + manager.terminate_calls = (manager.terminate_calls or 0) + 1 + manager.terminate_reason = reason + if manager.scope then + manager.scope:cancel(tostring(reason or 'terminated')) + end + return true, nil + end + + function manager.fault_op() + if manager.scope then + return manager.scope:fault_op() + end + return op.never() + end + + return manager +end + +local function with_real_hal(scope, patches, body, hal_opts) + return patch_modules(patches, function() + local hal = require 'services.hal' + local bus = busmod.new() + + local opts = { + name = 'hal', + heartbeat_s = 60.0, + } + if hal_opts then + for k, v in pairs(hal_opts) do + opts[k] = v + end + end + + local ok_spawn, spawn_err = scope:spawn(function() + hal.start(bus:connect(), opts) + end) + assert(ok_spawn, tostring(spawn_err)) + + local probe_conn = bus:connect() + return body(bus, probe_conn) + end) +end + +function T.non_legacy_managers_must_expose_op_methods() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local legacy_mgr = new_capability_manager{ + name = 'legacy_mgr', + cap_class = 'legacy_cap', + cap_id = 'cap1', + apply_mode = 'legacy', + offerings = { 'echo' }, + } + + -- Make this fixture explicitly strict, but leave apply_config_op absent. + legacy_mgr.api_mode = 'op_only' + + function legacy_mgr.start_op(_logger, dev_ev_ch, _cap_emit_ch) + return op.guard(function() + legacy_mgr.scope = fibers.current_scope() + legacy_mgr.dev_ev_ch = dev_ev_ch + return op.always(true, nil) + end) + end + + function legacy_mgr.shutdown_op(_timeout) + return op.guard(function() + legacy_mgr.shutdown_calls = (legacy_mgr.shutdown_calls or 0) + 1 + return op.always(true, nil) + end) + end + + local ok, err = safe.pcall(function() + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.legacy_mgr'] = legacy_mgr, + }, function(bus, _probe_conn) + local admin = bus:connect() + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + legacy_mgr = {}, + }, + }) + fibers.perform(require('fibers.sleep').sleep_op(0.05)) + end) + end) + + assert(ok == false) + assert(tostring(err):match('legacy_mgr')) + assert(tostring(err):match('apply_config_op')) + end) +end + +function T.op_manager_methods_work_through_real_hal() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local op_mgr = new_capability_manager{ + name = 'op_mgr', + cap_class = 'op_cap', + cap_id = 'cap2', + apply_mode = 'op', + offerings = { 'echo' }, + } + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.op_mgr'] = op_mgr, + }, function(bus, _probe_conn) + local admin = bus:connect() + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + op_mgr = {}, + }, + }) + + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'op_cap', 'cap2') + local ref, err = fibers.perform(listener:wait_for_cap_op()) + assert(ref, tostring(err)) + assert(ref.call_control == nil) + + local reply, call_err = fibers.perform(ref:call_control_op('echo', { value = 7 })) + assert(reply, tostring(call_err)) + assert(reply.ok == true) + assert(type(reply.reason) == 'table') + assert(reply.reason.echo == 7) + assert(reply.reason.verb == 'echo') + + assert((op_mgr.apply_calls or 0) == 1) + assert(#op_mgr.calls == 1) + + listener:close() + end) + end) +end + + +function T.strict_manager_shutdown_uses_shutdown_op() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local op_mgr = new_capability_manager{ + name = 'shutdown_mgr', + cap_class = 'shutdown_cap', + cap_id = 'cap_shutdown', + apply_mode = 'op', + offerings = { 'echo' }, + } + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.shutdown_mgr'] = op_mgr, + }, function(bus) + local admin = bus:connect() + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + shutdown_mgr = {}, + }, + }) + + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'shutdown_cap', 'cap_shutdown') + local ref, err = fibers.perform(listener:wait_for_cap_op()) + assert(ref, tostring(err)) + listener:close() + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + }, + }) + + fibers.perform(require('fibers.sleep').sleep_op(0.05)) + + assert((op_mgr.shutdown_calls or 0) == 1) + assert(op_mgr.stop_calls == nil) + end) + end) +end + +function T.legacy_manager_shutdown_uses_stop_compatibility() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local legacy_mgr = new_capability_manager{ + name = 'legacy_shutdown_mgr', + cap_class = 'legacy_shutdown_cap', + cap_id = 'cap_legacy_shutdown', + apply_mode = 'legacy', + offerings = { 'echo' }, + } + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.legacy_shutdown_mgr'] = legacy_mgr, + }, function(bus) + local admin = bus:connect() + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + legacy_shutdown_mgr = {}, + }, + }) + + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'legacy_shutdown_cap', 'cap_legacy_shutdown') + local ref, err = fibers.perform(listener:wait_for_cap_op()) + assert(ref, tostring(err)) + listener:close() + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + }, + }) + + fibers.perform(require('fibers.sleep').sleep_op(0.05)) + + assert((legacy_mgr.stop_calls or 0) == 1) + assert(legacy_mgr.shutdown_calls == nil) + end) + end) +end + +function T.op_manager_start_timeout_does_not_block_later_config() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local hanging = new_hanging_start_manager() + local good = new_capability_manager{ + name = 'good_mgr', + cap_class = 'good_cap', + cap_id = 'cap_ok', + apply_mode = 'op', + offerings = { 'echo' }, + } + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.hanging_mgr'] = hanging, + ['services.hal.managers.good_mgr'] = good, + }, function(bus) + local admin = bus:connect() + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + hanging_mgr = {}, + }, + }) + + fibers.perform(require('fibers.sleep').sleep_op(0.05)) + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + good_mgr = {}, + }, + }) + + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'good_cap', 'cap_ok') + local ref, err = fibers.perform(listener:wait_for_cap_op()) + assert(ref, tostring(err)) + + listener:close() + end, { + manager_start_timeout_s = 0.02, + manager_apply_timeout_s = 0.02, + }) + end, { timeout = 2.0 }) +end + +function T.op_manager_apply_timeout_does_not_block_later_config() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local hanging_apply = new_hanging_apply_manager() + local good = new_capability_manager{ + name = 'good_apply_mgr', + cap_class = 'good_apply_cap', + cap_id = 'cap_ok', + apply_mode = 'op', + offerings = { 'echo' }, + } + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.hanging_apply'] = hanging_apply, + ['services.hal.managers.good_apply_mgr'] = good, + }, function(bus) + local admin = bus:connect() + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + hanging_apply = {}, + }, + }) + + fibers.perform(require('fibers.sleep').sleep_op(0.05)) + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + good_apply_mgr = {}, + }, + }) + + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'good_apply_cap', 'cap_ok') + local ref, err = fibers.perform(listener:wait_for_cap_op()) + assert(ref, tostring(err)) + + assert(hanging_apply.started == true) + + listener:close() + end, { + manager_start_timeout_s = 0.02, + manager_apply_timeout_s = 0.02, + }) + end, { timeout = 2.0 }) +end + + +function T.op_manager_apply_timeout_reports_pending_not_failed() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local hanging_apply = new_hanging_apply_manager() + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.hanging_apply'] = hanging_apply, + }, function(bus) + local admin = bus:connect() + local reader = bus:connect() + local end_sub = reader:subscribe({ 'obs', 'v1', 'hal', 'event', 'config_end' }, { queue_len = 8, full = 'drop_oldest' }) + local log_sub = reader:subscribe({ 'obs', 'v1', 'hal', 'event', 'log' }, { queue_len = 16, full = 'drop_oldest' }) + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + hanging_apply = {}, + }, + }) + + local end_msg + for _ = 1, 8 do + local which, msg = fibers.perform(op.named_choice({ + msg = end_sub:recv_op(), + timeout = require('fibers.sleep').sleep_op(0.2):wrap(function () return nil end), + })) + if which == 'msg' and msg and msg.payload and msg.payload.pending_managers then + end_msg = msg + break + end + end + assert(end_msg and end_msg.payload, 'expected config_end with pending manager') + assert(end_msg.payload.ok == true, 'pending admitted manager should not make config_end fail') + assert(end_msg.payload.degraded == true, 'pending manager should mark config degraded') + assert(end_msg.payload.pending_managers[1] == 'hanging_apply') + + local saw_pending = false + local saw_failed = false + for _ = 1, 8 do + local which, msg = fibers.perform(op.named_choice({ + msg = log_sub:recv_op(), + timeout = require('fibers.sleep').sleep_op(0.05):wrap(function () return nil end), + })) + if which ~= 'msg' then break end + local p = msg and msg.payload or {} + if p.what == 'manager_apply_pending_timeout' and p.manager == 'hanging_apply' then saw_pending = true end + if p.what == 'manager_apply_failed' and p.manager == 'hanging_apply' then saw_failed = true end + end + assert(saw_pending == true, 'expected manager_apply_pending_timeout log') + assert(saw_failed == false, 'pending admitted manager must not be logged as failed') + + end_sub:unsubscribe() + log_sub:unsubscribe() + end, { + manager_start_timeout_s = 0.02, + manager_apply_timeout_s = 0.02, + }) + end, { timeout = 2.0 }) +end + + +function T.manager_lifecycle_pump_drains_device_events_during_config_apply() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local patches = { + ['services.hal.managers.filesystem'] = fs_manager, + } + local config = { + schema = 'devicecode.config/hal/1', + } + + for i = 1, 14 do + local name = string.format('burst_mgr_%02d', i) + patches['services.hal.managers.' .. name] = new_capability_manager{ + name = name, + cap_class = 'burst_cap', + cap_id = string.format('cap_%02d', i), + apply_mode = 'op', + offerings = { 'echo' }, + } + config[name] = {} + end + + with_real_hal(scope, patches, function(bus) + local admin = bus:connect() + admin:retain({ 'cfg', 'hal' }, { data = config }) + + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'burst_cap', 'cap_14') + local ref, err = fibers.perform(listener:wait_for_cap_op()) + assert(ref, tostring(err)) + listener:close() + end, { + manager_start_timeout_s = 0.1, + manager_apply_timeout_s = 0.1, + }) + end, { timeout = 2.0 }) +end + + +function T.hal_config_application_uses_generation_worker_completion_events() + local f = assert(io.open('../src/services/hal.lua', 'r')) + local src = f:read('*a') + f:close() + + assert(src:find('config_result_ch', 1, true), 'expected config result channel') + assert(src:find('active_config_generation', 1, true), 'expected active config generation guard') + assert(src:find('pending_config', 1, true), 'expected latest pending config coalescing') + assert(src:find('run_config_generation', 1, true), 'expected generation worker body') + assert(src:find('finish_config_result', 1, true), 'expected coordinator-side completion handler') + assert(src:find('hal_config_deferred', 1, true), 'expected config deferral while a generation is active') + assert(src:find('config_result = config_result_ch:get_op()', 1, true), 'main coordinator should receive config completion records') + + assert(not src:find('local rpc_op = active_config_generation and op.never()', 1, true), 'main loop should continue serving capability RPC during active config') + + local worker_start = assert(src:find('local function run_config_generation', 1, true), 'expected run_config_generation block') + local worker_end = assert(src:find('local start_config_generation', worker_start, true), 'expected run_config_generation terminator') + local worker = src:sub(worker_start, worker_end) + assert(not worker:find('on_device_event', 1, true), 'worker must not mutate device registry') + assert(not worker:find('on_cap_emit', 1, true), 'worker must not mutate capability registry') + assert(not worker:find('start_manager', 1, true), 'worker must not start managers from a short-lived config scope') + + local plan_start = assert(src:find('local function build_config_generation_plan', 1, true), 'expected config planning block') + local plan_end = assert(src:find('local function run_config_generation', plan_start, true), 'expected config planning terminator') + local plan_block = src:sub(plan_start, plan_end) + assert(plan_block:find('start_manager%(name, manager%)'), 'coordinator should start missing managers while planning') + assert(plan_block:find('managers%[name%] = manager'), 'coordinator should admit started managers before spawning apply worker') + + local start_start = assert(src:find('function start_config_generation', 1, true), 'expected start_config_generation block') + local start_end = assert(src:find('local function bootstrap', start_start, true), 'expected start_config_generation terminator') + local start_block = src:sub(start_start, start_end) + assert(start_block:find('fibers%.spawn%(function%('), 'apply generation should run in a worker') + + local finish_start = assert(src:find('local function finish_config_result', 1, true), 'expected finish_config_result block') + local finish_end = assert(src:find('function start_config_generation', finish_start, true), 'expected finish_config_result terminator') + local finish = src:sub(finish_start, finish_end) + assert(not finish:find('managers%[rec%.name%] = rec%.manager'), 'finish handler should not admit worker-started managers') + assert(finish:find('shutdown_manager_async'), 'coordinator should own manager removal') +end + +function T.unsupported_control_verb_returns_no_route() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local mgr = new_capability_manager{ + name = 'verb_mgr', + cap_class = 'verb_cap', + cap_id = 'cap3', + apply_mode = 'op', + offerings = { 'echo' }, + } + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.verb_mgr'] = mgr, + }, function(bus, _probe_conn) + local admin = bus:connect() + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + verb_mgr = {}, + }, + }) + + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'verb_cap', 'cap3') + local ref, err = fibers.perform(listener:wait_for_cap_op()) + assert(ref, tostring(err)) + + local reply, call_err = fibers.perform(ref:call_control_op('missing', {})) + assert(reply == nil) + assert(call_err == 'no_route' or tostring(call_err):match('no_route')) + + listener:close() + end) + end) +end + +function T.capability_no_reply_is_completed_by_hal_control_timeout() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local mgr = new_capability_manager{ + name = 'timeout_mgr', + cap_class = 'timeout_cap', + cap_id = 'cap4', + apply_mode = 'op', + offerings = { 'echo' }, + no_reply = true, + } + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.timeout_mgr'] = mgr, + }, function(bus, _probe_conn) + local admin = bus:connect() + + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + timeout_mgr = {}, + }, + }) + + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'timeout_cap', 'cap4') + local ref, wait_err = fibers.perform(listener:wait_for_cap_op()) + assert(ref, tostring(wait_err)) + + local t0 = fibers.now() + + local reply, err = fibers.perform(ref:call_control_op('echo', { value = 9 }, { + timeout = 0.5, + backoff = 0.01, + backoff_max = 0.01, + })) + + local elapsed = fibers.now() - t0 + + assert(reply ~= nil, tostring(err)) + assert(err == nil) + assert(reply.ok == false) + assert(type(reply.reason) == 'string') + assert(reply.reason:match('timeout')) + assert(elapsed < 0.25, 'request appears to have waited for client timeout, not HAL timeout') + assert(#mgr.calls == 1) + + listener:close() + end, { + control_timeout_s = 0.03, + }) + end, { timeout = 2.0 }) +end + +function T.strict_manager_contract_requires_terminate_and_fault_op() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local strict_mgr = { + api_mode = 'op_only', + start_op = function() return op.always(true, nil) end, + apply_config_op = function() return op.always(true, nil) end, + shutdown_op = function() return op.always(true, nil) end, + -- terminate and fault_op are deliberately absent. + } + + local ok, err = safe.pcall(function() + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.strict_mgr'] = strict_mgr, + }, function(bus, _probe_conn) + local admin = bus:connect() + admin:retain({ 'cfg', 'hal' }, { + data = { + schema = 'devicecode.config/hal/1', + strict_mgr = {}, + }, + }) + fibers.perform(require('fibers.sleep').sleep_op(0.05)) + end) + end) + + assert(ok == false) + assert(tostring(err):match('strict_mgr')) + assert(tostring(err):match('terminate')) + end) +end + +function T.strict_hal_managers_do_not_use_perform_raw() + local paths = { + '../src/services/hal/managers/artifact_store.lua', + '../src/services/hal/managers/control_store.lua', + '../src/services/hal/managers/signature_verify.lua', + '../src/services/hal/managers/uart.lua', + } + + for _, path in ipairs(paths) do + local f = assert(io.open(path, 'r')) + local s = f:read('*a') + f:close() + assert(not s:find('perform_raw', 1, true), path .. ' uses perform_raw') + end +end + + +function T.network_manager_control_loops_start_after_driver_configured() + local f = assert(io.open('../src/services/hal/managers/network.lua', 'r')) + local src = f:read('*a') + f:close() + local start_s = src:find('function M.start_op', 1, true) + local apply_s = src:find('function M.apply_config_op', 1, true) + local shutdown_s = src:find('function M.shutdown_op', 1, true) + assert(start_s and apply_s and shutdown_s, 'expected network manager lifecycle functions') + local start_block = src:sub(start_s, apply_s - 1) + local apply_block = src:sub(apply_s, shutdown_s - 1) + assert(start_block:find('emit_added', 1, true), 'network start should publish passive capability handles for legacy consumers') + assert(not start_block:find('start_control_loops', 1, true), 'network start must not serve requests before config') + assert(apply_block:find('state.driver = driver', 1, true), 'network apply should configure driver') + assert(apply_block:find('start_control_loops', 1, true), 'network apply should start control loops after driver configuration') +end + +return T diff --git a/tests/unit/hal/modem_linux_mm_spec.lua b/tests/unit/hal/modem_linux_mm_spec.lua new file mode 100644 index 00000000..5d364981 --- /dev/null +++ b/tests/unit/hal/modem_linux_mm_spec.lua @@ -0,0 +1,75 @@ +local linux_mm = require 'services.hal.backends.modem.providers.linux_mm.impl' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function eq(a, b, msg) + if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end +end +local function ok(v, msg) if not v then fail(msg or 'expected truthy') end end + +function tests.test_linux_mm_parses_sim_lock_fields_from_modem_json() + local info, err = linux_mm._test.parse_modem_info_json([[ +{ + "modem": { + "3gpp": { + "imei": "860195058017887", + "operator-name": "--" + }, + "generic": { + "equipment-identifier": "860195058017887", + "device": "/sys/devices/demo", + "primary-port": "cdc-wdm1", + "ports": ["cdc-wdm1 (qmi)", "ttyUSB6 (at)", "wwan1 (net)"], + "access-technologies": [], + "sim": "/org/freedesktop/ModemManager1/SIM/1", + "drivers": ["qmi_wwan", "option1"], + "plugin": "quectel", + "model": "QUECTEL Mobile Broadband Module", + "revision": "EG25GGBR07A08M2G", + "state": "locked", + "unlock-required": "sim-puk", + "unlock-retries": ["sim-pin (0)", "sim-puk (10)", "sim-pin2 (3)"] + } + } +} +]]) + ok(info, err) + eq(info.modem_state, 'locked') + eq(info.sim_lock, 'sim-puk') + eq(info.sim_lock_retries['sim-pin'], 0) + eq(info.sim_lock_retries['sim-puk'], 10) + eq(info.qmi_ports[1], 'cdc-wdm1') + eq(info.at_ports[1], 'ttyUSB6') + eq(info.net_ports[1], 'wwan1') +end + +function tests.test_linux_mm_normalises_absent_unlock_required() + local info, err = linux_mm._test.parse_modem_info_json([[ +{ + "modem": { + "3gpp": { + "imei": "868549060377253", + "operator-name": "Demo" + }, + "generic": { + "equipment-identifier": "868549060377253", + "ports": [], + "access-technologies": "lte", + "drivers": "qmi_wwan", + "state": "registered", + "unlock-required": "--", + "unlock-retries": [] + } + } +} +]]) + ok(info, err) + eq(info.modem_state, 'registered') + eq(info.sim_lock, nil) + eq(info.sim_lock_retries, nil) + eq(info.access_techs[1], 'lte') + eq(info.drivers[1], 'qmi_wwan') +end + +return tests diff --git a/tests/unit/hal/modem_process_flags_spec.lua b/tests/unit/hal/modem_process_flags_spec.lua new file mode 100644 index 00000000..064d6e80 --- /dev/null +++ b/tests/unit/hal/modem_process_flags_spec.lua @@ -0,0 +1,65 @@ +local T = {} + +local function with_exec_supports(supports_parent_death_signal, fn) + local saved_exec = package.loaded['fibers.io.exec'] + local saved_mod = package.loaded['services.hal.backends.modem.process_flags'] + + package.loaded['fibers.io.exec'] = { + supports = function(name) + return name == 'parent_death_signal' and supports_parent_death_signal or false + end, + } + package.loaded['services.hal.backends.modem.process_flags'] = nil + + local ok, a, b = pcall(fn) + + package.loaded['fibers.io.exec'] = saved_exec + package.loaded['services.hal.backends.modem.process_flags'] = saved_mod + + if not ok then error(a, 0) end + return a, b +end + +function T.owned_monitor_flags_always_requests_process_group() + with_exec_supports(false, function() + local flags = require('services.hal.backends.modem.process_flags').owned_monitor_flags() + assert(flags.process_group == true) + assert(flags.parent_death_signal == nil) + assert(flags.pdeathsig == nil) + end) +end + +function T.owned_monitor_flags_adds_parent_death_signal_only_when_supported() + with_exec_supports(true, function() + local flags = require('services.hal.backends.modem.process_flags').owned_monitor_flags() + assert(flags.process_group == true) + assert(flags.parent_death_signal == 'TERM') + assert(flags.pdeathsig == nil) + end) +end + +local function read_qmi_source() + local candidates = { + '../src/services/hal/backends/modem/modes/qmi.lua', + 'src/services/hal/backends/modem/modes/qmi.lua', + } + for _, path in ipairs(candidates) do + local f = io.open(path, 'r') + if f then + local src = f:read('*a') + f:close() + return src + end + end + error('could not read qmi.lua') +end + +function T.qmi_sim_power_cycle_commands_use_owned_process_flags() + local src = read_qmi_source() + assert(src:match('uim%-sim%-power%-off=1"[%s%S]-flags%s*=%s*process_flags%.owned_monitor_flags%(%)'), + 'qmicli SIM power-off command must use owned process flags') + assert(src:match('uim%-sim%-power%-on=1"[%s%S]-flags%s*=%s*process_flags%.owned_monitor_flags%(%)'), + 'qmicli SIM power-on command must use owned process flags') +end + +return T diff --git a/tests/unit/hal/modem_qmi_spec.lua b/tests/unit/hal/modem_qmi_spec.lua new file mode 100644 index 00000000..a9cd3329 --- /dev/null +++ b/tests/unit/hal/modem_qmi_spec.lua @@ -0,0 +1,32 @@ +local modem_types = require 'services.hal.types.modem' +local qmi = require 'services.hal.backends.modem.modes.qmi' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function eq(a, b, msg) + if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end +end +local function ok(v, msg) if not v then fail(msg or 'expected truthy') end end + +function tests.test_qmi_preserves_sim_lock_when_gid1_read_fails() + local info, err = modem_types.new.ModemSimInfo( + '/org/freedesktop/ModemManager1/SIM/2', + '8944', + '23415', + nil, + 'sim-pin', + { ['sim-pin'] = 3, ['sim-puk'] = 10 }, + 'locked' + ) + ok(info, err) + + local enriched, enrich_err = qmi._test.sim_info_with_gid1(info, nil, 'locked SIM cannot read gid1') + ok(enriched, enrich_err) + eq(enriched.gid1, nil) + eq(enriched.sim_lock, 'sim-pin') + eq(enriched.sim_lock_retries['sim-pin'], 3) + eq(enriched.modem_state, 'locked') +end + +return tests diff --git a/tests/unit/hal/network_manager_spec.lua b/tests/unit/hal/network_manager_spec.lua new file mode 100644 index 00000000..7edae5dd --- /dev/null +++ b/tests/unit/hal/network_manager_spec.lua @@ -0,0 +1,174 @@ +-- tests/unit/hal/network_manager_spec.lua + +local tests = {} + +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end +local function eq(a, b, msg) if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end end + +function tests.test_network_manager_is_strict_op_only() + local manager = require 'services.hal.managers.network' + eq(manager.api_mode, 'op_only') + ok(type(manager.start_op) == 'function', 'start_op required') + ok(type(manager.apply_config_op) == 'function', 'apply_config_op required') + ok(type(manager.shutdown_op) == 'function', 'shutdown_op required') + ok(type(manager.terminate) == 'function', 'terminate required') + ok(type(manager.fault_op) == 'function', 'fault_op required') +end + +function tests.test_network_capability_constructors_exist() + local caps = require 'services.hal.types.capabilities' + ok(type(caps.new.NetworkConfigCapability) == 'function', 'network-config constructor required') + ok(type(caps.new.NetworkStateCapability) == 'function', 'network-state constructor required') + ok(type(caps.new.NetworkDiagnosticsCapability) == 'function', 'network-diagnostics constructor required') +end + +function tests.test_network_backend_contract_requires_op_methods() + local contract = require 'services.hal.backends.network.contract' + local op = require 'fibers.op' + local provider = { + validate_op = function () return op.always({ ok = true }) end, + plan_op = function () return op.always({ ok = true }) end, + apply_op = function () return op.always({ ok = true }) end, + snapshot_op = function () return op.always({ ok = true }) end, + watch_op = function () return op.always({ ok = true }) end, + probe_link_op = function () return op.always({ ok = true }) end, + read_counters_op = function () return op.always({ ok = true }) end, + apply_live_weights_op = function () return op.always({ ok = true }) end, + apply_shaping_op = function () return op.always({ ok = true }) end, + speedtest_op = function () return op.always({ ok = true }) end, + terminate = function () return true end, + } + local valid, err = contract.validate_provider(provider) + ok(valid, err) + + provider.apply_op = nil + valid, err = contract.validate_provider(provider) + if valid ~= nil then error('provider without apply_op should be rejected', 2) end + ok(err and err:find('apply_op', 1, true), 'apply_op error expected') +end + +function tests.test_network_manager_cancels_driver_op_when_control_request_is_abandoned() + local fibers = require 'fibers' + local runtime = require 'fibers.runtime' + local op = require 'fibers.op' + local channel = require 'fibers.channel' + local runfibers = require 'tests.support.run_fibers' + local types = require 'services.hal.types.core' + local manager = require 'services.hal.managers.network' + local driver_mod = require 'services.hal.drivers.network' + + runfibers.run(function(scope) + manager.terminate('test reset') + local old_new = driver_mod.new + scope:finally(function () + driver_mod.new = old_new + manager.terminate('test cleanup') + end) + + local entered = channel.new(1) + local cancel_ch = channel.new(1) + local aborted = false + driver_mod.new = function () + return { + speedtest_op = function () + fibers.perform(entered:put_op(true)) + return op.never():on_abort(function () aborted = true end) + end, + terminate = function () return true, nil end, + }, nil + end + + local dev_ev_ch = channel.new(4) + local cap_emit_ch = channel.new(4) + local ok_start, start_err = fibers.perform(manager.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(start_err)) + local dev_ev = fibers.perform(dev_ev_ch:get_op()) + local ok_apply, apply_err = fibers.perform(manager.apply_config_op({ provider = 'test' })) + assert(ok_apply == true, tostring(apply_err)) + + local diag_cap + for _, cap in ipairs(dev_ev.capabilities or {}) do + if cap.class == 'network-diagnostics' then diag_cap = cap end + end + assert(diag_cap and diag_cap.control_ch, 'network diagnostics cap expected') + + local reply_ch = channel.new(1) + local cancel_op = cancel_ch:get_op():wrap(function (reason) return reason or 'caller_abandoned' end) + local req = assert(types.new.ControlRequest('speedtest', { interface = 'wan_a' }, reply_ch, cancel_op)) + assert(fibers.perform(diag_cap.control_ch:put_op(req)) ~= false) + assert(fibers.perform(entered:get_op()) == true) + assert(fibers.perform(cancel_ch:put_op('caller_abandoned')) ~= false) + + for _ = 1, 4 do runtime.yield() end + assert(aborted == true, 'driver op should be aborted after caller abandonment') + local got = fibers.perform(reply_ch:get_op():or_else(function () return nil, 'not_ready' end)) + assert(got == nil, 'abandoned request should not receive a late reply') + end) +end + + +function tests.test_network_apply_continues_after_caller_abandonment_once_admitted() + local fibers = require 'fibers' + local runtime = require 'fibers.runtime' + local op = require 'fibers.op' + local channel = require 'fibers.channel' + local runfibers = require 'tests.support.run_fibers' + local types = require 'services.hal.types.core' + local manager = require 'services.hal.managers.network' + local driver_mod = require 'services.hal.drivers.network' + + runfibers.run(function(scope) + manager.terminate('test reset') + local old_new = driver_mod.new + scope:finally(function () + driver_mod.new = old_new + manager.terminate('test cleanup') + end) + + local entered = channel.new(1) + local release = channel.new(1) + local cancel_ch = channel.new(1) + local aborted = false + local completed = false + driver_mod.new = function () + return { + apply_op = function () + fibers.perform(entered:put_op(true)) + return release:get_op():wrap(function () completed = true; return { ok = true } end) + :on_abort(function () aborted = true end) + end, + terminate = function () return true, nil end, + }, nil + end + + local dev_ev_ch = channel.new(4) + local cap_emit_ch = channel.new(4) + local ok_start, start_err = fibers.perform(manager.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(start_err)) + local dev_ev = fibers.perform(dev_ev_ch:get_op()) + local ok_apply, apply_err = fibers.perform(manager.apply_config_op({ provider = 'test' })) + assert(ok_apply == true, tostring(apply_err)) + + local cfg_cap + for _, cap in ipairs(dev_ev.capabilities or {}) do + if cap.class == 'network-config' then cfg_cap = cap end + end + assert(cfg_cap and cfg_cap.control_ch, 'network config cap expected') + + local reply_ch = channel.new(1) + local cancel_op = cancel_ch:get_op():wrap(function (reason) return reason or 'caller_abandoned' end) + local req = assert(types.new.ControlRequest('apply', { generation = 1 }, reply_ch, cancel_op)) + assert(fibers.perform(cfg_cap.control_ch:put_op(req)) ~= false) + assert(fibers.perform(entered:get_op()) == true) + assert(fibers.perform(cancel_ch:put_op('caller_abandoned')) ~= false) + for _ = 1, 4 do runtime.yield() end + assert(aborted == false, 'admitted network apply should not be aborted by caller abandonment') + assert(fibers.perform(release:put_op(true)) ~= false) + for _ = 1, 4 do runtime.yield() end + assert(completed == true, 'network apply should drain after caller detaches') + local got = fibers.perform(reply_ch:get_op():or_else(function () return nil, 'not_ready' end)) + assert(got == nil, 'detached caller should not receive a late reply') + end) +end + +return tests diff --git a/tests/unit/hal/openwrt_names_spec.lua b/tests/unit/hal/openwrt_names_spec.lua new file mode 100644 index 00000000..2b729d19 --- /dev/null +++ b/tests/unit/hal/openwrt_names_spec.lua @@ -0,0 +1,112 @@ +-- tests/unit/hal/openwrt_names_spec.lua + +local names = require 'services.hal.backends.network.providers.openwrt.names' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function ok(v, msg) if not v then fail(msg) end return v end +local function eq(a, b, msg) if a ~= b then fail((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end end + +local function starts(s, p, msg) + if type(s) ~= 'string' or s:sub(1, #p) ~= p then fail(msg or ('expected ' .. tostring(s) .. ' to start with ' .. tostring(p))) end +end + +function tests.test_generated_names_are_bounded_and_readable() + local ctx = ok(names.allocate({ + segments = { + ['administration-network-with-a-very-long-user-visible-name'] = { kind = 'system', firewall = { zone = 'restricted-admin-firewall-zone' } }, + ['123-invalid-prefix-and-extremely-long'] = { kind = 'lan' }, + }, + interfaces = { + ['cellular-primary-uplink-with-long-name'] = {}, + }, + firewall = { zones = { ['restricted-admin-firewall-zone'] = {} } }, + wan = { members = { ['cellular-primary-uplink-with-long-name'] = { interface = 'cellular-primary-uplink-with-long-name' } } }, + })) + local lim = ctx:limits() + local iface = ctx:iface('administration-network-with-a-very-long-user-visible-name') + local bridge = ctx:bridge('administration-network-with-a-very-long-user-visible-name') + local vlan = ctx:vlan('administration-network-with-a-very-long-user-visible-name') + local zone = ctx:zone('restricted-admin-firewall-zone') + local mwan = ctx:mwan_iface('cellular-primary-uplink-with-long-name') + local dns = ctx:dns_instance('ads-adult-host-policy-with-long-name') + ok(#iface <= lim.logical_interface, 'logical interface length') + ok(#bridge <= lim.bridge_device, 'bridge length') + ok(#vlan <= lim.linux_device, 'vlan length') + ok(#zone <= lim.firewall_zone, 'zone length') + ok(#mwan <= lim.mwan_name, 'mwan length') + ok(#dns <= lim.dnsmasq_instance, 'dnsmasq length') + starts(iface, 'ad', 'semantic prefix retained') + starts(bridge, 'brad', 'bridge semantic prefix retained') + starts(zone, 're', 'zone semantic prefix retained') + starts(ctx:iface('123-invalid-prefix-and-extremely-long'), 'x1', 'numeric leading prefix made safe') +end + + +function tests.test_modem_logical_interface_names_keep_readable_distinguishing_stems() + local ctx = ok(names.allocate({ + segments = {}, interfaces = { + modem_primary = {}, + modem_secondary = {}, + modem_primary_blue = {}, + modem_primary_green = {}, + modem_primary_bluey_green = {}, + }, firewall = { zones = {} }, wan = { members = {} }, + })) + starts(ctx:iface('modem_primary'), 'mopri', 'modem primary stem') + starts(ctx:iface('modem_secondary'), 'mosec', 'modem secondary stem') + starts(ctx:iface('modem_primary_blue'), 'mprbl', 'modem primary blue stem') + starts(ctx:iface('modem_primary_green'), 'mprgr', 'modem primary green stem') + starts(ctx:iface('modem_primary_bluey_green'), 'mpbgr', 'modem primary bluey green stem') + eq(#ctx:iface('modem_primary'), ctx:limits().logical_interface, 'stem plus hash length') +end + +function tests.test_name_snapshot_is_stable() + local intent = { segments = { adm = {}, jan = {} }, interfaces = {}, firewall = { zones = {} }, wan = { members = {} } } + local a = ok(names.allocate(intent)):snapshot() + local b = ok(names.allocate(intent)):snapshot() + eq(a.names.logical_interface.adm, b.names.logical_interface.adm) + eq(a.names.bridge_device.jan, b.names.bridge_device.jan) +end + + +function tests.test_mwan3_section_names_share_one_namespace() + local ctx = ok(names.allocate({ + segments = {}, interfaces = {}, firewall = { zones = {} }, + wan = { + members = { + wan = { interface = 'wan' }, + balanced = { interface = 'balanced' }, + }, + }, + })) + local lim = ctx:limits() + local iface_wan = ctx:mwan_iface('wan') + local member_wan = ctx:mwan_member('wan') + local iface_balanced = ctx:mwan_iface('balanced') + local member_balanced = ctx:mwan_member('balanced') + local policy_balanced = ctx:mwan_policy('balanced') + local rule_balanced = ctx:mwan_rule('balanced') + local seen = {} + for _, n in ipairs({ iface_wan, member_wan, iface_balanced, member_balanced, policy_balanced, rule_balanced }) do + ok(#n <= lim.mwan_name, 'mwan3 name length') + if seen[n] then fail('duplicate mwan3 UCI section name: ' .. tostring(n)) end + seen[n] = true + end +end + +function tests.test_baseline_names_are_reserved() + local ctx = ok(names.allocate({ + segments = { + loopback = { kind = 'lan', firewall = { zone = 'defaults' } }, + globals = { kind = 'lan' }, + }, + interfaces = {}, firewall = { zones = { defaults = {} } }, wan = { members = {} }, + })) + eq(ctx:iface('loopback') == 'loopback', false, 'segment id must not collide with network.loopback') + eq(ctx:iface('globals') == 'globals', false, 'segment id must not collide with network.globals') + eq(ctx:zone('defaults') == 'defaults', false, 'zone name must not collide with firewall defaults') +end + +return tests diff --git a/tests/unit/hal/openwrt_network_observer_spec.lua b/tests/unit/hal/openwrt_network_observer_spec.lua new file mode 100644 index 00000000..b0c45478 --- /dev/null +++ b/tests/unit/hal/openwrt_network_observer_spec.lua @@ -0,0 +1,175 @@ +-- tests/unit/hal/openwrt_network_observer_spec.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' + +local observer_mod = require 'services.hal.backends.network.providers.openwrt.observer' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function ok(v, msg) if not v then fail(msg) end return v end +local function eq(a, b, msg) if a ~= b then fail((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end + +function tests.test_observer_coalesces_wakeups_then_snapshots() + fibers.run(function (scope) + local emitted = {} + local snapshots = {} + local obs = ok(observer_mod.new({ + debounce_s = 0.01, + enable_socket = false, + enable_ubus = false, + initial_snapshot = false, + snapshot = function (subject, trigger) + snapshots[#snapshots + 1] = { subject = subject, trigger = trigger } + return { + ok = true, + backend = 'test', + observed = { + schema = 'devicecode.net.observed/1', + interfaces = { wan = { id = 'wan', enabled = true } }, + segments = {}, + }, + } + end, + emit = function (ev) + emitted[#emitted + 1] = ev + return true + end, + })) + ok(obs:start(scope)) + obs:ingest({ source = 'hotplug', directory = 'iface', env = { ACTION = 'ifup', INTERFACE = 'wan' } }) + obs:ingest({ source = 'ubus', payload = { action = 'ifupdate', interface = 'wan' } }) + fibers.perform(sleep.sleep_op(0.05)) + eq(#snapshots, 1) + eq(#emitted, 1) + eq(emitted[1].kind, 'interface_changed') + eq(emitted[1].subject, 'interface:wan') + eq(emitted[1].observed.interfaces.wan.enabled, true) + obs:terminate('test complete') + end) +end + + +function tests.test_observer_ignores_transient_mwan3_phases() + fibers.run(function (scope) + local emitted = {} + local snapshots = {} + local logs = {} + local obs = ok(observer_mod.new({ + debounce_s = 0.01, + enable_socket = false, + enable_ubus = false, + initial_snapshot = false, + logger = function (level, payload) + logs[#logs + 1] = { level = level, payload = payload } + end, + snapshot = function (subject, trigger) + snapshots[#snapshots + 1] = { subject = subject, trigger = trigger } + return { + ok = true, + backend = 'test', + observed = { + schema = 'devicecode.net.observed/1', + interfaces = {}, + segments = {}, + }, + } + end, + emit = function (ev) + emitted[#emitted + 1] = ev + return true + end, + })) + ok(obs:start(scope)) + + ok(obs:ingest({ source = 'mwan3', kind = 'mwan3', env = { ACTION = 'disconnecting', INTERFACE = 'wan' } })) + ok(obs:ingest({ source = 'hotplug', kind = 'hotplug', directory = 'iface', env = { ACTION = 'connecting', INTERFACE = 'wan' } })) + fibers.perform(sleep.sleep_op(0.05)) + eq(#snapshots, 0, 'transient mwan phases should not trigger snapshots') + eq(#emitted, 0, 'transient mwan phases should not emit observed-state events') + + ok(obs:ingest({ source = 'mwan3', kind = 'mwan3', env = { ACTION = 'connected', INTERFACE = 'wan' } })) + fibers.perform(sleep.sleep_op(0.05)) + eq(#snapshots, 1, 'terminal mwan events should still trigger snapshots') + eq(#emitted, 1, 'terminal mwan events should emit observed-state events') + eq(emitted[1].kind, 'mwan_member_changed') + eq(emitted[1].subject, 'mwan:wan') + + obs:terminate('test complete') + end) +end + +local function read_source(relpath) + local candidates = { + relpath, + '../' .. relpath, + '../../' .. relpath, + } + + for i = 1, #candidates do + local f = io.open(candidates[i], 'r') + if f then + local src = f:read('*a') + f:close() + return src, candidates[i] + end + end + + fail('source file not found: ' .. tostring(relpath)) +end + + +function tests.test_provider_installs_default_hotplug_and_mwan3_hooks() + local src = read_source('src/services/hal/backends/network/providers/openwrt/init.lua') + ok(src:find('install_observer_hooks(self)', 1, true), 'provider should install observer hooks before starting observer') + ok(src:find('/etc/hotplug.d/iface', 1, true), 'iface hotplug hook path expected') + ok(src:find('/etc/hotplug.d/net', 1, true), 'net hotplug hook path expected') + ok(src:find('/etc/mwan3.user', 1, true), 'mwan3.user hook path expected') + ok(src:find('cfg.install_observer_hooks == false', 1, true), 'hook installation should remain explicitly disableable') + ok(src:find('skipped:fake_uci', 1, true), 'fake UCI tests should not touch host /etc by default') + ok(src:find('source_tree_hotplug_sender_path', 1, true), 'hook sender path should be derived from the running source tree by default') + ok(src:find('logger %-t devicecode%-net%-observe'), 'generated hooks should leave syslog diagnostics when forwarding fails') + ok(src:find("hooks_ok == true and 'info'", 1, true), 'successful hook installation should be visible at info level') +end + +function tests.test_provider_watch_op_does_not_use_private_run_scope() + local src = read_source('src/services/hal/backends/network/providers/openwrt/init.lua') + local body = src:match('function Provider:watch_op%b()%s*(.-)\nfunction Provider:ingest_observation') + ok(body, 'watch_op body not found') + if body:find('run_scope_op', 1, true) then + fail('watch_op must not use run_scope_op; it starts long-lived observer workers in caller scope') + end + ok(body:find('op.guard', 1, true), 'watch_op should be an immediate guard op') +end + + +function tests.test_generated_hooks_pass_event_variables_explicitly_to_sender() + local provider = require 'services.hal.backends.network.providers.openwrt.init' + local src = read_source('src/services/hal/backends/network/providers/openwrt/init.lua') + ok(src:find('--ACTION="${ACTION:-}"', 1, true), 'hook should pass ACTION explicitly') + ok(src:find('--INTERFACE="${INTERFACE:-}"', 1, true), 'hook should pass INTERFACE explicitly') + ok(src:find('socket missing:', 1, true), 'hook should log missing observer socket') + ok(src:find('send failed source=', 1, true), 'hook should log sender failure diagnostics') + local block = provider._test.mwan3_user_block({ observer_socket_path = '/tmp/devicecode.sock', hotplug_sender_path = '/tmp/hotplug_send.lua' }) + ok(block:find('connected', 1, true), 'mwan3.user block should include connected events') + ok(block:find('disconnected', 1, true), 'mwan3.user block should include disconnected events') +end + +function tests.test_hotplug_sender_bootstraps_source_tree_lib_and_vendor_package_paths() + local src = read_source('src/services/hal/backends/network/providers/openwrt/hotplug_send.lua') + ok(src:find('bootstrap_package_path', 1, true), 'hotplug sender should bootstrap its Lua package path') + ok(src:find("/lib/?.lua", 1, true), 'hotplug sender should add flattened production lib path') + ok(src:find("/lib/?/init.lua", 1, true), 'hotplug sender should add flattened production lib init path') + ok(src:find('vendor/lua%-fibers/src'), 'hotplug sender should retain development lua-fibers path') + ok(src:find('vendor/lua%-trie/src'), 'hotplug sender should retain development lua-trie src path') + ok(src:find('split_sender_path', 1, true), 'hotplug sender should derive roots from its own path') +end + +function tests.test_observer_logs_mwan3_trigger_ingress_at_info_level() + local src = read_source('src/services/hal/backends/network/providers/openwrt/observer.lua') + ok(src:find('network_observer_mwan3_trigger', 1, true), 'mwan3 socket ingress should be visible at info level') + ok(src:find('network_observer_hotplug_trigger', 1, true), 'iface/net hotplug socket ingress should be visible at info level') +end + +return tests diff --git a/tests/unit/hal/openwrt_network_provider_advanced_spec.lua b/tests/unit/hal/openwrt_network_provider_advanced_spec.lua new file mode 100644 index 00000000..b14aaf6c --- /dev/null +++ b/tests/unit/hal/openwrt_network_provider_advanced_spec.lua @@ -0,0 +1,755 @@ +-- tests/unit/hal/openwrt_network_provider_advanced_spec.lua + +local fibers = require 'fibers' +local provider_loader = require 'services.hal.backends.network.provider' +local openwrt_provider = require 'services.hal.backends.network.providers.openwrt.init' +local names = require 'services.hal.backends.network.providers.openwrt.names' +local net_config = require 'services.net.config' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function ok(v, msg) if not v then fail(msg) end return v end +local function eq(a, b, msg) if a ~= b then fail((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end end +local function contains(s, needle, msg) if type(s) ~= 'string' or not s:find(needle, 1, true) then fail(msg or ('expected ' .. tostring(s) .. ' to contain ' .. tostring(needle))) end end + + +local MWAN_RULESET = [[ +*mangle +:mwan3_iface_in_wan - [0:0] +:mwan3_iface_in_wanb - [0:0] +:mwan3_iface_in_wanc - [0:0] +:mwan3_policy_balanced - [0:0] +-A mwan3_iface_in_wan -i eth1 -m mark --mark 0x0/0x3f00 -m comment --comment wan -j MARK --set-xmark 0x100/0x3f00 +-A mwan3_iface_in_wanb -i eth2 -m mark --mark 0x0/0x3f00 -m comment --comment wanb -j MARK --set-xmark 0x300/0x3f00 +-A mwan3_iface_in_wanc -i eth3 -m mark --mark 0x0/0x3f00 -m comment --comment wanc -j MARK --set-xmark 0x500/0x3f00 +-A mwan3_policy_balanced -m mark --mark 0x0/0x3f00 -m statistic --mode random --probability 0.50000000000 -m comment --comment "wanb 3 6" -j MARK --set-xmark 0x300/0x3f00 +-A mwan3_policy_balanced -m mark --mark 0x0/0x3f00 -m comment --comment "wan 3 3" -j MARK --set-xmark 0x100/0x3f00 +COMMIT +]] + +local function argv_s(argv) + return table.concat(argv or {}, ' ') +end + +local function intent() + return { + schema = 'devicecode.net.intent/1', + rev = 200, + segments = { + lan = { kind = 'lan', vlan = { id = 10 }, addressing = { ipv4 = { mode = 'static', cidr = '192.168.10.1/24' } }, dhcp = { enabled = true }, firewall = { zone = 'lan' } }, + guest = { kind = 'guest', vlan = 30, firewall = { zone = 'guest' } }, + wan = { kind = 'wan', firewall = { zone = 'wan' } }, + }, + interfaces = { + lan = { kind = 'bridge', role = 'lan', segment = 'lan', members = { 'eth0' }, addressing = { ipv4 = { mode = 'static', cidr = '192.168.10.1/24' } } }, + wan_a = { kind = 'cellular', role = 'wan', segment = 'wan', endpoint = { ifname = 'wwan0' }, addressing = { ipv4 = { mode = 'dhcp' } } }, + wan_b = { kind = 'cellular', role = 'wan', segment = 'wan', endpoint = { ifname = 'wwan1' }, addressing = { ipv4 = { mode = 'dhcp' } } }, + }, + firewall = { zones = { lan = {}, guest = {}, wan = { masq = true } }, policies = { guest_to_wan = { from = 'guest', to = 'wan' } } }, + routing = {}, dns = {}, dhcp = {}, + wan = { + enabled = true, + load_balancing = { speedtests = true, policy = 'balanced' }, + rules = { https = { family = 'ipv4', proto = 'tcp', dest_port = '443', policy = 'balanced', sticky = true } }, + health = { track_ip = { '1.1.1.1', '8.8.8.8' }, reliability = 1 }, + members = { + gsm_a = { interface = 'wan_a', mwan_metric = 1, weight = 1, shaping = { download = { limit = '5mbit' }, upload = { limit = '2mbit' } } }, + gsm_b = { interface = 'wan_b', mwan_metric = 1, weight = 1 }, + }, + }, + vpn = {}, diagnostics = {}, + } +end + +function tests.test_plan_reports_vlan_mwan3_and_shaping_domains() + fibers.run(function() + local provider = ok(provider_loader.new({ provider = 'openwrt', allow_fake_uci = true }, {})) + local plan = fibers.perform(provider:plan_op({ intent = intent() })) + eq(plan.ok, true) + eq(plan.plan.domains.vlan.status, 'implemented') + eq(plan.plan.domains.multiwan.status, 'implemented') + eq(plan.plan.domains.shaping.status, 'implemented') + ok(plan.plan.packages.mwan3.changes > 0, 'mwan3 should have UCI changes') + provider:terminate('test complete') + end) +end + +function tests.test_mwan_status_keeps_link_up_separate_from_online_health() + local before = os.time() + local observed = openwrt_provider._test.normalise_mwan3_status({ + interfaces = { + wan = { + status = 'offline', + enabled = true, + running = true, + tracking = 'active', + up = true, + uptime = 123, + online = 4, + offline = 1, + }, + }, + }) + local wan = observed.interfaces_by_semantic.wan + ok(wan, 'wan status expected') + eq(wan.state, 'offline') + eq(wan.mwan3_status, 'offline') + eq(wan.up, true) + eq(wan.online, false) + eq(wan.usable, false) + eq(wan.online_for, 4) + eq(wan.online_since, nil) + eq(wan.online_count, 4) + eq(wan.offline_for, 1) + ok(wan.offline_since >= before - 1 and wan.offline_since <= os.time(), 'offline_since should be epoch seconds') +end + + +function tests.test_mwan_uplinks_get_distinct_network_route_metrics_and_default_routes() + fibers.run(function() + local provider = ok(provider_loader.new({ provider = 'openwrt', allow_fake_uci = true }, {})) + local plan = fibers.perform(provider:plan_op({ intent = intent() })) + eq(plan.ok, true) + local names = plan.openwrt_names or {} + local seen = {} + local metrics = {} + for _, ch in ipairs(plan.plan and plan.plan.raw_changes and plan.plan.raw_changes.network or {}) do + if ch.op == 'set' and ch.config == 'network' and ch.option == 'metric' then + metrics[ch.section] = tostring(ch.value) + elseif ch.op == 'set' and ch.config == 'network' and ch.option == 'defaultroute' and ch.value == '0' then + fail('mwan uplink must not emit defaultroute 0 on ' .. tostring(ch.section)) + end + end + local ctx = require('services.hal.backends.network.providers.openwrt.names').allocate(intent()) + for _, semantic in ipairs({ 'wan_a', 'wan_b' }) do + local sec = ctx:iface(semantic) + ok(metrics[sec], 'route metric expected on ' .. semantic) + if seen[metrics[sec]] then fail('duplicate route metric ' .. tostring(metrics[sec])) end + seen[metrics[sec]] = true + end + provider:terminate('test complete') + end) +end + + +function tests.test_mwan_rules_flow_from_config_to_mwan3_uci() + fibers.run(function() + local provider = ok(provider_loader.new({ provider = 'openwrt', allow_fake_uci = true }, {})) + local plan = fibers.perform(provider:plan_op({ intent = intent() })) + eq(plan.ok, true) + local ctx = require('services.hal.backends.network.providers.openwrt.names').allocate(intent()) + local rule = ctx:mwan_rule('https') + local seen = {} + for _, ch in ipairs(plan.plan and plan.plan.raw_changes and plan.plan.raw_changes.mwan3 or {}) do + if ch.config == 'mwan3' and ch.section == rule then seen[ch.option] = tostring(ch.value) end + end + eq(seen.proto, 'tcp', 'https sticky proto') + eq(seen.dest_port, '443', 'https sticky dest port') + eq(seen.family, 'ipv4', 'https sticky family') + eq(seen.sticky, '1', 'https sticky flag') + eq(seen.use_policy, ctx:mwan_policy('balanced'), 'https sticky policy') + provider:terminate('test complete') + end) +end + + +function tests.test_read_counters_reads_requested_device_stats_and_reports_missing() + fibers.run(function() + local calls = {} + local values = { + ['br-adm'] = { rx_bytes = 12345, tx_packets = 77 }, + } + local provider = ok(provider_loader.new({ + provider = 'openwrt', + counter_reader = function(device, stat) + calls[#calls + 1] = { device = device, stat = stat } + local v = values[device] and values[device][stat] + if v == nil then return nil, 'missing counter' end + return v, nil + end, + }, {})) + local result = fibers.perform(provider:read_counters_op({ + interfaces = { 'adm' }, + devices = { adm = 'br-adm' }, + stats = { 'rx_bytes', 'tx_packets', 'rx_errors' }, + })) + eq(result.ok, true) + eq(result.counters.adm.device, 'br-adm') + eq(result.counters.adm.statistics.rx_bytes, 12345) + eq(result.counters.adm.statistics.tx_packets, 77) + eq(result.errors.adm.device, 'br-adm') + contains(result.errors.adm.rx_errors, 'missing counter') + eq(#calls, 3) + provider:terminate('test complete') + end) +end + +function tests.test_apply_uses_shaper_and_schedules_structural_mwan3_activation() + fibers.run(function(scope) + local restart_cmds = {} + local shaper_cmds = {} + local provider = ok(provider_loader.new({ + provider = 'openwrt', + allow_fake_uci = true, + debounce_s = 0.01, + run_cmd = function(argv) restart_cmds[#restart_cmds + 1] = table.concat(argv, ' '); return true, nil end, + shaper_run_cmd = function(argv) shaper_cmds[#shaper_cmds + 1] = table.concat(argv, ' '); return true, nil end, + shaper_run_restore = function(_) return true, nil, '' end, + }, {})) + local result = fibers.perform(provider:apply_op({ intent = intent() })) + eq(result.ok, true) + ok(result.multiwan and result.multiwan.enabled == true, 'multiwan plan expected') + ok(result.shaping and result.shaping.ok == true, 'shaping result expected') + ok(#shaper_cmds > 0, 'tc shaper commands expected') + provider:terminate('test complete') + end) +end + +function tests.test_segment_shaping_targets_segment_data_device() + fibers.run(function() + local shaper_cmds = {} + local segment_intent = { + schema = 'devicecode.net.intent/1', + rev = 1, + segments = { + jan = { + kind = 'user', + vlan = { id = 32 }, + addressing = { ipv4 = { mode = 'static', cidr = '172.28.32.1/30' } }, + shaping = { + download = { limit = '10mbit' }, + upload = { limit = '10mbit' }, + host_default = { + mode = 'budgeted_peak', + all_hosts = true, + download = { sustained_rate = '1mbit', peak_rate = '2mbit', burst_budget = '100k' }, + upload = { sustained_rate = '1mbit', peak_rate = '2mbit', burst_budget = '100k' }, + }, + }, + }, + direct = { + kind = 'user', + l2 = { mode = 'direct' }, + vlan = { id = 40 }, + addressing = { ipv4 = { mode = 'static', cidr = '172.28.40.1/30' } }, + shaping = { + download = { limit = '10mbit' }, + upload = { limit = '10mbit' }, + host_default = { + mode = 'budgeted_peak', + all_hosts = true, + download = { sustained_rate = '1mbit', peak_rate = '2mbit', burst_budget = '100k' }, + upload = { sustained_rate = '1mbit', peak_rate = '2mbit', burst_budget = '100k' }, + }, + }, + }, + }, + interfaces = {}, + dns = {}, dhcp = {}, firewall = { zones = {}, policies = {}, rules = {} }, + routing = {}, wan = {}, vpn = {}, diagnostics = {}, + } + local provider = ok(provider_loader.new({ + provider = 'openwrt', + allow_fake_uci = true, + platform = { segment_trunk = { ifname = 'eth0' } }, + shaper_run_cmd = function(argv) shaper_cmds[#shaper_cmds + 1] = table.concat(argv, ' '); return true, nil end, + shaper_run_restore = function(_) return true, nil, '' end, + }, {})) + local result = fibers.perform(provider:apply_op({ intent = segment_intent })) + eq(result.ok, true) + ok(result.shaping and result.shaping.ok == true, 'segment shaping should be applied') + eq(result.shaping.links.jan.iface, 'br-jan', 'bridged segment shaping should target the bridge data device') + eq(result.shaping.links.direct.iface, 'vl-direct', 'direct segment shaping should target the VLAN data device') + ok(#shaper_cmds > 0, 'segment shaping should emit tc commands') + provider:terminate('test complete') + end) +end + +function tests.test_live_weight_update_uses_iptables_restore_and_persists_without_restart() + fibers.run(function() + local restores, restart_cmds = {}, {} + local provider = ok(provider_loader.new({ + provider = 'openwrt', + allow_fake_uci = true, + debounce_s = 0.01, + run_cmd = function(argv) restart_cmds[#restart_cmds + 1] = argv_s(argv); return true, nil end, + mwan_run_cmd_capture = function(argv) + eq(argv_s(argv), 'iptables-save -t mangle') + return true, MWAN_RULESET, nil + end, + mwan_run_restore = function(content) restores[#restores + 1] = content; return true, nil end, + mwan_run_cmd = function(_argv) fail('live weights must not use sequential iptables commands') end, + }, {})) + local result = fibers.perform(provider:apply_live_weights_op({ + policy = 'balanced', + members = { + { interface = 'wan', metric = 1, weight = 70 }, + { interface = 'wanb', metric = 1, weight = 30 }, + }, + persist = true, + })) + eq(result.ok, true) + eq(result.persisted, true) + eq(#restores, 1, 'one iptables-restore payload expected') + contains(restores[1], '*mangle') + contains(restores[1], '-F mwan3_policy_balanced') + contains(restores[1], '--probability 0.70000000000') + contains(restores[1], '--set-xmark 0x100/0x3f00') + contains(restores[1], '--comment "wanb 30 30"') + contains(restores[1], '--set-xmark 0x300/0x3f00') + contains(restores[1], 'COMMIT') + if table.concat(restart_cmds, '\n'):find('mwan3', 1, true) then fail('live weights must not restart mwan3') end + provider:terminate('test complete') + end) +end + +function tests.test_live_weight_three_member_conditional_probabilities() + local mwan3 = require 'services.hal.backends.network.providers.openwrt.mwan3' + local restore, err = mwan3.build_live_weight_restore({ + policy = 'balanced', + members = { + { interface = 'wan', weight = 50 }, + { interface = 'wanb', weight = 30 }, + { interface = 'wanc', weight = 20 }, + }, + }, MWAN_RULESET) + ok(restore, err) + contains(restore, '--probability 0.50000000000') -- 50 / 100 + contains(restore, '--set-xmark 0x100/0x3f00') + contains(restore, '--probability 0.60000000000') -- 30 / remaining 50 + contains(restore, '--set-xmark 0x300/0x3f00') + contains(restore, '--comment "wanc 20 20"') + contains(restore, '--set-xmark 0x500/0x3f00') +end + +function tests.test_live_weight_negative_cases() + local mwan3 = require 'services.hal.backends.network.providers.openwrt.mwan3' + local restore, err = mwan3.build_live_weight_restore({ policy = 'balanced', members = { { interface = 'missing', weight = 1 } } }, MWAN_RULESET) + eq(restore, nil) + contains(err, 'no MWAN3 firewall mark found') + + restore, err = mwan3.build_live_weight_restore({ policy = 'absent', members = { { interface = 'wan', weight = 1 } } }, MWAN_RULESET) + eq(restore, nil) + contains(err, 'MWAN3 policy chain not found') + + restore, err = mwan3.build_live_weight_restore({ policy = 'balanced', members = { { interface = 'wan', weight = 1, enabled = false } } }, MWAN_RULESET) + eq(restore, nil) + contains(err, 'no enabled positive-weight') + + restore, err = mwan3.build_live_weight_restore({ policy = 'balanced', members = { { interface = 'wan', weight = 1 } } }, MWAN_RULESET) + ok(restore, err) + if restore:find('%-%-probability', 1, true) then fail('single-member policy must not include random probability') end + contains(restore, '--comment "wan 1 1"') +end + +function tests.test_speedtest_uses_mwan3_use_boundary() + fibers.run(function() + local argv_seen + local provider = ok(provider_loader.new({ + provider = 'openwrt', + allow_fake_uci = true, + speedtest_run_cmd = function(argv) + argv_seen = argv + return true, '42', nil + end, + }, {})) + local result = fibers.perform(provider:speedtest_op({ interface = 'wan_a', device = 'wwan0' })) + eq(result.ok, true) + eq(result.peak_mbps, 42) + eq(argv_seen[1], 'mwan3') + eq(argv_seen[2], 'use') + eq(argv_seen[3], 'wan_a') + provider:terminate('test complete') + end) +end + +function tests.test_speedtest_translates_semantic_device_to_linux_counter_device() + fibers.run(function() + local seen_req + local provider = ok(provider_loader.new({ + provider = 'openwrt', + allow_fake_uci = true, + speedtest_run_cmd = function() + return true, '42', nil + end, + }, {})) + provider._last_name_ctx = ok(names.allocate({ + segments = { wan = { kind = 'wan', vlan = 10 } }, + interfaces = { wan = { kind = 'ethernet', role = 'wan', segment = 'wan' } }, + wan = { members = { wan = { interface = 'wan' } } }, + })) + provider.speedtest_run_cmd = function(_argv) + return true, '42', nil + end + local speedtest = require 'services.hal.backends.network.providers.openwrt.speedtest' + local original = speedtest.run_op + speedtest.run_op = function(req, opts) + seen_req = req + return original(req, opts) + end + local result = fibers.perform(provider:speedtest_op({ interface = 'wan' })) + speedtest.run_op = original + eq(result.ok, true) + eq(seen_req.interface, 'wan') + eq(seen_req.device, 'vl-wan') + provider:terminate('test complete') + end) +end + + +function tests.test_segment_trunk_realises_segments_without_cfg_net_interfaces() + fibers.run(function() + local provider = ok(provider_loader.new({ + provider = 'openwrt', + allow_fake_uci = true, + platform = { segment_trunk = { ifname = 'eth0', protected = true } }, + }, {})) + local plan = fibers.perform(provider:plan_op({ intent = { + schema = 'devicecode.net.intent/1', + rev = 1, + segments = { + mgmt = { kind = 'system', protected = true, vlan = { id = 10 }, addressing = { ipv4 = { mode = 'static', cidr = '192.168.8.1/24' } } }, + guest = { kind = 'guest', vlan = { id = 101 }, addressing = { ipv4 = { mode = 'static', cidr = '192.168.101.1/24' } } }, + }, + interfaces = {}, + firewall = { zones = { mgmt = {}, guest = {} }, policies = {} }, + routing = {}, dns = {}, dhcp = {}, wan = {}, vpn = {}, diagnostics = {}, + } })) + eq(plan.ok, true) + ok(plan.plan.packages.network.sections >= 5, 'segment trunk should create globals plus interface/device sections') + provider:terminate('test complete') + end) +end + + +local function read_project_file(rel) + local candidates = { rel, '../' .. rel } + for i = 1, #candidates do + local f = io.open(candidates[i], 'rb') + if f then local data = f:read('*a'); f:close(); return data end + end + return nil, 'unable to read ' .. rel +end + +function tests.test_bigbox_clean_config_plans_dns_rules_routes_and_segment_shaping() + fibers.run(function() + local cjson = require 'cjson.safe' + local cfg_mod = require 'services.net.config' + local text = ok(read_project_file('src/configs/bigbox-v1-cm-2.json')) + local doc = ok(cjson.decode(text), 'bigbox config must decode') + local intent = ok(cfg_mod.normalise(doc.net, { generation = 1 })) + local shaper_cmds = {} + local provider = ok(provider_loader.new({ + provider = 'openwrt', + allow_fake_uci = true, + platform = { segment_trunk = { ifname = 'eth0' } }, + shaper_run_cmd = function(argv) shaper_cmds[#shaper_cmds + 1] = table.concat(argv, ' '); return true, nil end, + shaper_run_restore = function(_) return true, nil, '' end, + }, {})) + local plan = fibers.perform(provider:plan_op({ intent = intent })) + eq(plan.ok, true) + ok(plan.plan.packages.dhcp.changes > 0, 'dns/dhcp changes expected') + ok(plan.plan.packages.firewall.changes > 0, 'firewall changes expected') + ok(plan.plan.packages.network.changes > 0, 'network changes expected') + local result = fibers.perform(provider:apply_op({ intent = intent })) + eq(result.ok, true) + ok(result.shaping and result.shaping.ok == true, 'segment shaping should be applied') + ok(result.shaping.links and result.shaping.links.jan, 'jan shaping link should be reported') + eq(result.shaping.links.jan.iface, 'br-jan', 'jan shaping should target the bridge data device') + ok(#shaper_cmds > 0, 'segment shaping should emit tc commands') + provider:terminate('test complete') + end) +end + + + +function tests.test_bigbox_starlink_admin_route_is_explicit_host_route() + fibers.run(function() + local cjson = require 'cjson.safe' + local cfg_mod = require 'services.net.config' + local text = ok(read_project_file('src/configs/bigbox-v1-cm-2.json')) + local doc = ok(cjson.decode(text), 'bigbox config must decode') + local intent = ok(cfg_mod.normalise(doc.net, { generation = 1 })) + local provider = ok(provider_loader.new({ + provider = 'openwrt', + allow_fake_uci = true, + platform = { segment_trunk = { ifname = 'eth0' } }, + }, {})) + local plan = fibers.perform(provider:plan_op({ intent = intent })) + eq(plan.ok, true) + local route = {} + for _, ch in ipairs(plan.plan and plan.plan.raw_changes and plan.plan.raw_changes.network or {}) do + if ch.config == 'network' and ch.section == 'route_starlink_admin' then route[ch.option] = tostring(ch.value) end + end + eq(route.interface, 'wan', 'Starlink route interface') + eq(route.target, '192.168.100.1', 'Starlink route target') + eq(route.netmask, '255.255.255.255', 'Starlink host route netmask') + provider:terminate('test complete') + end) +end + +function tests.test_mwan3_builder_uses_distinct_section_names_for_same_interface_and_member_ids() + local names = require 'services.hal.backends.network.providers.openwrt.names' + local mwan3 = require 'services.hal.backends.network.providers.openwrt.mwan3' + local intent_doc = { + wan = { + enabled = true, + health = { track_ip = { '1.1.1.1' } }, + members = { + wan = { interface = 'wan', mwan_metric = 1, weight = 1 }, + modem_primary = { interface = 'modem_primary', mwan_metric = 1, weight = 1 }, + modem_secondary = { interface = 'modem_secondary', mwan_metric = 1, weight = 1 }, + }, + }, + } + local ctx = ok(names.allocate(intent_doc)) + local changes = ok(mwan3.build_changes(intent_doc, ctx)) + local sections = {} + local policy = ctx:mwan_policy('balanced') + local use_members = {} + for _, ch in ipairs(changes) do + if ch.op == 'set' and ch.config == 'mwan3' and ch.value == nil then + if sections[ch.section] then fail('duplicate mwan3 section name generated: ' .. tostring(ch.section)) end + sections[ch.section] = ch.option + elseif ch.config == 'mwan3' and ch.section == policy and ch.option == 'use_member' then + if ch.op == 'delete' then fail('full mwan3 package replacement should not delete use_member before it exists') end + if ch.op == 'add_list' then use_members[tostring(ch.value)] = (use_members[tostring(ch.value)] or 0) + 1 end + end + end + ok(sections[ctx:mwan_iface('wan')], 'wan interface section expected') + ok(sections[ctx:mwan_member('wan')], 'wan member section expected') + eq(ctx:mwan_iface('wan') == ctx:mwan_member('wan'), false, 'interface/member names must differ') + for _, mid in ipairs({ 'wan', 'modem_primary', 'modem_secondary' }) do + eq(use_members[ctx:mwan_member(mid)], 1, 'use_member should contain each intended member exactly once') + end + + local weight_changes = ok(mwan3.build_weight_only_changes({ members = { + { id = 'wan', interface = 'wan', metric = 2, weight = 70 }, + { id = 'modem_primary', interface = 'modem_primary', metric = 1, weight = 30 }, + } }, ctx)) + local seen = {} + for _, ch in ipairs(weight_changes) do + eq(ch.config, 'mwan3') + eq(ch.op, 'set') + ok(ch.option == 'weight' or ch.option == 'metric', 'weight-only persistence must only set member weight/metric') + ok(ch.section == ctx:mwan_member('wan') or ch.section == ctx:mwan_member('modem_primary'), 'weight-only persistence must target member sections') + seen[ch.section .. '.' .. ch.option] = ch.value + end + eq(seen[ctx:mwan_member('wan') .. '.weight'], 70) + eq(seen[ctx:mwan_member('wan') .. '.metric'], 2) + eq(seen[ctx:mwan_member('modem_primary') .. '.weight'], 30) + eq(seen[ctx:mwan_member('modem_primary') .. '.metric'], 1) + + fibers.run(function() + local op = require 'fibers.op' + local submitted + local mgr = { + submit_op = function(_, record) + submitted = record + return op.always(true, nil, true) + end, + } + local ok_persist, err, admitted = fibers.perform(mwan3.persist_weights_op(mgr, { members = { + { id = 'wan', interface = 'wan', metric = 3, weight = 44 }, + } }, ctx)) + eq(ok_persist, true, tostring(err)) + eq(admitted, true) + eq(submitted.config, 'mwan3') + eq(#submitted.restart_cmds, 0, 'weight persistence must not restart mwan3') + for _, ch in ipairs(submitted.changes or {}) do + eq(ch.op, 'set') + ok(ch.option == 'weight' or ch.option == 'metric', 'persist_weights_op must only set weight/metric') + end + end) +end + + +local function find_change(changes, fields) + for _, ch in ipairs(changes or {}) do + local ok = true + for k, v in pairs(fields or {}) do + if ch[k] ~= v then ok = false; break end + end + if ok then return ch end + end + return nil +end + +function tests.test_bigbox_net_intent_still_renders_openwrt_segment_trunk_independent_of_wired_assembly() + fibers.run(function() + local intent, ierr = net_config.normalise({ + schema = net_config.SCHEMA, + version = 1, + segments = { + adm = { kind = 'system', protected = true, vlan = { id = 8 }, addressing = { ipv4 = { mode = 'static', cidr = '172.28.8.1/24' } }, firewall = { zone = 'lan' } }, + jan = { kind = 'user', vlan = { id = 32 }, addressing = { ipv4 = { mode = 'static', cidr = '172.28.32.1/24' } }, firewall = { zone = 'lan_rst' } }, + int = { kind = 'system', protected = true, vlan = { id = 100 }, addressing = { ipv4 = { mode = 'static', cidr = '172.28.100.1/24' } }, firewall = { zone = 'lan' } }, + wan = { kind = 'wan', vlan = { id = 4 }, addressing = { ipv4 = { mode = 'dhcp', peerdns = false } }, firewall = { zone = 'wan' } }, + }, + interfaces = {}, + dns = {}, dhcp = {}, firewall = { zones = { lan = {}, lan_rst = {}, wan = { masq = true } }, policies = {}, rules = {} }, + routing = { routes = { starlink_admin = { kind = 'host', target = '192.168.100.1', interface = 'wan' } } }, + wan = { enabled = true, members = { wan = { interface = 'wan', weight = 1, mwan_metric = 1 } } }, + vpn = {}, diagnostics = {}, + }, { rev = 1 }) + ok(intent, ierr) + local provider = ok(provider_loader.new({ + provider = 'openwrt', + allow_fake_uci = true, + platform = { segment_trunk = { ifname = 'eth0', protected = true } }, + }, {})) + local plan = fibers.perform(provider:plan_op({ intent = intent })) + eq(plan.ok, true) + local changes = plan.plan and plan.plan.raw_changes and plan.plan.raw_changes.network or {} + + for _, rec in ipairs({ + { id = 'adm', vid = 8, vlan = 'vl-adm', bridge = 'br-adm', proto = 'static', ipaddr = '172.28.8.1' }, + { id = 'jan', vid = 32, vlan = 'vl-jan', bridge = 'br-jan', proto = 'static', ipaddr = '172.28.32.1' }, + { id = 'int', vid = 100, vlan = 'vl-int', bridge = 'br-int', proto = 'static', ipaddr = '172.28.100.1' }, + }) do + local vlan_sec = 'dev_vlan_' .. rec.id + local bridge_sec = 'dev_bridge_' .. rec.id + ok(find_change(changes, { section = vlan_sec, option = 'ifname', value = 'eth0' }), rec.id .. ' VLAN should use eth0 trunk') + ok(find_change(changes, { section = vlan_sec, option = 'vid', value = rec.vid }), rec.id .. ' VLAN id should render') + ok(find_change(changes, { section = vlan_sec, option = 'name', value = rec.vlan }), rec.id .. ' VLAN device should render') + ok(find_change(changes, { section = bridge_sec, option = 'name', value = rec.bridge }), rec.id .. ' bridge should render') + ok(find_change(changes, { section = rec.id, option = 'device', value = rec.bridge }), rec.id .. ' interface should use bridge') + ok(find_change(changes, { section = rec.id, option = 'proto', value = rec.proto }), rec.id .. ' proto should render') + ok(find_change(changes, { section = rec.id, option = 'ipaddr', value = rec.ipaddr }), rec.id .. ' IP address should render') + end + + ok(find_change(changes, { section = 'dev_vlan_wan', option = 'ifname', value = 'eth0' }), 'wan VLAN should use eth0 trunk') + ok(find_change(changes, { section = 'dev_vlan_wan', option = 'vid', value = 4 }), 'wan VLAN id should render') + ok(find_change(changes, { section = 'wan', option = 'device', value = 'vl-wan' }), 'wan interface should use WAN VLAN device') + ok(find_change(changes, { section = 'wan', option = 'proto', value = 'dhcp' }), 'wan interface should remain DHCP') + ok(find_change(changes, { section = 'wan', option = 'peerdns', value = '0' }), 'wan peerdns false should render') + ok(find_change(changes, { section = 'route_starlink_admin', option = 'interface', value = 'wan' }), 'Starlink route should remain on semantic wan interface') + ok(find_change(changes, { section = 'route_starlink_admin', option = 'target', value = '192.168.100.1' }), 'Starlink route target should render') + provider:terminate('test complete') + end) +end + +function tests.test_segment_shaping_compiles_budgeted_peak_layout() + fibers.run(function() + local batch_text = {} + local segment_intent = { + schema = 'devicecode.net.intent/1', rev = 1, + segments = { + bpagg = { kind = 'user', vlan = { id = 232 }, addressing = { ipv4 = { mode = 'static', cidr = '172.28.32.1/30' } }, shaping = { download = { limit = '40mbit' }, upload = { limit = '10mbit' }, host_default = { mode = 'budgeted_peak', download = { sustained_rate = '2mbit', peak_rate = '8mbit', burst_budget = '500k' }, upload = { sustained_rate = '1500kbit', peak_rate = '6mbit', burst_budget = '225k' } } } }, + }, + interfaces = {}, dns = {}, dhcp = {}, firewall = { zones = {}, policies = {}, rules = {} }, routing = {}, wan = {}, vpn = {}, diagnostics = {}, + } + local provider = ok(provider_loader.new({ + provider = 'openwrt', allow_fake_uci = true, platform = { segment_trunk = { ifname = 'eth0' } }, + shaper_run_cmd = function(argv) + if argv[1] == 'tc' and argv[2] == '-batch' and type(argv[3]) == 'string' then + local f = io.open(argv[3], 'rb') + if f then batch_text[#batch_text + 1] = f:read('*a') or ''; f:close() end + end + return true, '', nil + end, + }, {})) + local result = fibers.perform(provider:apply_op({ intent = segment_intent })) + eq(result.ok, true) + eq(result.shaping.links.bpagg.iface, 'br-bpagg') + local all = table.concat(batch_text, '\n') + contains(all, 'qdisc add dev br-bpagg root handle 1: htb default 1', 'aggregate budgeted_peak should keep unmatched root traffic on root default class') + contains(all, 'class replace dev br-bpagg parent 20: classid 20:1002 htb rate 2mbit burst 500k ceil 2mbit cburst 500k', 'download host budget should be under aggregate with sustained ceil') + contains(all, 'qdisc add dev br-bpagg parent 20:1002 handle 1002: htb default 1', 'download host budget should own a peak HTB qdisc') + contains(all, 'class replace dev br-bpagg parent 1002: classid 1002:1 htb rate 8mbit ceil 8mbit', 'download host peak class should cap burst spend rate') + contains(all, 'filter add dev br-bpagg parent 20: protocol ip prio 99 handle 1: u32 divisor 256', 'aggregate budgeted_peak should classify hosts under the segment aggregate qdisc') + contains(all, 'qdisc add dev br-bpagg parent 1:20 handle 20: htb default 100', 'aggregate budgeted_peak should install an inner HTB under the segment aggregate') + provider:terminate('test complete') + end) +end + + +function tests.test_inline_segment_shaping_compiles_budgeted_peak_layout() + fibers.run(function() + local batch_text = {} + local segment_intent = { + schema = 'devicecode.net.intent/1', rev = 1, + segments = { + inlinebp = { + kind = 'user', vlan = { id = 233 }, addressing = { ipv4 = { mode = 'static', cidr = '172.28.32.1/30' } }, + shaping = { + download = { limit = '40mbit' }, + upload = { limit = '10mbit' }, + host_default = { + mode = 'budgeted_peak', + download = { sustained_rate = '2mbit', peak_rate = '8mbit', burst_budget = '500k' }, + upload = { sustained_rate = '1500kbit', peak_rate = '6mbit', burst_budget = '225k' }, + }, + }, + }, + }, + interfaces = {}, dns = {}, dhcp = {}, firewall = { zones = {}, policies = {}, rules = {} }, routing = {}, wan = {}, vpn = {}, diagnostics = {}, + } + local provider = ok(provider_loader.new({ + provider = 'openwrt', allow_fake_uci = true, platform = { segment_trunk = { ifname = 'eth0' } }, + shaper_run_cmd = function(argv) + if argv[1] == 'tc' and argv[2] == '-batch' and type(argv[3]) == 'string' then + local f = io.open(argv[3], 'rb') + if f then batch_text[#batch_text + 1] = f:read('*a') or ''; f:close() end + end + return true, '', nil + end, + }, {})) + local result = fibers.perform(provider:apply_op({ intent = segment_intent })) + eq(result.ok, true) + eq(result.shaping.links.inlinebp.iface, 'br-inlinebp') + local all = table.concat(batch_text, '\n') + contains(all, 'class replace dev br-inlinebp parent 20: classid 20:1002 htb rate 2mbit burst 500k ceil 2mbit cburst 500k', 'inline segment host budget should compile') + contains(all, 'class replace dev br-inlinebp parent 1002: classid 1002:1 htb rate 8mbit ceil 8mbit', 'inline segment peak class should compile') + provider:terminate('test complete') + end) +end + +function tests.test_wan_member_shaping_renders_router_exemption_marks_and_wan_mark_link() + fibers.run(function() + local restores = {} + local cmds = {} + local i = intent() + i.wan.members.wired = { + interface = 'wan_a', mwan_metric = 1, weight = 1, + shaping = { download = { limit = '80mbit' }, upload = { limit = '20mbit' } }, + } + local provider = ok(provider_loader.new({ + provider = 'openwrt', allow_fake_uci = true, platform = { shaping = { marks = { mask = '0x00f00000', control = '0x00100000', client = '0x00200000' } } }, + shaper_run_cmd = function(argv) cmds[#cmds + 1] = table.concat(argv, ' '); return true, '', nil end, + shaper_run_restore = function(payload) restores[#restores + 1] = payload; return true, nil, '' end, + }, {})) + local result = fibers.perform(provider:apply_op({ intent = i })) + eq(result.ok, true) + ok(result.shaping.links.backhaul_wired, 'backhaul shaping link should be reported') + eq(result.shaping.links.backhaul_wired.kind, 'wan_mark') + eq(#restores, 1, 'one shaping mangle restore expected') + local r = restores[1] + contains(r, ':DEVICECODE_SHAPING_OUTPUT', 'router-output chain expected') + contains(r, ':DEVICECODE_SHAPING_FORWARD', 'forwarded-client chain expected') + contains(r, '-A DEVICECODE_SHAPING_OUTPUT -o wwan0', 'router traffic should be marked on the WAN device') + contains(r, 'devicecode-shaping router exempt', 'router exemption comment expected') + contains(r, 'CONNMARK --save-mark --mask 0x00f00000', 'router/client marks should be saved to conntrack') + contains(r, '-A DEVICECODE_SHAPING_FORWARD -o wwan0', 'forwarded traffic should be marked client') + local all_cmds = table.concat(cmds, '\n') + contains(all_cmds, 'tc class replace dev wwan0 parent 1:1 classid 1:20 htb rate 20mbit ceil 20mbit', 'WAN upload client class should use upload limit') + contains(all_cmds, 'tc filter add dev wwan0 parent ffff: protocol ip prio 1 u32 match u32 0 0 action ctinfo cpmark 0x00f00000 action mirred egress redirect dev ifb_wwan0', 'WAN download should restore connmark before IFB redirect') + contains(all_cmds, 'tc class replace dev ifb_wwan0 parent 1:1 classid 1:20 htb rate 80mbit ceil 80mbit', 'WAN download client class should use download limit') + provider:terminate('test complete') + end) +end + + +function tests.test_shaping_mark_namespace_does_not_overlap_mwan_default_mask() + local marks = require 'services.hal.backends.network.providers.openwrt.shaping_marks' + local spec, err = marks.validate_marks({ marks = marks.default_marks() }, '0x3f00') + ok(spec, tostring(err)) + eq(spec.mask, '0x00f00000') + local bad, berr = marks.validate_marks({ marks = { mask = '0x00003f00', control = '0x00000100', client = '0x00000200' } }, '0x3f00') + eq(bad, nil) + contains(tostring(berr), 'overlaps MWAN mask') +end + + +return tests diff --git a/tests/unit/hal/openwrt_uci_manager_spec.lua b/tests/unit/hal/openwrt_uci_manager_spec.lua new file mode 100644 index 00000000..ffafe892 --- /dev/null +++ b/tests/unit/hal/openwrt_uci_manager_spec.lua @@ -0,0 +1,381 @@ +-- tests/unit/hal/openwrt_uci_manager_spec.lua + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local queue = require 'devicecode.support.queue' + +local uci_manager = require 'services.hal.backends.openwrt.uci_manager' +local probe = require 'tests.support.bus_probe' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function ok(v, msg) if not v then fail(msg) end return v end +local function eq(a, b, msg) if a ~= b then fail((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function contains(s, needle, msg) if type(s) ~= 'string' or not s:find(needle, 1, true) then fail(msg or ('expected ' .. tostring(s) .. ' to contain ' .. tostring(needle))) end end + +local function fake_cursor(calls) + return { + set = function (_, ...) + calls[#calls + 1] = { op = 'set', ... } + return true + end, + delete = function (_, ...) + calls[#calls + 1] = { op = 'delete', ... } + return true + end, + commit = function (_, config) + calls[#calls + 1] = { op = 'commit', config } + return true + end, + revert = function (_, config) + calls[#calls + 1] = { op = 'revert', config } + return true + end, + } +end + +function tests.test_commit_converts_values_and_runs_restart_commands() + fibers.run(function (scope) + local calls = {} + local restarts = {} + local mgr = ok(uci_manager.new({ + cursor = fake_cursor(calls), + run_cmd = function (argv) + restarts[#restarts + 1] = table.concat(argv, ' ') + return true, nil + end, + })) + ok(mgr:start(scope)) + + local s = mgr:new_session() + s:set('wireless', 'radio0', 'disabled', false) + s:set('wireless', 'ssid0', 'network', { 'lan', 'guest' }) + s:delete('wireless', 'old_ssid') + local ok_commit, err = fibers.perform(s:commit_op('wireless', { + { '/sbin/wifi', 'reload' }, + { kind = 'restart', target = 'network' }, + })) + ok(ok_commit, err) + + eq(calls[1].op, 'set') + eq(calls[1][1], 'wireless') + eq(calls[1][2], 'radio0') + eq(calls[1][3], 'disabled') + eq(calls[1][4], '0') + eq(calls[2][4][1], 'lan') + eq(calls[2][4][2], 'guest') + eq(calls[3].op, 'delete') + eq(calls[4].op, 'commit') + eq(restarts[1], '/sbin/wifi reload') + eq(restarts[2], '/etc/init.d/network restart') + end) +end + + +function tests.test_start_accepts_already_started_owner_scope_from_caller_scope() + fibers.run(function (scope) + local owner = ok(scope:child()) + local hold_tx, hold_rx = mailbox.new(1) + ok(owner:spawn(function () fibers.perform(hold_rx:recv_op()) end)) + + local mgr = ok(uci_manager.new({ + cursor = fake_cursor({}), + run_cmd = function () return true, nil end, + })) + + local started, err = mgr:start(owner) + ok(started, err) + owner:cancel('test_done') + hold_tx:close('test_done') + end) +end + +function tests.test_queue_full_is_reported_without_admission() + fibers.run(function () + local mgr = ok(uci_manager.new({ queue_len = 0, cursor = fake_cursor({}) })) + local ok_submit, err, admitted = fibers.perform(mgr:submit_op({ + config = 'network', + changes = { + { op = 'set', config = 'network', section = 'lan', option = 'proto', value = 'static' }, + }, + })) + eq(ok_submit, false) + eq(admitted, false) + contains(err, 'uci_manager_busy') + end) +end + +function tests.test_restart_commands_are_deduplicated_across_debounce_batch() + fibers.run(function (scope) + local calls = {} + local restarts = {} + local mgr = ok(uci_manager.new({ + cursor = fake_cursor(calls), + debounce_s = 0.02, + run_cmd = function (argv) + restarts[#restarts + 1] = table.concat(argv, ' ') + return true, nil + end, + })) + ok(mgr:start(scope)) + + local results = {} + local r1 = { + config = 'network', + changes = { { op = 'set', config = 'network', section = 'lan', option = 'proto', value = 'static' } }, + restart_cmds = { { '/etc/init.d/network', 'reload' } }, + } + local r2 = { + config = 'network', + changes = { { op = 'set', config = 'network', section = 'wan', option = 'proto', value = 'dhcp' } }, + restart_cmds = { { '/etc/init.d/network', 'reload' } }, + } + + scope:spawn(function () results[1] = { fibers.perform(mgr:submit_op(r1)) } end) + scope:spawn(function () results[2] = { fibers.perform(mgr:submit_op(r2)) } end) + + ok(probe.wait_until(function () return results[1] and results[2] end, { timeout = 0.5 }), 'commit results expected') + eq(results[1][1], true) + eq(results[2][1], true) + eq(#restarts, 1) + eq(restarts[1], '/etc/init.d/network reload') + end) +end + + +function tests.test_record_normalisation_does_not_mutate_caller_owned_record() + local record = { + config = 'network', + changes = { + { op = 'set', config = 'network', section = 'lan', option = 'proto', value = 'static' }, + }, + restart_cmds = { + { kind = 'reload', target = 'network' }, + }, + } + local original_restart = record.restart_cmds[1] + local normalised, err = uci_manager._normalise_record_for_test(record) + ok(normalised, err) + eq(record.restart_cmds[1], original_restart, 'caller restart table should be untouched') + eq(record.restart_cmds[1].kind, 'reload') + eq(normalised.restart_cmds[1][1], '/etc/init.d/network') + normalised.changes[1].value = 'dhcp' + eq(record.changes[1].value, 'static', 'caller changes should be deep-copied') +end + +function tests.test_add_alias_lists_rename_and_reorder_are_applied_in_order() + fibers.run(function (scope) + local calls = {} + local cursor = fake_cursor(calls) + cursor.add = function (_, config, stype) + calls[#calls + 1] = { op = 'add', config, stype } + return 'cfg123abc' + end + local get_count = 0 + cursor.get = function (_, config, section, option) + calls[#calls + 1] = { op = 'get', config, section, option } + get_count = get_count + 1 + if option == 'server' and get_count == 1 then return { '0.openwrt.pool.ntp.org' } end + if option == 'server' then return { '0.openwrt.pool.ntp.org', '1.openwrt.pool.ntp.org' } end + return nil + end + cursor.rename = function (_, ...) + calls[#calls + 1] = { op = 'rename', ... } + return true + end + cursor.reorder = function (_, ...) + calls[#calls + 1] = { op = 'reorder', ... } + return true + end + + local mgr = ok(uci_manager.new({ cursor = cursor, run_cmd = function () return true end })) + ok(mgr:start(scope)) + local s = mgr:new_session() + local anon = s:add('system', 'timeserver') + s:set('system', anon, 'enabled', true) + s:add_list('system', anon, 'server', '1.openwrt.pool.ntp.org') + s:del_list('system', anon, 'server', '0.openwrt.pool.ntp.org') + s:rename('system', anon, 'ntp_runtime') + s:reorder('system', 'ntp_runtime', 0) + local ok_commit, err = fibers.perform(s:commit_op('system')) + ok(ok_commit, err) + + eq(calls[1].op, 'add') + eq(calls[1][1], 'system') + eq(calls[1][2], 'timeserver') + eq(calls[2].op, 'set') + eq(calls[2][2], 'cfg123abc') + eq(calls[2][4], '1') + eq(calls[3].op, 'get') + eq(calls[4].op, 'set') + eq(calls[4][4][2], '1.openwrt.pool.ntp.org') + eq(calls[5].op, 'get') + eq(calls[6].op, 'set') + eq(#calls[6][4], 1) + eq(calls[6][4][1], '1.openwrt.pool.ntp.org') + eq(calls[7].op, 'rename') + eq(calls[7][2], 'cfg123abc') + eq(calls[8].op, 'reorder') + eq(calls[8][2], 'ntp_runtime') + eq(calls[9].op, 'commit') + end) +end + +function tests.test_legacy_restart_shorthands_are_normalised() + local record = { + config = 'wireless', + changes = { + { op = 'set', config = 'wireless', section = 'radio0', option = 'disabled', value = false }, + }, + restart_cmds = { + { 'wifi', 'reload' }, + { 'service', 'dawn', 'restart' }, + }, + } + local normalised, err = uci_manager._normalise_record_for_test(record) + ok(normalised, err) + eq(normalised.restart_cmds[1][1], '/sbin/wifi') + eq(normalised.restart_cmds[1][2], 'reload') + eq(normalised.restart_cmds[2][1], '/etc/init.d/dawn') + eq(normalised.restart_cmds[2][2], 'restart') +end + + +function tests.test_transaction_rolls_back_touched_packages_on_partial_failure() + fibers.run(function (scope) + local calls = {} + local snapshots = { + network = { + lan = { ['.type'] = 'interface', proto = 'static' }, + }, + dhcp = { + dnsmasq = { ['.type'] = 'dnsmasq', domainneeded = '1' }, + }, + } + local cursor = fake_cursor(calls) + cursor.get_all = function (_, pkg) + calls[#calls + 1] = { op = 'get_all', pkg } + return snapshots[pkg] or {} + end + cursor.set = function (_, config, section, option, value) + calls[#calls + 1] = { op = 'set', config, section, option, value } + if config == 'dhcp' and section == 'bad' then return nil, 'synthetic failure' end + return true + end + local mgr = ok(uci_manager.new({ cursor = cursor, run_cmd = function () return true end })) + ok(mgr:start(scope)) + local result, admitted = fibers.perform(mgr:transaction_op({ + packages = { 'network', 'dhcp' }, + records = { + { config = 'network', changes = { { op = 'set', config = 'network', section = 'wan', option = 'interface' } } }, + { config = 'dhcp', changes = { { op = 'set', config = 'dhcp', section = 'bad', option = 'dhcp' } } }, + }, + })) + eq(admitted, true) + eq(result.ok, false) + eq(result.status, 'failed_rolled_back') + eq(result.rollback.ok, true) + end) +end + + +function tests.test_replace_package_deletes_existing_sections_before_writing_desired_state() + fibers.run(function (scope) + local calls = {} + local cursor = fake_cursor(calls) + cursor.get_all = function (_, pkg) + calls[#calls + 1] = { op = 'get_all', pkg } + return { + old = { ['.type'] = 'interface', proto = 'dhcp' }, + lan = { ['.type'] = 'interface', proto = 'static', stale = 'yes' }, + } + end + local mgr = ok(uci_manager.new({ cursor = cursor, run_cmd = function () return true end })) + ok(mgr:start(scope)) + local ok_commit, err = fibers.perform(mgr:submit_op({ + config = 'network', + replace_package = true, + changes = { + { op = 'set', config = 'network', section = 'lan', option = 'interface' }, + { op = 'set', config = 'network', section = 'lan', option = 'proto', value = 'static' }, + }, + })) + ok(ok_commit, err) + local seen_delete_old, seen_delete_lan, seen_set_lan = false, false, false + for _, c in ipairs(calls) do + if c.op == 'delete' and c[1] == 'network' and c[2] == 'old' then seen_delete_old = true end + if c.op == 'delete' and c[1] == 'network' and c[2] == 'lan' then seen_delete_lan = true end + if c.op == 'set' and c[1] == 'network' and c[2] == 'lan' and c[3] == 'interface' then seen_set_lan = true end + end + ok(seen_delete_old, 'old section should be removed') + ok(seen_delete_lan, 'surviving section should be recreated to drop stale options') + ok(seen_set_lan, 'desired section should be written after delete') + end) +end + +function tests.test_manager_creates_missing_package_files_before_transaction() + fibers.run(function (scope) + local tmp = os.tmpname() + os.remove(tmp) + local calls = {} + local cursor = fake_cursor(calls) + cursor.get_all = function (_, _pkg) return {} end + local mgr = ok(uci_manager.new({ confdir = tmp, cursor = cursor, run_cmd = function () return true end })) + ok(mgr:start(scope)) + local result = fibers.perform(mgr:transaction_op({ + packages = { 'network', 'dhcp' }, + records = { + { config = 'network', replace_package = true, changes = {} }, + { config = 'dhcp', replace_package = true, changes = {} }, + }, + })) + ok(result and result.ok == true, result and result.err) + local f = io.open(tmp .. '/network', 'rb') + ok(f, 'network file should be created') + f:close() + f = io.open(tmp .. '/dhcp', 'rb') + ok(f, 'dhcp file should be created') + f:close() + os.remove(tmp .. '/network'); os.remove(tmp .. '/dhcp'); os.remove(tmp) + end) +end + +function tests.test_activation_command_replies_without_waiting_for_command_completion() + fibers.run(function(scope) + local calls = {} + local activation_restarts = {} + local unblock_tx, unblock_rx = mailbox.new(1, { full = 'reject_newest' }) + local cursor = fake_cursor(calls) + cursor.get_all = function (_, _pkg) return {} end + local mgr = ok(uci_manager.new({ + cursor = cursor, + debounce_s = 0.01, + run_cmd = function (argv) + activation_restarts[#activation_restarts + 1] = table.concat(argv, ' ') + fibers.perform(unblock_rx:recv_op()) + return true, nil + end, + })) + ok(mgr:start(scope)) + + local result = { fibers.perform(mgr:transaction_op({ + packages = { 'network' }, + records = { + { + config = 'network', + changes = { { op = 'set', config = 'network', section = 'lan', option = 'proto', value = 'static' } }, + restart_cmds = { { kind = 'reload', target = 'network', wait = false } }, + }, + }, + })) } + + eq(result[1].ok, true) + ok(result[1].activation and result[1].activation.state == 'scheduled', 'activation should be scheduled') + ok(probe.wait_until(function () return #activation_restarts == 1 end, { timeout = 0.5 }), 'activation runner should start command') + eq(activation_restarts[1], '/etc/init.d/network reload') + queue.try_admit_now(unblock_tx, true) + end) +end + +return tests diff --git a/tests/unit/hal/openwrt_uci_singleton_compat_spec.lua b/tests/unit/hal/openwrt_uci_singleton_compat_spec.lua new file mode 100644 index 00000000..1644c96e --- /dev/null +++ b/tests/unit/hal/openwrt_uci_singleton_compat_spec.lua @@ -0,0 +1,42 @@ +-- tests/unit/hal/openwrt_uci_singleton_compat_spec.lua + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end + +local function read_file(path) + local f = assert(io.open(path, 'r')) + local s = f:read('*a') + f:close() + return s +end + +function tests.test_uci_compat_does_not_spawn_into_root_scope() + local s = read_file('../src/services/hal/backends/openwrt/uci_singleton_compat.lua') + if s:find('Scope.root', 1, true) then + fail('UCI compatibility layer must not spawn into Scope.root') + end +end + +function tests.test_uci_manager_exposes_op_first_submit_path() + local s = read_file('../src/services/hal/backends/openwrt/uci_manager.lua') + if not s:find('function Manager:submit_op', 1, true) then + fail('UCI manager should expose submit_op') + end + if not s:find('function Session:commit_op', 1, true) then + fail('UCI sessions should expose commit_op') + end +end + +-- extra compatibility ownership checks +function tests.test_uci_compat_requires_bound_manager_by_default() + local compat = require 'services.hal.backends.openwrt.uci_singleton_compat' + compat.clear_bound_manager() + local ok, err = compat.ensure_started() + if ok ~= false then fail('ensure_started should fail without a bound manager by default') end + if type(err) ~= 'string' or not err:find('not bound', 1, true) then + fail('expected not-bound error, got ' .. tostring(err)) + end +end + +return tests diff --git a/tests/unit/hal/sdk_cap_spec.lua b/tests/unit/hal/sdk_cap_spec.lua new file mode 100644 index 00000000..46c383fb --- /dev/null +++ b/tests/unit/hal/sdk_cap_spec.lua @@ -0,0 +1,291 @@ +local busmod = require 'bus' +local fibers = require 'fibers' + +local probe = require 'tests.support.bus_probe' +local runfibers = require 'tests.support.run_fibers' + +local cap_sdk = require 'services.hal.sdk.cap' + +local T = {} + +local function wait_payload(conn, topic, timeout) + return probe.wait_payload(conn, topic, { timeout = timeout or 0.2 }) +end + +local function wait_until(fn, timeout, interval) + return probe.wait_until(fn, { + timeout = timeout or 1.0, + interval = interval or 0.01, + }) +end + +function T.cap_sdk_legacy_listener_accepts_added_on_public_state_topic() + runfibers.run(function(scope) + local bus = busmod.new() + local pub = bus:connect() + local client = bus:connect() + + local listener = cap_sdk.new_cap_listener(client, 'demo', 'one') + + local got_cap, got_err + local ok, err = scope:spawn(function() + got_cap, got_err = listener:wait_for_cap({ timeout = 0.5 }) + end) + assert(ok, tostring(err)) + + pub:retain({ 'cap', 'demo', 'one', 'state' }, 'added') + + assert(wait_until(function() + return got_cap ~= nil + end, 1.0, 0.01)) + + assert(got_err == '') + assert(got_cap.class == 'demo') + assert(got_cap.id == 'one') + + listener:close() + end, { timeout = 2.0 }) +end + +function T.cap_sdk_curated_listener_accepts_available_status_payload() + runfibers.run(function(scope) + local bus = busmod.new() + local pub = bus:connect() + local client = bus:connect() + + local listener = cap_sdk.new_curated_cap_listener(client, 'demo', 'one') + + local got_cap, got_err + local ok, err = scope:spawn(function() + got_cap, got_err = listener:wait_for_cap({ timeout = 0.5 }) + end) + assert(ok, tostring(err)) + + pub:retain({ 'cap', 'demo', 'one', 'status' }, { state = 'available', version = 1 }) + + assert(wait_until(function() + return got_cap ~= nil + end, 1.0, 0.01)) + + assert(got_err == '') + assert(got_cap.class == 'demo') + assert(got_cap.id == 'one') + + listener:close() + end, { timeout = 2.0 }) +end + +function T.cap_sdk_curated_listener_accepts_available_true_payload() + runfibers.run(function(scope) + local bus = busmod.new() + local pub = bus:connect() + local client = bus:connect() + + local listener = cap_sdk.new_curated_cap_listener(client, 'demo', 'one') + + local got_cap, got_err + local ok, err = scope:spawn(function() + got_cap, got_err = listener:wait_for_cap({ timeout = 0.5 }) + end) + assert(ok, tostring(err)) + + pub:retain({ 'cap', 'demo', 'one', 'status' }, { available = true }) + + assert(wait_until(function() + return got_cap ~= nil + end, 1.0, 0.01)) + + assert(got_err == '') + assert(got_cap.class == 'demo') + assert(got_cap.id == 'one') + + listener:close() + end, { timeout = 2.0 }) +end + +function T.cap_sdk_curated_listener_ignores_removed_status_payload() + runfibers.run(function(scope) + local bus = busmod.new() + local pub = bus:connect() + local client = bus:connect() + + local listener = cap_sdk.new_curated_cap_listener(client, 'demo', 'one') + + local got_cap, got_err + local ok, err = scope:spawn(function() + got_cap, got_err = listener:wait_for_cap({ timeout = 0.1 }) + end) + assert(ok, tostring(err)) + + pub:retain({ 'cap', 'demo', 'one', 'status' }, { + state = 'removed', + available = false, + }) + + fibers.perform(require('fibers.sleep').sleep_op(0.15)) + + assert(got_cap == nil) + assert(type(got_err) == 'string') + assert(got_err:match('timeout')) + + listener:close() + end, { timeout = 1.0 }) +end + +function T.cap_sdk_raw_host_listener_and_ref_follow_raw_host_topics() + runfibers.run(function(scope) + local bus = busmod.new() + local pub = bus:connect() + local client = bus:connect() + + local listener = cap_sdk.new_raw_host_cap_listener(client, 'platform', 'artifact-store', 'main') + local ref = cap_sdk.new_raw_host_cap_ref(client, 'platform', 'artifact-store', 'main') + + local got_cap, got_err + local ok, err = scope:spawn(function() + got_cap, got_err = listener:wait_for_cap({ timeout = 0.5 }) + end) + assert(ok, tostring(err)) + + -- Subscribe BEFORE publishing. + local meta_sub = ref:get_meta_sub() + local status_sub = ref:get_status_sub() + local state_sub = ref:get_state_sub('mode') + local event_sub = ref:get_event_sub('changed') + + pub:retain( + { 'raw', 'host', 'platform', 'cap', 'artifact-store', 'main', 'status' }, + { state = 'available' } + ) + + assert(wait_until(function() + return got_cap ~= nil + end, 1.0, 0.01)) + + assert(got_err == '') + assert(got_cap.class == 'artifact-store') + assert(got_cap.id == 'main') + + pub:retain( + { 'raw', 'host', 'platform', 'cap', 'artifact-store', 'main', 'meta' }, + { provider = 'hal' } + ) + pub:retain( + { 'raw', 'host', 'platform', 'cap', 'artifact-store', 'main', 'state', 'mode' }, + 'ready' + ) + pub:publish( + { 'raw', 'host', 'platform', 'cap', 'artifact-store', 'main', 'event', 'changed' }, + { what = 'mode' } + ) + + local status_ev, err1 = status_sub:recv() + assert(status_ev, tostring(err1)) + assert(type(status_ev.payload) == 'table') + assert(status_ev.payload.state == 'available') + + local meta_ev, err2 = meta_sub:recv() + assert(meta_ev, tostring(err2)) + assert(type(meta_ev.payload) == 'table') + assert(meta_ev.payload.provider == 'hal') + + local state_ev, err3 = state_sub:recv() + assert(state_ev, tostring(err3)) + assert(state_ev.payload == 'ready') + + local event_ev, err4 = event_sub:recv() + assert(event_ev, tostring(err4)) + assert(type(event_ev.payload) == 'table') + assert(event_ev.payload.what == 'mode') + + meta_sub:unsubscribe() + status_sub:unsubscribe() + state_sub:unsubscribe() + event_sub:unsubscribe() + listener:close() + end, { timeout = 2.0 }) +end + +function T.cap_sdk_raw_member_listener_and_ref_follow_raw_member_topics() + runfibers.run(function(scope) + local bus = busmod.new() + local pub = bus:connect() + local client = bus:connect() + + local listener = cap_sdk.new_raw_member_cap_listener(client, 'mcu', 'updater', 'main') + local ref = cap_sdk.new_raw_member_cap_ref(client, 'mcu', 'updater', 'main') + + local got_cap, got_err + local ok, err = scope:spawn(function() + got_cap, got_err = listener:wait_for_cap({ timeout = 0.5 }) + end) + assert(ok, tostring(err)) + + -- Subscribe BEFORE publishing. + local meta_sub = ref:get_meta_sub() + local status_sub = ref:get_status_sub() + local state_sub = ref:get_state_sub('version') + local event_sub = ref:get_event_sub('ready') + + pub:retain( + { 'raw', 'member', 'mcu', 'cap', 'updater', 'main', 'status' }, + { state = 'available' } + ) + + assert(wait_until(function() + return got_cap ~= nil + end, 1.0, 0.01)) + + assert(got_err == '') + assert(got_cap.class == 'updater') + assert(got_cap.id == 'main') + + pub:retain( + { 'raw', 'member', 'mcu', 'cap', 'updater', 'main', 'meta' }, + { backing = 'rp2350' } + ) + pub:retain( + { 'raw', 'member', 'mcu', 'cap', 'updater', 'main', 'state', 'version' }, + '1.2.3' + ) + pub:publish( + { 'raw', 'member', 'mcu', 'cap', 'updater', 'main', 'event', 'ready' }, + { ok = true } + ) + + local status_ev, err1 = status_sub:recv() + assert(status_ev, tostring(err1)) + assert(type(status_ev.payload) == 'table') + assert(status_ev.payload.state == 'available') + + local meta_ev, err2 = meta_sub:recv() + assert(meta_ev, tostring(err2)) + assert(type(meta_ev.payload) == 'table') + assert(meta_ev.payload.backing == 'rp2350') + + local state_ev, err3 = state_sub:recv() + assert(state_ev, tostring(err3)) + assert(state_ev.payload == '1.2.3') + + local event_ev, err4 = event_sub:recv() + assert(event_ev, tostring(err4)) + assert(type(event_ev.payload) == 'table') + assert(event_ev.payload.ok == true) + + meta_sub:unsubscribe() + status_sub:unsubscribe() + state_sub:unsubscribe() + event_sub:unsubscribe() + listener:close() + end, { timeout = 2.0 }) +end + + +function T.core_capability_control_calls_default_to_compositional_timeout() + local f = assert(io.open('../src/services/hal/sdk/cap.lua', 'r')) + local src = f:read('*a'); f:close() + assert(src:find('out.timeout = false', 1, true), 'HAL cap SDK should default control calls to timeout=false') + assert(not src:find('timeout or 1%.0'), 'HAL cap SDK must not install hidden control timeouts') +end + +return T diff --git a/tests/unit/hal/service_raw_host_spec.lua b/tests/unit/hal/service_raw_host_spec.lua new file mode 100644 index 00000000..1b7137c5 --- /dev/null +++ b/tests/unit/hal/service_raw_host_spec.lua @@ -0,0 +1,527 @@ +local busmod = require 'bus' +local safe = require 'coxpcall' +local fibers = require 'fibers' +local channel = require 'fibers.channel' +local op = require 'fibers.op' + +local probe = require 'tests.support.bus_probe' +local runfibers = require 'tests.support.run_fibers' +local cap_sdk = require 'services.hal.sdk.cap' + +local hal_types = require 'services.hal.types.core' +local cap_types = require 'services.hal.types.capabilities' + +local T = {} + +local function wait_payload(conn, topic, timeout) + return probe.wait_payload(conn, topic, { timeout = timeout or 0.5 }) +end + +local function expect_no_message(conn, topic, timeout) + timeout = timeout or 0.05 + local sub = conn:subscribe(topic) + + local which = fibers.perform(op.named_choice{ + msg = sub:recv_op():wrap(function() return 'msg' end), + timeout = require('fibers.sleep').sleep_op(timeout):wrap(function() return 'timeout' end), + }) + + sub:unsubscribe() + assert(which == 'timeout', 'unexpected retained/publication on ' .. table.concat(topic, '/')) +end + +local function patch_modules(patches, fn) + local saved = {} + for name, value in pairs(patches) do + saved[name] = package.loaded[name] + package.loaded[name] = value + end + + local old_hal = package.loaded['services.hal'] + package.loaded['services.hal'] = nil + + local ok, res = safe.pcall(fn) + + package.loaded['services.hal'] = old_hal + for name, old in pairs(saved) do + package.loaded[name] = old + end + + if not ok then + error(res, 0) + end + + return res +end + +local function new_bootstrap_filesystem_manager() + local manager = {} + + function manager.start(_logger, _dev_ev_ch, _cap_emit_ch) + manager.scope = fibers.current_scope() + return '' + end + + function manager.apply_config(_self, cfg) + manager.last_cfg = cfg + return true, nil + end + + function manager.stop(_self) + -- Explicitly legacy: allowed by hal.lua. + end + + return manager +end + +local function new_rawprobe_manager() + local manager = { + scope = nil, + dev_ev_ch = nil, + control_ch = nil, + last_device = nil, + worker_started = false, + } + + local function ensure_control_worker() + if manager.worker_started then + return + end + manager.worker_started = true + + local ok, err = manager.scope:spawn(function(scope) + while true do + local which, a = fibers.perform(fibers.named_choice{ + req = manager.control_ch:get_op(), + stop = scope:not_ok_op(), + }) + + if which == 'stop' then + return + end + + local req = a + local reply, reply_err = hal_types.new.Reply(true, { + verb = req.verb, + value = req.opts and req.opts.value, + mode = 'raw-host', + }) + assert(reply, tostring(reply_err)) + + local sent, send_err = fibers.perform(req.reply_ch:put_op(reply)) + assert(sent ~= false, tostring(send_err)) + end + end) + assert(ok, tostring(err)) + end + + function manager.start(_logger, dev_ev_ch, _cap_emit_ch) + local child, err = fibers.current_scope():child() + if not child then + return tostring(err) + end + + manager.scope = child + manager.dev_ev_ch = dev_ev_ch + return '' + end + + function manager.stop_op(_self) + return op.guard(function() + if manager.scope then + manager.scope:cancel('rawprobe stop') + local st = fibers.perform(manager.scope:join_op()) + manager.scope = nil + return op.always(st == 'ok' or st == 'cancelled', nil) + end + return op.always(true, nil) + end) + end + + function manager.apply_config_op(cfg) + cfg = cfg or {} + + return op.guard(function() + local action = cfg.op or 'add' + + if action == 'add' then + manager.control_ch = channel.new() + ensure_control_worker() + + local cap, cap_err = cap_types.new.Capability( + cfg.cap_class or 'uart', + cfg.cap_id or 'main', + manager.control_ch, + cfg.offerings or { 'open' } + ) + assert(cap, tostring(cap_err)) + + local meta = { + provider = cfg.provider or 'hal.test.rawprobe', + source_id = cfg.source_id or 'uart_main', + extra = cfg.extra, + } + + local dev, dev_err = hal_types.new.Device( + cfg.device_class or 'uart', + cfg.device_id or 'main', + meta, + { cap } + ) + assert(dev, tostring(dev_err)) + manager.last_device = dev + + local ev, ev_err = hal_types.new.DeviceEvent( + 'added', + dev.class, + dev.id, + dev.meta, + dev.capabilities + ) + assert(ev, tostring(ev_err)) + + local sent, send_err = fibers.perform(manager.dev_ev_ch:put_op(ev)) + assert(sent ~= false, tostring(send_err)) + return op.always(true, nil) + end + + if action == 'remove' then + if not manager.last_device then + return op.always(true, nil) + end + + local dev = manager.last_device + local ev, ev_err = hal_types.new.DeviceEvent( + 'removed', + dev.class, + dev.id, + dev.meta, + dev.capabilities + ) + assert(ev, tostring(ev_err)) + + local sent, send_err = fibers.perform(manager.dev_ev_ch:put_op(ev)) + assert(sent ~= false, tostring(send_err)) + return op.always(true, nil) + end + + return op.always(false, 'unsupported op: ' .. tostring(action)) + end) + end + + return manager +end + +local function with_real_hal(scope, patches, body, hal_opts) + return patch_modules(patches, function() + local hal_service = require 'services.hal' + local bus = busmod.new() + + local opts = { + name = 'hal', + env = 'test', + heartbeat_s = 60.0, + } + if hal_opts then + for k, v in pairs(hal_opts) do + opts[k] = v + end + end + + local ok_spawn, spawn_err = scope:spawn(function() + hal_service.start(bus:connect(), opts) + end) + assert(ok_spawn, tostring(spawn_err)) + + return body(bus) + end) +end + +local function publish_hal_config(conn, cfg) + conn:retain({ 'cfg', 'hal' }, { + data = cfg, + }) +end + +local function add_rawprobe_config(overrides) + local cfg = { + op = 'add', + source_id = 'uart_main', + device_class = 'uart', + device_id = 'main', + cap_class = 'uart', + cap_id = 'main', + offerings = { 'open' }, + provider = 'hal.test.rawprobe', + } + + if overrides then + for k, v in pairs(overrides) do + cfg[k] = v + end + end + + return { + schema = 'devicecode.config/hal/1', + rawprobe = cfg, + } +end + +function T.hal_publishes_raw_host_source_meta_and_status_on_add() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local rawprobe = new_rawprobe_manager() + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.rawprobe'] = rawprobe, + }, function(bus) + local reader = bus:connect() + local admin = bus:connect() + + publish_hal_config(admin, add_rawprobe_config()) + + local meta = wait_payload(reader, { 'raw', 'host', 'uart_main', 'meta' }, 0.5) + assert(type(meta) == 'table') + assert(meta.provider == 'hal.test.rawprobe') + assert(meta.source_id == 'uart_main') + assert(meta.source == 'uart_main') + assert(meta.class == 'uart') + assert(meta.id == 'main') + + local status = wait_payload(reader, { 'raw', 'host', 'uart_main', 'status' }, 0.5) + assert(type(status) == 'table') + assert(status.state == 'available') + assert(status.available == true) + assert(status.source == 'uart_main') + assert(status.class == 'uart') + assert(status.id == 'main') + end) + end) +end + +function T.hal_publishes_raw_host_cap_meta_status_and_rpc_on_add() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local rawprobe = new_rawprobe_manager() + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.rawprobe'] = rawprobe, + }, function(bus) + local reader = bus:connect() + local admin = bus:connect() + + publish_hal_config(admin, add_rawprobe_config()) + + local meta = wait_payload(reader, { + 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'meta' + }, 0.5) + assert(type(meta) == 'table') + assert(type(meta.offerings) == 'table') + assert(meta.offerings.open == true) + assert(meta.source_kind == 'host') + assert(meta.source == 'uart_main') + + local status = wait_payload(reader, { + 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'status' + }, 0.5) + assert(type(status) == 'table') + assert(status.state == 'available') + assert(status.available == true) + assert(status.source_kind == 'host') + assert(status.source == 'uart_main') + + local listener = cap_sdk.new_raw_host_cap_listener(bus:connect(), 'uart_main', 'uart', 'main') + local ref, err = fibers.perform(listener:wait_for_cap_op()) + assert(ref, tostring(err)) + + local reply, call_err = fibers.perform(ref:call_control_op('open', { value = 115200 })) + assert(reply, tostring(call_err)) + assert(reply.ok == true) + assert(type(reply.reason) == 'table') + assert(reply.reason.verb == 'open') + assert(reply.reason.value == 115200) + assert(reply.reason.mode == 'raw-host') + + listener:close() + end) + end) +end + +function T.raw_host_rpc_is_scoped_to_declared_source() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local rawprobe = new_rawprobe_manager() + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.rawprobe'] = rawprobe, + }, function(bus) + local admin = bus:connect() + + publish_hal_config(admin, add_rawprobe_config()) + + local good_listener = cap_sdk.new_raw_host_cap_listener(bus:connect(), 'uart_main', 'uart', 'main') + local good_ref, wait_err = fibers.perform(good_listener:wait_for_cap_op()) + assert(good_ref, tostring(wait_err)) + + local good_reply, good_err = fibers.perform(good_ref:call_control_op('open', { value = 115200 })) + assert(good_reply, tostring(good_err)) + assert(good_reply.ok == true) + + local wrong_ref = cap_sdk.new_raw_host_cap_ref(bus:connect(), 'other_source', 'uart', 'main') + local bad_reply, bad_err = fibers.perform(wrong_ref:call_control_op('open', { value = 9600 }, { + timeout = 0.05, + })) + + assert(bad_reply == nil) + assert(type(bad_err) == 'string') + assert(bad_err == 'no_route' or bad_err:match('no_route')) + + good_listener:close() + end) + end, { timeout = 2.0 }) +end + +function T.hal_marks_raw_host_source_and_capability_removed_and_unretains_meta() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local rawprobe = new_rawprobe_manager() + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.rawprobe'] = rawprobe, + }, function(bus) + local reader = bus:connect() + local admin = bus:connect() + + publish_hal_config(admin, add_rawprobe_config()) + + assert(type(wait_payload(reader, { 'raw', 'host', 'uart_main', 'status' }, 0.5)) == 'table') + assert(type(wait_payload(reader, { + 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'status' + }, 0.5)) == 'table') + + publish_hal_config(admin, { + schema = 'devicecode.config/hal/1', + rawprobe = { op = 'remove' }, + }) + + local source_status + local ok = probe.wait_until(function() + source_status = wait_payload(reader, { 'raw', 'host', 'uart_main', 'status' }, 0.05) + return type(source_status) == 'table' and source_status.state == 'removed' + end, { + timeout = 0.5, + interval = 0.01, + }) + assert(ok, 'timed out waiting for raw host source status to become removed') + assert(source_status.available == false) + + local cap_status + ok = probe.wait_until(function() + cap_status = wait_payload(reader, { + 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'status' + }, 0.05) + return type(cap_status) == 'table' and cap_status.state == 'removed' + end, { + timeout = 0.5, + interval = 0.01, + }) + assert(ok, 'timed out waiting for raw host capability status to become removed') + assert(cap_status.available == false) + assert(cap_status.source_kind == 'host') + assert(cap_status.source == 'uart_main') + + expect_no_message(reader, { 'raw', 'host', 'uart_main', 'meta' }, 0.05) + expect_no_message(reader, { + 'raw', 'host', 'uart_main', 'cap', 'uart', 'main', 'meta' + }, 0.05) + expect_no_message(reader, { 'cap', 'uart', 'main', 'meta' }, 0.05) + end) + end) +end + +function T.capability_rpc_is_unbound_after_removal() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local rawprobe = new_rawprobe_manager() + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.rawprobe'] = rawprobe, + }, function(bus) + local admin = bus:connect() + local reader = bus:connect() + + publish_hal_config(admin, add_rawprobe_config()) + + local listener = cap_sdk.new_curated_cap_listener(bus:connect(), 'uart', 'main') + local ref, err = fibers.perform(listener:wait_for_cap_op()) + assert(ref, tostring(err)) + + local reply, call_err = fibers.perform(ref:call_control_op('open', { value = 1 })) + assert(reply, tostring(call_err)) + assert(reply.ok == true) + + publish_hal_config(admin, { + schema = 'devicecode.config/hal/1', + rawprobe = { op = 'remove' }, + }) + + local removed + assert(probe.wait_until(function() + removed = wait_payload(reader, { 'cap', 'uart', 'main', 'status' }, 0.05) + return type(removed) == 'table' and removed.state == 'removed' + end, { + timeout = 0.5, + interval = 0.01, + })) + + local reply2, err2 = fibers.perform(ref:call_control_op('open', { value = 2 }, { + timeout = 0.05, + })) + + assert(reply2 == nil) + assert(type(err2) == 'string') + assert(err2 == 'no_route' or err2:match('no_route')) + + listener:close() + end) + end, { timeout = 2.0 }) +end + +function T.hal_keeps_legacy_public_capability_topics_for_compatibility() + runfibers.run(function(scope) + local fs_manager = new_bootstrap_filesystem_manager() + local rawprobe = new_rawprobe_manager() + + with_real_hal(scope, { + ['services.hal.managers.filesystem'] = fs_manager, + ['services.hal.managers.rawprobe'] = rawprobe, + }, function(bus) + local reader = bus:connect() + local admin = bus:connect() + + publish_hal_config(admin, add_rawprobe_config()) + + local legacy_state = wait_payload(reader, { 'cap', 'uart', 'main', 'state' }, 0.5) + assert(legacy_state == 'added') + + local curated_status = wait_payload(reader, { 'cap', 'uart', 'main', 'status' }, 0.5) + assert(type(curated_status) == 'table') + assert(curated_status.state == 'available') + assert(curated_status.available == true) + assert(curated_status.source_kind == 'host') + assert(curated_status.source == 'uart_main') + + local legacy_meta = wait_payload(reader, { 'cap', 'uart', 'main', 'meta' }, 0.5) + assert(type(legacy_meta) == 'table') + assert(type(legacy_meta.offerings) == 'table') + assert(legacy_meta.offerings.open == true) + end) + end) +end + +return T diff --git a/tests/unit/hal/signature_verify_manager_spec.lua b/tests/unit/hal/signature_verify_manager_spec.lua new file mode 100644 index 00000000..0da1de5b --- /dev/null +++ b/tests/unit/hal/signature_verify_manager_spec.lua @@ -0,0 +1,218 @@ +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local channel = require 'fibers.channel' + +local runfibers = require 'tests.support.run_fibers' + +local T = {} + +local function fresh_manager() + package.loaded['services.hal.managers.signature_verify'] = nil + return require 'services.hal.managers.signature_verify' +end + +local function recv_or_fail(ch) + local v, err = fibers.perform(ch:get_op()) + assert(v, tostring(err)) + return v +end + +local function fake_backend(name, ok, value_or_err) + return { + backend_name = name or 'fake-manager-backend', + + verify_ed25519_op = function() + return require('fibers').always(ok, value_or_err) + end, + } +end + +function T.apply_config_fails_when_not_started() + local M = fresh_manager() + + runfibers.run(function() + local ok, err = fibers.perform(M.apply_config_op({ + providers = { + { id = 'main', backend = fake_backend('fake-a', true, nil) }, + }, + })) + assert(ok == false) + assert(tostring(err):match('not started')) + end) +end + +function T.start_apply_config_and_stop_round_trip() + local M = fresh_manager() + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok_cfg, err_cfg = fibers.perform(M.apply_config_op({ + providers = { + { + id = 'main', + backend = fake_backend('fake-sig-manager', true, { verified = true }), + max_in_flight = 2, + }, + }, + })) + assert(ok_cfg == true, tostring(err_cfg)) + + local ev = recv_or_fail(dev_ev_ch) + assert(ev.event_type == 'added') + assert(ev.class == 'signature_verify') + assert(ev.id == 'main') + assert(#ev.capabilities == 1) + assert(ev.capabilities[1].class == 'signature_verify') + assert(ev.capabilities[1].offerings.verify_ed25519 == true) + + local e1 = recv_or_fail(cap_emit_ch) + local e2 = recv_or_fail(cap_emit_ch) + + local by_mode = { + [e1.mode] = e1, + [e2.mode] = e2, + } + + assert(by_mode.meta ~= nil) + assert(by_mode.state ~= nil) + assert(by_mode.meta.class == 'signature_verify') + assert(by_mode.meta.id == 'main') + assert(by_mode.meta.key == 'info') + assert(by_mode.meta.data.backend == 'fake-sig-manager') + assert(by_mode.meta.data.max_in_flight == 2) + assert(by_mode.state.key == 'status') + assert(by_mode.state.data.state == 'available') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.reapply_same_config_is_idempotent() + local M = fresh_manager() + local backend = fake_backend('same-backend', true, nil) + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local cfg = { + providers = { + { + id = 'main', + backend = backend, + max_in_flight = 1, + }, + }, + } + + local ok1, err1 = fibers.perform(M.apply_config_op(cfg)) + assert(ok1 == true, tostring(err1)) + local added = recv_or_fail(dev_ev_ch) + assert(added.event_type == 'added') + + local ok2, err2 = fibers.perform(M.apply_config_op(cfg)) + assert(ok2 == true, tostring(err2)) + + local which = fibers.perform(require('fibers').named_choice{ + msg = dev_ev_ch:get_op():wrap(function(v) return 'msg', v end), + timeout = sleep.sleep_op(0.05):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout', 'reapplying same config should not emit new device events') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.config_change_removes_then_adds_provider() + local M = fresh_manager() + local backend = fake_backend('change-backend', true, nil) + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok1, err1 = fibers.perform(M.apply_config_op({ + providers = { + { + id = 'main', + backend = backend, + max_in_flight = 1, + }, + }, + })) + assert(ok1 == true, tostring(err1)) + local first = recv_or_fail(dev_ev_ch) + assert(first.event_type == 'added') + + local ok2, err2 = fibers.perform(M.apply_config_op({ + providers = { + { + id = 'main', + backend = backend, + max_in_flight = 2, + }, + }, + })) + assert(ok2 == true, tostring(err2)) + + local ev_a = recv_or_fail(dev_ev_ch) + local ev_b = recv_or_fail(dev_ev_ch) + assert(ev_a.event_type == 'removed') + assert(ev_b.event_type == 'added') + assert(ev_b.id == 'main') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.invalid_config_is_rejected() + local M = fresh_manager() + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok_cfg, err_cfg = fibers.perform(M.apply_config_op({ + providers = 'nope', + })) + assert(ok_cfg == false) + assert(tostring(err_cfg):match('providers must be a table')) + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.shutdown_op_before_start_is_ok_and_fault_op_is_inert() + local M = fresh_manager() + + runfibers.run(function() + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + + local which = fibers.perform(require('fibers').named_choice{ + fault = M.fault_op():wrap(function(...) return 'fault', ... end), + timeout = sleep.sleep_op(0.05):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout') + end) +end + +return T diff --git a/tests/unit/hal/signature_verify_openssl_spec.lua b/tests/unit/hal/signature_verify_openssl_spec.lua new file mode 100644 index 00000000..b81a892a --- /dev/null +++ b/tests/unit/hal/signature_verify_openssl_spec.lua @@ -0,0 +1,240 @@ +local fibers = require 'fibers' +local op = require 'fibers.op' +local runfibers = require 'tests.support.run_fibers' + +local backend_mod = require 'services.hal.drivers.signature_verify_openssl' + +local T = {} + +local function make_stream(path, cfg) + cfg = cfg or {} + cfg.writes = cfg.writes or {} + cfg.closed = 0 + + local stream = {} + + function stream:filename() + return path + end + + function stream:write_op(data) + if cfg.write_err then + return op.always(nil, cfg.write_err) + end + cfg.writes[#cfg.writes + 1] = data + return op.always(#data, nil) + end + + function stream:flush_op() + if cfg.flush_err then + return op.always(nil, cfg.flush_err) + end + return op.always(true, nil) + end + + function stream:close_op() + cfg.closed = cfg.closed + 1 + return op.always(true, nil) + end + + function stream:terminate(_reason) + cfg.closed = cfg.closed + 1 + return true, nil + end + + return stream, cfg +end + +local function make_fake_file(tmp_specs) + local i = 0 + + return { + tmpfile = function(_perms, _tmpdir) + i = i + 1 + local spec = tmp_specs[i] + if not spec then + return nil, 'unexpected tmpfile request #' .. tostring(i) + end + if spec.err then + return nil, spec.err + end + + local stream, state = make_stream('/tmp/sv-' .. tostring(i), spec) + spec._state = state + return stream + end, + } +end + +local function make_fake_exec(result, capture) + capture = capture or {} + + return { + command = function(...) + local args = { ... } + if type(args[1]) == 'table' and args[2] == nil then + capture.spec = args[1] + capture.argv = args[1] + else + capture.argv = args + end + + return { + combined_output_op = function() + return op.always( + result.out, + result.st, + result.code, + result.sig, + result.err + ) + end, + } + end, + } +end + +function T.verify_ed25519_rejects_invalid_inputs_before_touching_file_or_exec() + runfibers.run(function() + local calls = { tmpfile = 0, command = 0 } + + local driver = backend_mod.new({ + file = { + tmpfile = function() + calls.tmpfile = calls.tmpfile + 1 + return nil, 'should not be called' + end, + }, + exec = { + command = function() + calls.command = calls.command + 1 + return { + combined_output_op = function() + return op.always('', 'exited', 0, nil, nil) + end, + } + end, + }, + tmpdir = '/tmp', + }) + + local ok, err = fibers.perform(driver:verify_ed25519_op('', 'msg', 'sig')) + assert(ok == false) + assert(err == 'public_key_required') + assert(calls.tmpfile == 0) + assert(calls.command == 0) + end) +end + +function T.verify_ed25519_success_writes_all_inputs_and_invokes_openssl() + runfibers.run(function() + local specs = { + {}, + {}, + {}, + } + local capture = {} + + local driver = backend_mod.new({ + file = make_fake_file(specs), + exec = make_fake_exec({ + out = 'Signature Verified Successfully\n', + st = 'exited', + code = 0, + sig = nil, + err = nil, + }, capture), + tmpdir = '/tmp', + }) + + local ok, err = fibers.perform(driver:verify_ed25519_op( + 'PUBKEY', + 'MESSAGE', + 'SIGNATURE' + )) + + assert(ok == true) + assert(err == nil) + + assert(specs[1]._state.writes[1] == 'PUBKEY') + assert(specs[2]._state.writes[1] == 'MESSAGE') + assert(specs[3]._state.writes[1] == 'SIGNATURE') + + assert(specs[1]._state.closed == 1) + assert(specs[2]._state.closed == 1) + assert(specs[3]._state.closed == 1) + + assert(capture.argv[1] == 'openssl') + assert(capture.argv[2] == 'pkeyutl') + assert(capture.argv[3] == '-verify') + assert(capture.argv[4] == '-pubin') + assert(capture.argv[5] == '-inkey') + assert(capture.argv[6] == '/tmp/sv-1') + assert(capture.argv[7] == '-sigfile') + assert(capture.argv[8] == '/tmp/sv-3') + assert(capture.argv[9] == '-in') + assert(capture.argv[10] == '/tmp/sv-2') + assert(capture.argv[11] == '-rawin') + assert(capture.spec.stdin == 'null') + assert(capture.spec.stdout == 'pipe') + assert(capture.spec.stderr == 'stdout') + end) +end + +function T.verify_ed25519_classifies_bad_signature() + runfibers.run(function() + local driver = backend_mod.new({ + file = make_fake_file({ {}, {}, {} }), + exec = make_fake_exec({ + out = 'Signature Verification Failure\n', + st = 'exited', + code = 1, + sig = nil, + err = nil, + }), + tmpdir = '/tmp', + }) + + local ok, err = fibers.perform(driver:verify_ed25519_op( + 'PUBKEY', + 'MESSAGE', + 'SIGNATURE' + )) + + assert(ok == false) + assert(err == 'signature_verify_failed') + end) +end + +function T.verify_ed25519_surfaces_write_failure_with_labelled_error() + runfibers.run(function() + local specs = { + {}, + { write_err = 'diskfull' }, + {}, + } + + local driver = backend_mod.new({ + file = make_fake_file(specs), + exec = make_fake_exec({ + out = '', + st = 'exited', + code = 0, + sig = nil, + err = nil, + }), + tmpdir = '/tmp', + }) + + local ok, err = fibers.perform(driver:verify_ed25519_op( + 'PUBKEY', + 'MESSAGE', + 'SIGNATURE' + )) + + assert(ok == false) + assert(err == 'message_write_failed:diskfull') + end) +end + +return T diff --git a/tests/unit/hal/signature_verify_provider_spec.lua b/tests/unit/hal/signature_verify_provider_spec.lua new file mode 100644 index 00000000..8981abee --- /dev/null +++ b/tests/unit/hal/signature_verify_provider_spec.lua @@ -0,0 +1,244 @@ +local fibers = require 'fibers' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local channel = require 'fibers.channel' + +local runfibers = require 'tests.support.run_fibers' +local core_types = require 'services.hal.types.core' +local cap_args = require 'services.hal.types.capability_args' + +local provider_mod = require 'services.hal.drivers.signature_verify_provider' + +local T = {} + +local function recv_or_fail(ch) + local v, err = fibers.perform(ch:get_op()) + assert(v, tostring(err)) + return v +end + +local function fake_backend(result_ok, result_value, extra) + extra = extra or {} + + return { + backend_name = extra.backend_name or 'fake-backend', + + verify_ed25519_op = function(_self, pubkey_pem, message, signature) + if extra.observe then + extra.observe[#extra.observe + 1] = { + pubkey_pem = pubkey_pem, + message = message, + signature = signature, + } + end + return op.always(result_ok, result_value) + end, + } +end + +local function request(driver, verb, opts) + local reply_ch = channel.new(1) + local req, err = core_types.new.ControlRequest(verb, opts or {}, reply_ch) + assert(req, tostring(err)) + + local sent, send_err = fibers.perform(driver.control_ch:put_op(req)) + assert(sent ~= false, tostring(send_err)) + + local reply, recv_err = fibers.perform(reply_ch:get_op()) + assert(reply, tostring(recv_err)) + return reply +end + +function T.capabilities_op_returns_signature_verify_capability() + runfibers.run(function() + local emit_ch = channel.new(8) + local driver = provider_mod.new('main', { + backend = fake_backend(true, nil), + }, nil) + + local ok, caps_or_err = fibers.perform(driver:capabilities_op(emit_ch)) + assert(ok == true, tostring(caps_or_err)) + assert(type(caps_or_err) == 'table') + assert(#caps_or_err == 1) + + local cap = caps_or_err[1] + assert(cap.class == 'signature_verify') + assert(cap.id == 'main') + assert(cap.offerings.verify_ed25519 == true) + end) +end + +function T.start_op_emits_initial_meta_and_available_state() + runfibers.run(function(scope) + local emit_ch = channel.new(8) + local driver = provider_mod.new('main', { + backend = fake_backend(true, nil, { backend_name = 'fake-sig' }), + max_in_flight = 2, + }, nil) + + local ok_caps, caps_err = fibers.perform(driver:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_err)) + + local ok_start, start_err = fibers.perform(driver:start_op(scope)) + assert(ok_start == true, tostring(start_err)) + + local e1 = recv_or_fail(emit_ch) + local e2 = recv_or_fail(emit_ch) + + local by_mode = { + [e1.mode] = e1, + [e2.mode] = e2, + } + + assert(by_mode.meta ~= nil) + assert(by_mode.state ~= nil) + + assert(by_mode.meta.class == 'signature_verify') + assert(by_mode.meta.id == 'main') + assert(by_mode.meta.key == 'info') + assert(by_mode.meta.data.backend == 'fake-sig') + assert(by_mode.meta.data.max_in_flight == 2) + + assert(by_mode.state.key == 'status') + assert(by_mode.state.data.state == 'available') + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + assert(ok_stop == true, tostring(stop_err)) + end) +end + +function T.verify_ed25519_request_round_trips_success() + runfibers.run(function(scope) + local seen = {} + local emit_ch = channel.new(8) + local driver = provider_mod.new('main', { + backend = fake_backend(true, { verified = true }, { observe = seen }), + }, nil) + + local ok_caps, caps_err = fibers.perform(driver:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_err)) + + local ok_start, start_err = fibers.perform(driver:start_op(scope)) + assert(ok_start == true, tostring(start_err)) + + local opts, opts_err = cap_args.new.SignatureVerifyEd25519Opts( + 'PUBKEY', + 'MESSAGE', + 'SIGNATURE' + ) + assert(opts, tostring(opts_err)) + + local reply = request(driver, 'verify_ed25519', opts) + assert(reply.ok == true) + assert(type(reply.reason) == 'table') + assert(reply.reason.verified == true) + + assert(#seen == 1) + assert(seen[1].pubkey_pem == 'PUBKEY') + assert(seen[1].message == 'MESSAGE') + assert(seen[1].signature == 'SIGNATURE') + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + assert(ok_stop == true, tostring(stop_err)) + end) +end + +function T.busy_is_returned_when_max_in_flight_reached() + runfibers.run(function(scope) + local gate = channel.new(1) + + local backend = { + backend_name = 'gated', + + verify_ed25519_op = function() + return gate:get_op():wrap(function(token) + return true, token + end) + end, + } + + local emit_ch = channel.new(8) + local driver = provider_mod.new('main', { + backend = backend, + max_in_flight = 1, + }, nil) + + local ok_caps, caps_err = fibers.perform(driver:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_err)) + + local ok_start, start_err = fibers.perform(driver:start_op(scope)) + assert(ok_start == true, tostring(start_err)) + + local opts, opts_err = cap_args.new.SignatureVerifyEd25519Opts( + 'PUBKEY', + 'MESSAGE', + 'SIGNATURE' + ) + assert(opts, tostring(opts_err)) + + local first_reply_ch = channel.new(1) + local req1, req1_err = core_types.new.ControlRequest('verify_ed25519', opts, first_reply_ch) + assert(req1, tostring(req1_err)) + + local sent1, send1_err = fibers.perform(driver.control_ch:put_op(req1)) + assert(sent1 ~= false, tostring(send1_err)) + + fibers.perform(sleep.sleep_op(0.01)) + + local reply2 = request(driver, 'verify_ed25519', opts) + assert(reply2.ok == false) + assert(reply2.reason == 'busy') + + local gate_sent, gate_err = fibers.perform(gate:put_op('done')) + assert(gate_sent ~= false, tostring(gate_err)) + + local reply1, recv1_err = fibers.perform(first_reply_ch:get_op()) + assert(reply1, tostring(recv1_err)) + assert(reply1.ok == true) + assert(reply1.reason == 'done') + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + assert(ok_stop == true, tostring(stop_err)) + end) +end + +function T.unsupported_verb_returns_negative_reply() + runfibers.run(function(scope) + local emit_ch = channel.new(8) + local driver = provider_mod.new('main', { + backend = fake_backend(true, nil), + }, nil) + + local ok_caps, caps_err = fibers.perform(driver:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_err)) + + local ok_start, start_err = fibers.perform(driver:start_op(scope)) + assert(ok_start == true, tostring(start_err)) + + local reply = request(driver, 'nope', {}) + assert(reply.ok == false) + assert(tostring(reply.reason):match('unsupported verb')) + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + assert(ok_stop == true, tostring(stop_err)) + end) +end + +function T.shutdown_op_before_start_is_ok_and_fault_op_is_inert() + runfibers.run(function() + local driver = provider_mod.new('main', { + backend = fake_backend(true, nil), + }, nil) + + local ok_stop, stop_err = fibers.perform(driver:shutdown_op()) + assert(ok_stop == true, tostring(stop_err)) + + local which = fibers.perform(fibers.named_choice{ + fault = driver:fault_op():wrap(function(...) return 'fault', ... end), + timeout = sleep.sleep_op(0.05):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout') + end) +end + +return T diff --git a/tests/unit/hal/uart_driver_spec.lua b/tests/unit/hal/uart_driver_spec.lua new file mode 100644 index 00000000..265b0fac --- /dev/null +++ b/tests/unit/hal/uart_driver_spec.lua @@ -0,0 +1,459 @@ +local fibers = require 'fibers' +local channel = require 'fibers.channel' +local sleep = require 'fibers.sleep' + +local runfibers = require 'tests.support.run_fibers' +local pty = require 'tests.support.pty' + +local core_types = require 'services.hal.types.core' +local cap_args = require 'services.hal.types.capability_args' + +local T = {} + +local function fresh_driver() + package.loaded['services.hal.drivers.uart'] = nil + return require('services.hal.drivers.uart') +end + +local function recv_or_fail(ch) + local v, err = fibers.perform(ch:get_op()) + assert(v ~= nil, tostring(err)) + return v +end + +local function wait_emit(cap_emit_ch, class, id, mode, key, timeout_s) + local deadline = fibers.now() + (timeout_s or 1.0) + + while fibers.now() < deadline do + local remain = deadline - fibers.now() + local which, a = fibers.perform(fibers.named_choice{ + item = cap_emit_ch:get_op(), + timeout = sleep.sleep_op(remain):wrap(function() + return 'timeout' + end), + }) + + if which == 'timeout' then + break + end + + local ev = a + if ev and ev.class == class and ev.id == id and ev.mode == mode and ev.key == key then + return ev + end + end + + error(('timed out waiting for %s/%s %s %s'):format( + tostring(class), tostring(id), tostring(mode), tostring(key) + ), 0) +end + +local function call_control_op(control_ch, verb, opts) + return fibers.run_scope_op(function() + local reply_ch = channel.new(1) + + local req, err = core_types.new.ControlRequest(verb, opts or {}, reply_ch) + assert(req, tostring(err)) + + -- channel:put_op() returns no values on success; success is "no error raised" + fibers.perform(control_ch:put_op(req)) + + local reply, recv_err = fibers.perform(reply_ch:get_op()) + if not reply then + return false, tostring(recv_err or 'missing reply') + end + + assert(type(reply) == 'table', 'control reply must be a table') + assert(type(reply.ok) == 'boolean', 'control reply ok must be boolean') + + if reply.ok then + return true, reply.reason + end + + return false, reply.reason + end):wrap(function(st, rep, ok, value_or_err) + if st ~= 'ok' then + return false, tostring(value_or_err or rep) + end + return ok, value_or_err + end) +end + +local function drain_initial_emits(cap_emit_ch, id) + local e1 = recv_or_fail(cap_emit_ch) + local e2 = recv_or_fail(cap_emit_ch) + + local by_mode = { + [e1.mode] = e1, + [e2.mode] = e2, + } + + assert(by_mode.meta ~= nil) + assert(by_mode.state ~= nil) + assert(by_mode.meta.class == 'uart') + assert(by_mode.meta.id == id) + assert(by_mode.meta.key == 'details') + assert(by_mode.state.class == 'uart') + assert(by_mode.state.id == id) + assert(by_mode.state.key == 'status') + + return by_mode +end + +function T.capabilities_op_returns_one_uart_capability() + local uart = fresh_driver() + + runfibers.run(function() + local emit_ch = channel.new(8) + local d = uart.new('mcu', '/dev/null', 115200, '8N1', nil) + + local ok, caps_or_err = fibers.perform(d:capabilities_op(emit_ch)) + assert(ok == true, tostring(caps_or_err)) + assert(type(caps_or_err) == 'table' and #caps_or_err == 1) + + local cap = caps_or_err[1] + assert(cap.class == 'uart') + assert(cap.id == 'mcu') + assert(cap.offerings.open == true) + assert(cap.offerings.status == true) + assert(cap.offerings.close == nil) + assert(cap.offerings.write == nil) + end) +end + +function T.start_op_emits_initial_meta_and_status() + local uart = fresh_driver() + + runfibers.run(function(scope) + local port = pty.open(scope) + local emit_ch = channel.new(8) + + local d = uart.new('mcu', port.slave_name, 115200, '8N1', nil) + local ok_caps, caps_or_err = fibers.perform(d:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_or_err)) + + local ok_start, err_start = fibers.perform(d:start_op(scope)) + assert(ok_start == true, tostring(err_start)) + + local initial = drain_initial_emits(emit_ch, 'mcu') + assert(initial.meta.data.path == port.slave_name) + assert(initial.meta.data.kind == 'uart') + assert(initial.state.data.open == false) + assert(initial.state.data.available == true) + + local ok_stop, err_stop = fibers.perform(d:shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.status_reports_open_false_before_any_session_exists() + local uart = fresh_driver() + + runfibers.run(function(scope) + local port = pty.open(scope) + local emit_ch = channel.new(8) + + local d = uart.new('mcu', port.slave_name, 115200, '8N1', nil) + local ok_caps, caps_or_err = fibers.perform(d:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_or_err)) + + local ok_start, err_start = fibers.perform(d:start_op(scope)) + assert(ok_start == true, tostring(err_start)) + + drain_initial_emits(emit_ch, 'mcu') + + local ok, status = fibers.perform(call_control_op(d.control_ch, 'status', {})) + assert(ok == true, tostring(status)) + assert(type(status) == 'table') + assert(status.open == false) + assert(status.available == true) + assert(status.path == port.slave_name) + + local ok_stop, err_stop = fibers.perform(d:shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.open_returns_uart_open_reply_with_wrapped_session() + local uart = fresh_driver() + + runfibers.run(function(scope) + local port = pty.open(scope) + local emit_ch = channel.new(16) + + local d = uart.new('mcu', port.slave_name, 115200, '8N1', nil) + + local ok_caps, caps_or_err = fibers.perform(d:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_or_err)) + + local ok_start, err_start = fibers.perform(d:start_op(scope)) + assert(ok_start == true, tostring(err_start)) + + drain_initial_emits(emit_ch, 'mcu') + + local opts, opts_err = cap_args.new.UARTOpenOpts() + assert(opts, tostring(opts_err)) + + local ok, reply = fibers.perform(call_control_op(d.control_ch, 'open', opts)) + assert(ok == true, tostring(reply)) + assert(type(reply) == 'table') + assert(type(reply.lease_id) == 'string' and reply.lease_id ~= '') + assert(reply.path == port.slave_name) + assert(type(reply.session) == 'table') + assert(type(reply.session.read_some_op) == 'function') + assert(type(reply.session.read_exactly_op) == 'function') + assert(type(reply.session.read_line_op) == 'function') + assert(type(reply.session.read_all_op) == 'function') + assert(type(reply.session.write_op) == 'function') + assert(type(reply.session.flush_op) == 'function') + assert(type(reply.session.close_op) == 'function') + + local st = wait_emit(emit_ch, 'uart', 'mcu', 'state', 'status', 1.0) + assert(st.data.open == true) + assert(st.data.lease_id == reply.lease_id) + + local ev = wait_emit(emit_ch, 'uart', 'mcu', 'event', 'opened', 1.0) + assert(ev.data.lease_id == reply.lease_id) + + local ok_close, err_close = fibers.perform(reply.session:close_op()) + assert(ok_close == true, tostring(err_close)) + + local st2 = wait_emit(emit_ch, 'uart', 'mcu', 'state', 'status', 1.0) + assert(st2.data.open == false) + assert(st2.data.lease_id == nil) + + local ev2 = wait_emit(emit_ch, 'uart', 'mcu', 'event', 'closed', 1.0) + assert(ev2.data.lease_id == reply.lease_id) + + local ok_stop, err_stop = fibers.perform(d:shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.second_open_while_active_returns_busy() + local uart = fresh_driver() + + runfibers.run(function(scope) + local port = pty.open(scope) + local emit_ch = channel.new(16) + + local d = uart.new('mcu', port.slave_name, 115200, '8N1', nil) + + local ok_caps, caps_or_err = fibers.perform(d:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_or_err)) + + local ok_start, err_start = fibers.perform(d:start_op(scope)) + assert(ok_start == true, tostring(err_start)) + + drain_initial_emits(emit_ch, 'mcu') + + local opts = assert(cap_args.new.UARTOpenOpts()) + local ok1, reply1 = fibers.perform(call_control_op(d.control_ch, 'open', opts)) + assert(ok1 == true, tostring(reply1)) + wait_emit(emit_ch, 'uart', 'mcu', 'state', 'status', 1.0) + wait_emit(emit_ch, 'uart', 'mcu', 'event', 'opened', 1.0) + + local ok2, err2 = fibers.perform(call_control_op(d.control_ch, 'open', opts)) + assert(ok2 == false) + assert(type(err2) == 'string') + assert(err2:match('busy')) + + local ok_close, err_close = fibers.perform(reply1.session:close_op()) + assert(ok_close == true, tostring(err_close)) + wait_emit(emit_ch, 'uart', 'mcu', 'state', 'status', 1.0) + wait_emit(emit_ch, 'uart', 'mcu', 'event', 'closed', 1.0) + + local ok_stop, err_stop = fibers.perform(d:shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + + +function T.session_terminate_releases_active_driver_lease() + local uart = fresh_driver() + + runfibers.run(function(scope) + local port = pty.open(scope) + local emit_ch = channel.new(16) + + local d = uart.new('mcu', port.slave_name, 115200, '8N1', nil) + + local ok_caps, caps_or_err = fibers.perform(d:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_or_err)) + + local ok_start, err_start = fibers.perform(d:start_op(scope)) + assert(ok_start == true, tostring(err_start)) + + drain_initial_emits(emit_ch, 'mcu') + + local opts = assert(cap_args.new.UARTOpenOpts()) + local ok, reply = fibers.perform(call_control_op(d.control_ch, 'open', opts)) + assert(ok == true, tostring(reply)) + wait_emit(emit_ch, 'uart', 'mcu', 'state', 'status', 1.0) + wait_emit(emit_ch, 'uart', 'mcu', 'event', 'opened', 1.0) + + local ok_term, term_err = reply.session:terminate('test terminate') + assert(ok_term == true, tostring(term_err)) + assert(d.active_session == nil) + assert(d.active_lease_id == nil) + + local ok2, reply2 = fibers.perform(call_control_op(d.control_ch, 'open', opts)) + assert(ok2 == true, tostring(reply2)) + wait_emit(emit_ch, 'uart', 'mcu', 'state', 'status', 1.0) + wait_emit(emit_ch, 'uart', 'mcu', 'event', 'opened', 1.0) + + local ok_stop, err_stop = fibers.perform(d:shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.shutdown_op_closes_active_session_best_effort() + local uart = fresh_driver() + + runfibers.run(function(scope) + local port = pty.open(scope) + local emit_ch = channel.new(16) + + local d = uart.new('mcu', port.slave_name, 115200, '8N1', nil) + + local ok_caps, caps_or_err = fibers.perform(d:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_or_err)) + + local ok_start, err_start = fibers.perform(d:start_op(scope)) + assert(ok_start == true, tostring(err_start)) + + drain_initial_emits(emit_ch, 'mcu') + + local opts = assert(cap_args.new.UARTOpenOpts()) + local ok, reply = fibers.perform(call_control_op(d.control_ch, 'open', opts)) + assert(ok == true, tostring(reply)) + wait_emit(emit_ch, 'uart', 'mcu', 'state', 'status', 1.0) + wait_emit(emit_ch, 'uart', 'mcu', 'event', 'opened', 1.0) + + local ok_stop, err_stop = fibers.perform(d:shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + + local n, werr = fibers.perform(reply.session:write_op('x')) + assert(n == nil) + assert(werr ~= nil) + end) +end + + +function T.read_ops_fail_after_stop() + local uart = fresh_driver() + + runfibers.run(function(scope) + local port = pty.open(scope) + local emit_ch = channel.new(16) + + local d = uart.new('mcu', port.slave_name, 115200, '8N1', nil) + + local ok_caps, caps_or_err = fibers.perform(d:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_or_err)) + + local ok_start, err_start = fibers.perform(d:start_op(scope)) + assert(ok_start == true, tostring(err_start)) + + drain_initial_emits(emit_ch, 'mcu') + + local opts = assert(cap_args.new.UARTOpenOpts()) + local ok, reply = fibers.perform(call_control_op(d.control_ch, 'open', opts)) + assert(ok == true, tostring(reply)) + wait_emit(emit_ch, 'uart', 'mcu', 'state', 'status', 1.0) + wait_emit(emit_ch, 'uart', 'mcu', 'event', 'opened', 1.0) + + local ok_stop, err_stop = fibers.perform(d:shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + + local chunk, rerr = fibers.perform(reply.session:read_some_op(1)) + assert(chunk == nil) + assert(rerr ~= nil) + + local n, werr = fibers.perform(reply.session:write_op('x')) + assert(n == nil) + assert(werr ~= nil) + end) +end + +function T.concurrent_open_allows_exactly_one_winner() + local uart = fresh_driver() + + runfibers.run(function(scope) + local port = pty.open(scope) + local emit_ch = channel.new(16) + local results_ch = channel.new(2) + + local d = uart.new('mcu', port.slave_name, 115200, '8N1', nil) + local ok_caps, caps_or_err = fibers.perform(d:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_or_err)) + local ok_start, err_start = fibers.perform(d:start_op(scope)) + assert(ok_start == true, tostring(err_start)) + drain_initial_emits(emit_ch, 'mcu') + + local opts = assert(cap_args.new.UARTOpenOpts()) + local function contender(name) + local ok, value = fibers.perform(call_control_op(d.control_ch, 'open', opts)) + fibers.perform(results_ch:put_op({ name = name, ok = ok, value = value })) + end + + local s1, e1 = scope:spawn(function() contender('a') end) + assert(s1 == true, tostring(e1)) + local s2, e2 = scope:spawn(function() contender('b') end) + assert(s2 == true, tostring(e2)) + + local r1 = recv_or_fail(results_ch) + local r2 = recv_or_fail(results_ch) + local results = { r1, r2 } + local winners, losers = {}, {} + for _, r in ipairs(results) do + if r.ok then winners[#winners + 1] = r else losers[#losers + 1] = r end + end + assert(#winners == 1) + assert(#losers == 1) + assert(type(losers[1].value) == 'string' and losers[1].value:match('busy')) + + local st = wait_emit(emit_ch, 'uart', 'mcu', 'state', 'status', 1.0) + assert(st.data.open == true) + assert(st.data.lease_id == winners[1].value.lease_id) + local ev = wait_emit(emit_ch, 'uart', 'mcu', 'event', 'opened', 1.0) + assert(ev.data.lease_id == winners[1].value.lease_id) + + local ok_close, err_close = fibers.perform(winners[1].value.session:close_op()) + assert(ok_close == true, tostring(err_close)) + wait_emit(emit_ch, 'uart', 'mcu', 'state', 'status', 1.0) + wait_emit(emit_ch, 'uart', 'mcu', 'event', 'closed', 1.0) + + local ok_stop, err_stop = fibers.perform(d:shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.open_rejects_invalid_opts() + local uart = fresh_driver() + + runfibers.run(function(scope) + local port = pty.open(scope) + local emit_ch = channel.new(8) + + local d = uart.new('mcu', port.slave_name, 115200, '8N1', nil) + + local ok_caps, caps_or_err = fibers.perform(d:capabilities_op(emit_ch)) + assert(ok_caps == true, tostring(caps_or_err)) + + local ok_start, err_start = fibers.perform(d:start_op(scope)) + assert(ok_start == true, tostring(err_start)) + + drain_initial_emits(emit_ch, 'mcu') + + local ok, err = fibers.perform(call_control_op(d.control_ch, 'open', { bogus = true })) + assert(ok == false) + assert(type(err) == 'string') + assert(err:match('invalid open opts') or err:match('unsupported')) + + local ok_stop, err_stop = fibers.perform(d:shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +return T diff --git a/tests/unit/hal/uart_manager_spec.lua b/tests/unit/hal/uart_manager_spec.lua new file mode 100644 index 00000000..c618669a --- /dev/null +++ b/tests/unit/hal/uart_manager_spec.lua @@ -0,0 +1,225 @@ +local fibers = require 'fibers' +local channel = require 'fibers.channel' + +local runfibers = require 'tests.support.run_fibers' +local pty = require 'tests.support.pty' + +local T = {} + +local function fresh_manager() + package.loaded['services.hal.managers.uart'] = nil + package.loaded['services.hal.drivers.uart'] = nil + return require('services.hal.managers.uart') +end + +local function recv_or_fail(ch) + local v, err = fibers.perform(ch:get_op()) + assert(v ~= nil, tostring(err)) + return v +end + +function T.start_op_and_shutdown_op_round_trip() + local M = fresh_manager() + + runfibers.run(function() + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(8) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.apply_config_op_before_start_fails() + local M = fresh_manager() + + runfibers.run(function() + local ok, err = fibers.perform(M.apply_config_op({ + serial_ports = { + { id = 'mcu', path = '/dev/ttyS0', baud = 115200, mode = '8N1' }, + }, + })) + assert(ok == false) + assert(tostring(err):match('not started')) + end) +end + +function T.apply_config_op_adds_uart_driver_and_emits_added_event() + local M = fresh_manager() + + runfibers.run(function(scope) + local port = pty.open(scope) + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(16) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok_cfg, err_cfg = fibers.perform(M.apply_config_op({ + serial_ports = { + { id = 'mcu', path = port.slave_name, baud = 115200, mode = '8N1' }, + }, + })) + assert(ok_cfg == true, tostring(err_cfg)) + + local ev = recv_or_fail(dev_ev_ch) + assert(ev.event_type == 'added') + assert(ev.class == 'uart') + assert(ev.id == 'mcu') + assert(type(ev.capabilities) == 'table' and #ev.capabilities == 1) + assert(ev.capabilities[1].class == 'uart') + assert(ev.meta.provider == 'hal.uart') + assert(ev.meta.source_id == 'uart_manager') + assert(ev.meta.path == port.slave_name) + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.reapply_same_config_is_idempotent() + local M = fresh_manager() + + runfibers.run(function(scope) + local port = pty.open(scope) + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(16) + + assert(fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) == true) + + local cfg = { + serial_ports = { + { id = 'mcu', path = port.slave_name, baud = 115200, mode = '8N1' }, + }, + } + + local ok1, err1 = fibers.perform(M.apply_config_op(cfg)) + assert(ok1 == true, tostring(err1)) + local added = recv_or_fail(dev_ev_ch) + assert(added.event_type == 'added') + + local ok2, err2 = fibers.perform(M.apply_config_op(cfg)) + assert(ok2 == true, tostring(err2)) + + local which = fibers.perform(require('fibers').named_choice{ + msg = dev_ev_ch:get_op():wrap(function(v) return 'msg', v end), + timeout = require('fibers.sleep').sleep_op(0.05):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.changing_path_causes_removed_then_added() + local M = fresh_manager() + + runfibers.run(function(scope) + local port1 = pty.open(scope) + local port2 = pty.open(scope) + local dev_ev_ch = channel.new(16) + local cap_emit_ch = channel.new(16) + + assert(fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) == true) + + local ok1, err1 = fibers.perform(M.apply_config_op({ + serial_ports = { + { id = 'mcu', path = port1.slave_name, baud = 115200, mode = '8N1' }, + }, + })) + assert(ok1 == true, tostring(err1)) + assert(recv_or_fail(dev_ev_ch).event_type == 'added') + + local ok2, err2 = fibers.perform(M.apply_config_op({ + serial_ports = { + { id = 'mcu', path = port2.slave_name, baud = 115200, mode = '8N1' }, + }, + })) + assert(ok2 == true, tostring(err2)) + + local ev1 = recv_or_fail(dev_ev_ch) + local ev2 = recv_or_fail(dev_ev_ch) + assert(ev1.event_type == 'removed') + assert(ev1.meta.source_id == 'uart_manager') + assert(ev2.event_type == 'added') + assert(ev2.id == 'mcu') + assert(ev2.meta.source_id == 'uart_manager') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.changing_baud_or_mode_causes_removed_then_added() + local M = fresh_manager() + + runfibers.run(function(scope) + local port = pty.open(scope) + local dev_ev_ch = channel.new(16) + local cap_emit_ch = channel.new(16) + + assert(fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) == true) + + local ok1, err1 = fibers.perform(M.apply_config_op({ + serial_ports = { + { id = 'mcu', path = port.slave_name, baud = 115200, mode = '8N1' }, + }, + })) + assert(ok1 == true, tostring(err1)) + assert(recv_or_fail(dev_ev_ch).event_type == 'added') + + local ok2, err2 = fibers.perform(M.apply_config_op({ + serial_ports = { + { id = 'mcu', path = port.slave_name, baud = 9600, mode = '8N1' }, + }, + })) + assert(ok2 == true, tostring(err2)) + + local ev1 = recv_or_fail(dev_ev_ch) + local ev2 = recv_or_fail(dev_ev_ch) + assert(ev1.event_type == 'removed') + assert(ev2.event_type == 'added') + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +function T.fault_op_is_inert_before_start() + local M = fresh_manager() + + runfibers.run(function() + local which = fibers.perform(fibers.named_choice{ + fault = M.fault_op():wrap(function(...) return 'fault', ... end), + timeout = require('fibers.sleep').sleep_op(0.05):wrap(function() return 'timeout' end), + }) + assert(which == 'timeout') + end) +end + +function T.apply_config_op_rejects_bare_uart_list() + local M = fresh_manager() + + runfibers.run(function(scope) + local dev_ev_ch = channel.new(8) + local cap_emit_ch = channel.new(16) + + local ok_start, err_start = fibers.perform(M.start_op(nil, dev_ev_ch, cap_emit_ch)) + assert(ok_start == true, tostring(err_start)) + + local ok_cfg, err_cfg = fibers.perform(M.apply_config_op({ + { id = 'mcu', path = '/dev/ttyS0', baud = 115200, mode = '8N1' }, + })) + assert(ok_cfg == false) + assert(tostring(err_cfg):match('serial_ports') or tostring(err_cfg):match('only supports')) + + local ok_stop, err_stop = fibers.perform(M.shutdown_op()) + assert(ok_stop == true, tostring(err_stop)) + end) +end + +return T diff --git a/tests/unit/hal/wired_provider_spec.lua b/tests/unit/hal/wired_provider_spec.lua new file mode 100644 index 00000000..77f51430 --- /dev/null +++ b/tests/unit/hal/wired_provider_spec.lua @@ -0,0 +1,290 @@ +local fibers = require 'fibers' +local provider = require 'services.hal.backends.wired.providers.rtl8380m_http' +local ok_cjson, cjson = pcall(require, 'cjson.safe') +if not ok_cjson then cjson = require 'cjson' end + +local tests = {} +local function assert_true(v,msg) if v ~= true then error(msg or 'expected true',2) end end +local function assert_eq(a,b,msg) if a ~= b then error(msg or ('expected '..tostring(b)..', got '..tostring(a)),2) end end +local function assert_not_nil(v,msg) if v == nil then error(msg or 'expected non-nil',2) end end + + +function tests.test_wired_manager_provider_ids_are_config_driven() + local manager = require 'services.hal.managers.wired' + local ids, err = manager._test.normalise_provider_ids({ + providers = { + ['cm5-local-wired'] = { provider = 'static' }, + ['expansion-a'] = { provider = 'static' }, + }, + }) + assert_not_nil(ids, err) + assert_eq(#ids, 2) + assert_eq(ids[1], 'cm5-local-wired') + assert_eq(ids[2], 'expansion-a') + + ids, err = manager._test.normalise_provider_ids({ providers = {} }) + assert_not_nil(ids, err) + assert_eq(#ids, 0) +end + + +function tests.test_wired_manager_provider_ids_must_be_map_keys() + local manager = require 'services.hal.managers.wired' + local ids, err = manager._test.normalise_provider_ids({ + providers = { + ['switch-main'] = { provider = 'rtl8380m_http', id = 'legacy-id' }, + }, + }) + assert_eq(ids, nil) + assert_true(type(err) == 'string' and err:find('map key', 1, true) ~= nil, tostring(err)) +end + + +function tests.test_wired_provider_loader_requires_provider_field() + local loader = require 'services.hal.backends.wired.provider' + local p, err = loader.new({}, { provider_id = 'x' }) + assert_eq(p, nil) + assert_true(type(err) == 'string' and err:find('requires provider', 1, true) ~= nil, tostring(err)) +end + +function tests.test_static_provider_requires_manager_provider_id_and_surfaces() + local static = require 'services.hal.backends.wired.providers.static' + local p, err = static.new({ provider = 'static', surfaces = { eth0 = {} } }, {}) + assert_eq(p, nil) + assert_true(type(err) == 'string' and err:find('provider_id', 1, true) ~= nil, tostring(err)) + + p, err = static.new({ provider = 'static' }, { provider_id = 'cm5-local-wired' }) + assert_eq(p, nil) + assert_true(type(err) == 'string' and err:find('surfaces', 1, true) ~= nil, tostring(err)) +end + +function tests.test_rtl8380m_http_requires_canonical_provider_and_http_config() + local p, err = provider.new({ + id = 'switch-main', + base_url = 'http://192.168.1.1/', + username = 'admin', + password = 'admin', + timeout_s = 0.8, + http = { capability = 'main', response_parser = 'legacy-http1-close' }, + }, { provider_id = 'switch-main', http_client_for = function () return { exchange_op = function () end } end }) + assert_eq(p, nil) + assert_true(type(err) == 'string' and err:find('id', 1, true) ~= nil, tostring(err)) + + p, err = provider.new({ + base_url = '192.168.1.1/', + username = 'admin', + password = 'admin', + timeout_s = 0.8, + http = { capability = 'main', response_parser = 'legacy-http1-close' }, + }, { provider_id = 'switch-main', http_client_for = function () return { exchange_op = function () end } end }) + assert_eq(p, nil) + assert_true(type(err) == 'string' and err:find('scheme', 1, true) ~= nil, tostring(err)) + + p, err = provider.new({ + base_url = 'http://192.168.1.1/', + username = 'admin', + password = 'admin', + timeout_s = 0.8, + http = { capability = 'main' }, + }, { provider_id = 'switch-main', http_client_for = function () return { exchange_op = function () end } end }) + assert_eq(p, nil) + assert_true(type(err) == 'string' and err:find('response_parser', 1, true) ~= nil, tostring(err)) +end + + +function tests.test_rtl8380m_http_snapshot_uses_legacy_http1_close_parser_for_cgi() + fibers.run(function () + local seen = {} + local payloads = { + home_main = { ports = { { port = 'GE1' }, { port = 'GE9' } }, model = 'RTL8380' }, + panel_info = { ports = { { linkup = true, speed = '100', dupFull = true }, { linkup = false, media = 'fiber' } } }, + sys_sysinfo = { hostname = 'Switch', sysMac = '00:E0:5C:24:16:39', fwVer = '1.0.0.6', loaderVer = '3.6.7.55090', currIpv4 = '192.168.1.1' }, + port_port = { ports = { { adminStatus = true, operStatus = true, operSpeed = '100M', operDuplex = 'Full', type = '1000M Copper' }, { adminStatus = true, operStatus = false, type = '1000M Fiber' } } }, + vlan_create = { vlans = { { vlan = 1, name = 'default' } } }, + vlan_conf = { vlan = 1, ports = { { membership = 3, pvid = true, forbidden = false }, { membership = 3, pvid = true, forbidden = false } } }, + vlan_port = { ports = { { mode = 2, pvid = 1, accFrameType = 0, ingressFilter = true, tpid = '0x8100' }, { mode = 2, pvid = 1, accFrameType = 0, ingressFilter = true, tpid = '0x8100' } } }, + vlan_membership = { ports = { { adminVlans = '1UP', operVlans = '1UP' }, { adminVlans = '1UP', operVlans = '1UP' } } }, + poe_poe = { devPower = 0, devTemp = 28, ports = { { portEnable = true, portStatus = false, portType = 'N/A', portPowerLimit = 0 } } }, + sys_cpumem = { cpu = 3, mem = 61 }, + rmon_statistics = { ports = { { bytesRec = 1234, pktsRec = 12, dropEvents = 1, CRCAlignErr = 2, bPktsRec = 3, mPktsRec = 4 } } }, + lldp_local = {}, + lldp_neighbor = {}, + } + local http_ref = { + exchange_op = function(_, args) + return fibers.run_scope_op(function () + assert_eq(args.response_parser, 'legacy-http1-close', 'CGI request should ask HTTP service for legacy HTTP/1 close parsing') + assert_eq(args.timeout_s, 8) + assert_not_nil(args.max_response_bytes, 'legacy CGI requests should be bounded') + seen[#seen + 1] = args.uri + local cmd = tostring(args.uri):match('cmd=([^&]+)') + local body = cjson.encode({ data = payloads[cmd] or {} }) + local ok, err = fibers.perform(args.response_sink:write_chunk_op(body)) + assert_true(ok, tostring(err)) + return { result = { status = '200', headers = {} } } + end):wrap(function (status, _report, result_or_primary, err) + if status == 'ok' then return result_or_primary, err end + return nil, result_or_primary or status + end) + end, + } + local p = assert(provider.new({ + base_url = 'http://192.168.1.1/', + username = 'admin', + password = 'admin', + timeout_s = 8, + disable_login = true, + http = { capability = 'main', response_parser = 'legacy-http1-close' }, + }, { + provider_id = 'switch-main', + http_client_for = function () return http_ref end, + })) + local snap = fibers.perform(p:snapshot_op({})) + assert_true(snap.ok, snap.status and snap.status.err) + assert_eq(snap.surfaces.GE1.link.state, 'up') + assert_eq(snap.surfaces.GE1.link.speed_mbps, 100) + assert_eq(snap.surfaces.GE1.attachment.mode, 'trunk') + assert_eq(snap.surfaces.GE1.attachment.accept_frame_type, 'all') + assert_true(snap.surfaces.GE1.capabilities.poe) + assert_eq(snap.runtime.cpu.utilisation_pct, 3) + assert_eq(snap.runtime.memory.utilisation_pct, 61) + assert_eq(snap.power.poe.temperature_c, 28) + assert_eq(snap.surfaces.GE1.counters, nil, 'stable surfaces must not carry volatile counters') + assert_eq(snap.counters.GE1.rx.bytes, 1234) + assert_eq(snap.counters.GE1.rx.errors, 2) + assert_eq(snap.surfaces.GE9.link.media, 'fiber') + assert_true(#seen >= 10, 'expected seeded CGI snapshot reads') + end) +end + + +function tests.test_rtl8380m_http_accepts_narrow_http_client_factory() + fibers.run(function () + local requested + local calls = 0 + local http_ref = { + exchange_op = function(_, args) + calls = calls + 1 + return fibers.run_scope_op(function () + assert_eq(args.response_parser, 'legacy-http1-close') + local cmd = tostring(args.uri):match('cmd=([^&]+)') + local payloads = { + home_main = { ports = { { port = 'GE1' } }, model = 'RTL8380' }, + panel_info = { ports = { { linkup = false } } }, + sys_sysinfo = { hostname = 'Switch' }, + port_port = { ports = { {} } }, + vlan_create = {}, + vlan_conf = { ports = { {} } }, + vlan_port = { ports = { {} } }, + vlan_membership = { ports = { {} } }, + poe_poe = { ports = {} }, + sys_cpumem = {}, + rmon_statistics = { ports = {} }, + lldp_local = {}, + lldp_neighbor = {}, + } + local body = cjson.encode({ data = payloads[cmd] or {} }) + local ok, err = fibers.perform(args.response_sink:write_chunk_op(body)) + assert_true(ok, tostring(err)) + return { result = { status = '200', headers = {} } } + end):wrap(function (status, _report, result_or_primary, err) + if status == 'ok' then return result_or_primary, err end + return nil, result_or_primary or status + end) + end, + } + local p = assert(provider.new({ + base_url = 'http://192.168.1.1/', + username = 'admin', + password = 'admin', + timeout_s = 8, + disable_login = true, + http = { capability = 'switch-http', response_parser = 'legacy-http1-close' }, + }, { + provider_id = 'switch-main', + http_client_for = function (cap_id) + requested = cap_id + return http_ref + end, + })) + local snap = fibers.perform(p:snapshot_op({})) + assert_true(snap.ok, snap.status and snap.status.err) + assert_eq(requested, 'switch-http') + assert_true(calls >= 10, 'expected snapshot CGI calls through HTTP dependency port') + end) +end + + + +function tests.test_wired_driver_separates_backend_provider_name_and_provider_id() + local driver_mod = require 'services.hal.drivers.wired' + local d, err = driver_mod.new({ + provider = 'static', + mode = 'read_only', + surfaces = { eth0 = { provider_surface_id = 'eth0' } }, + }, { provider_id = 'cm5-local-wired' }) + assert_not_nil(d, err) + assert_not_nil(d.backend) + assert_eq(d.provider, nil) + assert_eq(d.provider_name, 'static') + assert_eq(d.provider_id, 'cm5-local-wired') + assert_not_nil(d.snapshot_op) +end + +function tests.test_wired_driver_requires_observe_groups_backend_contract() + package.loaded['services.hal.backends.wired.providers.invalid_missing_observe_groups'] = { + new = function () + return { + snapshot_op = function () end, + watch_op = function () end, + apply_attachments_op = function () end, + set_poe_op = function () end, + bounce_op = function () end, + terminate = function () end, + } + end, + } + local driver_mod = require 'services.hal.drivers.wired' + local d, err = driver_mod.new({ provider = 'invalid_missing_observe_groups' }, { provider_id = 'bad' }) + assert_eq(d, nil) + assert_true(type(err) == 'string' and err:find('observe_groups_op', 1, true) ~= nil, tostring(err)) + package.loaded['services.hal.backends.wired.providers.invalid_missing_observe_groups'] = nil +end + +function tests.test_rtl8380m_observe_groups_deduplicates_shared_commands() + local commands, err = provider._test.commands_for_groups({ 'panel', 'poe', 'counters' }) + assert_not_nil(commands, err) + local seen = {} + for _, cmd in ipairs(commands) do + assert_eq(seen[cmd], nil, 'duplicate command ' .. tostring(cmd)) + seen[cmd] = true + end + assert_true(seen.home_main == true, 'home_main should be included once') + assert_true(seen.panel_info == true, 'panel_info should be included') + assert_true(seen.poe_poe == true, 'poe_poe should be included') + assert_true(seen.rmon_statistics == true, 'rmon_statistics should be included') +end + +function tests.test_wired_manager_requires_canonical_poll_table() + local manager = require 'services.hal.managers.wired' + local plan, err = manager._test.provider_poll_plan({}) + assert_eq(plan, nil) + assert_true(type(err) == 'string' and err:find('poll is required', 1, true) ~= nil, tostring(err)) + + plan, err = manager._test.provider_poll_plan({ poll_interval_s = 0.5 }) + assert_eq(plan, nil) + assert_true(type(err) == 'string' and err:find('poll_interval_s', 1, true) ~= nil, tostring(err)) + + plan, err = manager._test.provider_poll_plan({ + poll = { + static = { interval_s = 30.0, groups = { 'snapshot' } }, + }, + }) + assert_not_nil(plan, err) + assert_eq(#plan, 1) + assert_eq(plan[1].name, 'static') + assert_eq(plan[1].interval_s, 30.0) + assert_eq(plan[1].groups[1], 'snapshot') +end + +return tests diff --git a/tests/unit/http/test_body.lua b/tests/unit/http/test_body.lua new file mode 100644 index 00000000..29467444 --- /dev/null +++ b/tests/unit/http/test_body.lua @@ -0,0 +1,73 @@ +local fibers = require 'fibers' +local body = require 'services.http.body' +local blob = require 'devicecode.blob_source' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +function M.test_rejects_inline_bytes() + local bodies, err = body.validate_exchange_bodies({ body = 'bytes' }) + eq(bodies, nil) + eq(err, 'invalid_args') + bodies, err = body.validate_exchange_bodies({ data = 'bytes' }) + eq(bodies, nil) + eq(err, 'invalid_args') +end + +function M.test_accepts_lua_body_source_and_response_sink_capabilities() + local source = blob.from_string('abcd') + local sink = blob.to_memory() + local bodies = ok(body.validate_exchange_bodies({ body_source = source, response_sink = sink })) + eq(bodies.source, source) + eq(bodies.sink, sink) +end + +function M.test_rejects_bad_body_source_and_sink_shapes() + local bodies, err = body.validate_exchange_bodies({ body_source = { terminate = function () return true end } }) + eq(bodies, nil) + eq(err, 'invalid_args') + bodies, err = body.validate_exchange_bodies({ response_sink = { write_chunk_op = function () end } }) + eq(bodies, nil) + eq(err, 'invalid_args') +end + +function M.test_source_and_sink_capabilities_copy_inside_ops() + fibers.run(function () + local source = blob.from_string('abcd') + local sink = blob.to_memory() + local rep = ok(fibers.perform(body.copy_source_to_sink_op(source, sink, { max_bytes = 16 }))) + eq(rep.bytes, 4) + eq(sink:result(), 'abcd') + end) +end + +function M.test_request_body_pipe_is_a_bounded_fibers_sink_and_lua_http_iterator() + fibers.run(function () + local request_body = require 'services.http.transport.request_body' + local c = require 'fibers.cond' + local wake = c.new() + local pipe = ok(request_body.new_pipe({ + condition_factory = function () + return { + wait = function () return fibers.perform(wake:wait_op()) end, + signal = function () wake:signal(); wake = c.new(); return true end, + } + end, + max_buffered_chunks = 2, + })) + + ok(fibers.perform(pipe:write_chunk_op('ab'))) + ok(fibers.perform(pipe:write_chunk_op('cd'))) + ok(fibers.perform(pipe:finish_op())) + local iter = pipe:body_iterator() + eq(iter(), 'ab') + eq(iter(), 'cd') + eq(iter(), nil) + end) +end + +return M diff --git a/tests/unit/http/test_client.lua b/tests/unit/http/test_client.lua new file mode 100644 index 00000000..60a02987 --- /dev/null +++ b/tests/unit/http/test_client.lua @@ -0,0 +1,377 @@ +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local runtime = require 'fibers.runtime' +local driver_mod = require 'services.http.transport.cqueues_driver' +local client = require 'services.http.client' +local blob = require 'devicecode.blob_source' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +local function yield_once() + runtime.yield() +end + +local function yield_until(pred, msg) + for _ = 1, 50 do + if pred() then return true end + yield_once() + end + error(msg or 'condition was not reached', 2) +end + +local function join_child_with_timeout(child, timeout_s) + local which = fibers.perform(fibers.named_choice { + joined = child:join_op(), + timeout = sleep.sleep_op(timeout_s or 1), + }) + return which == 'joined' +end + +local function shell_quote(s) + s = tostring(s or '') + return "'" .. s:gsub("'", "'\\''") .. "'" +end + +local function shell_exit_status(a, b, c) + if type(a) == 'number' then + if a >= 256 then return math.floor(a / 256) end + return a + end + if a == true then return 0 end + if b == 'exit' and type(c) == 'number' then return c end + if type(c) == 'number' then return c end + return 1 +end + +local function run_filtered_child(filter, timeout_s) + local cmd = ('timeout %s env TEST_FILTER=%s luajit run.lua'):format( + tostring(timeout_s or 2), shell_quote(filter)) + return shell_exit_status(os.execute(cmd)) +end + +local function fake_controller() + local q = {} + return { + wrap = function (self, fn) q[#q + 1] = fn; return self end, + step = function () local fn = table.remove(q, 1); if fn then fn() end; return true end, + pollfd = function () return nil end, + events = function () return '' end, + timeout = function () return nil end, + } +end + +local function fake_driver() + return { + run_op = function (_, _, fn) + return fibers.guard(function () return fibers.always(fn()) end) + end, + } +end + +local function fake_condition_factory() + local c = require 'fibers.cond' + local wake = c.new() + return { + wait = function () return fibers.perform(wake:wait_op()) end, + signal = function () wake:signal(); wake = c.new(); return true end, + } +end + +local function fake_headers(status) + return { + get = function (_, name) if name == ':status' then return tostring(status or 200) end end, + each = function () + local rows = { { ':status', tostring(status or 200) }, { 'content-type', 'text/plain' } } + local i = 0 + return function () + i = i + 1 + local row = rows[i] + if row then return row[1], row[2] end + end + end, + } +end + +local function request_module(body_text) + return { + new_from_uri = function (uri) + return { + uri = uri, + headers = { + set = {}, + upsert = function (self, k, v) self.set[k] = v end, + append = function (self, k, v) self.set[k] = v end, + }, + set_body = function (self, body_iter) self.body_iter = body_iter end, + go = function (self) + local chunks = { body_text or ('body for ' .. self.uri) } + local i = 0 + return fake_headers(202), { + get_next_chunk = function () i = i + 1; return chunks[i] end, + shutdown = function () self.shutdown = true; return true end, + } + end, + } + end, + } +end + +function M.test_exchange_op_streams_response_to_sink_without_returning_body() + fibers.run(function () + local sink = blob.to_memory() + local result = ok(fibers.perform(client.exchange_op(fake_driver(), { + uri = 'http://example.test/path', + method = 'GET', + headers = { ['x-test'] = 'yes' }, + response_sink = sink, + }, { request_module = request_module('hello') }))) + eq(result.status, '202') + eq(result.body, nil) + eq(result.response_sink.bytes, 5) + eq(sink:result(), 'hello') + eq(result.headers['content-type'], 'text/plain') + end) +end + +function M.test_open_exchange_rejects_raw_body_before_backend_job() + fibers.run(function () + local called = false + local drv = { run_op = function () called = true; return fibers.always('bad') end } + local ex, err = fibers.perform(client.open_exchange_op(drv, { + uri = 'http://example.test/', + method = 'POST', + body = 'raw bytes', + }, { request_module = request_module() })) + eq(ex, nil) + eq(err, 'invalid_args') + eq(called, false) + end) +end + +function M.test_request_source_streams_through_fibers_native_body_pipe() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + local sent_body = {} + local request_mod = { + new_from_uri = function (uri) + return { + uri = uri, + headers = { + set = {}, + upsert = function (self, k, v) self.set[k] = v end, + append = function (self, k, v) self.set[k] = v end, + }, + set_body = function (self, iter) self.body_iter = iter end, + go = function (self) + while true do + local chunk = self.body_iter and self.body_iter() or nil + if chunk == nil then break end + sent_body[#sent_body + 1] = chunk + end + local chunks = { 'ok' } + local i = 0 + return fake_headers(201), { + get_next_chunk = function () i = i + 1; return chunks[i] end, + shutdown = function () return true end, + } + end, + } + end, + } + + fibers.run(function (scope) + assert(drv:start(scope)) + local result = ok(fibers.perform(client.exchange_op(drv, { + uri = 'http://example.test/', method = 'POST', body_source = blob.from_string('abcd'), + }, { + request_module = request_mod, + request_body_condition_factory = fake_condition_factory, + }))) + eq(result.status, '201') + eq(table.concat(sent_body), 'abcd') + drv:terminate('done') + end) +end + + +function M.test_exchange_op_does_not_terminate_committed_response_sink_after_success() + fibers.run(function () + local committed = false + local terminated = nil + local sink = { + write_chunk_op = function () return fibers.always(true) end, + commit_op = function () committed = true; return fibers.always(true) end, + terminate = function (_, reason) terminated = reason or true; return true end, + } + + local result = ok(fibers.perform(client.exchange_op(fake_driver(), { + uri = 'http://example.test/path', + method = 'GET', + response_sink = sink, + }, { request_module = request_module('hello') }))) + eq(result.response_sink.bytes, 5) + eq(committed, true, 'sink commit must run') + eq(terminated, nil, 'committed sink ownership must not be reclaimed by the exchange finaliser') + end) +end + +function M.test_exchange_op_terminates_uncommitted_response_sink_on_copy_failure() + fibers.run(function () + local terminated = nil + local sink = { + write_chunk_op = function () return fibers.always(nil, 'sink_failed') end, + commit_op = function () error('commit must not run after write failure', 0) end, + terminate = function (_, reason) terminated = reason or true; return true end, + } + + local result, err = fibers.perform(client.exchange_op(fake_driver(), { + uri = 'http://example.test/path', + method = 'GET', + response_sink = sink, + }, { request_module = request_module('hello') })) + eq(result, nil) + eq(err, 'sink_failed') + eq(terminated, 'failed', 'uncommitted sink must be terminated by the operation scope') + end) +end + + +function M.test_request_body_source_without_read_op_rejects_before_request_construction() + fibers.run(function () + local constructed = 0 + local request_mod = { + new_from_uri = function () constructed = constructed + 1; return request_module().new_from_uri('http://example.test/') end, + } + local result, err = fibers.perform(client.exchange_op(fake_driver(), { + uri = 'http://example.test/', method = 'POST', body_source = { terminate = function () return true end }, + }, { request_module = request_mod })) + eq(result, nil) + eq(err, 'invalid_args') + eq(constructed, 0, 'invalid request source must reject before lua-http request construction') + end) +end + +function M.test_open_exchange_active_abort_detaches_then_cleans_request_in_cqueues_job() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + local go_active = false + local release_go = false + local req_cancelled = false + local request_mod = { + new_from_uri = function (uri) + return { + uri = uri, + headers = { + upsert = function () end, + append = function () end, + }, + go = function () + go_active = true + while not release_go do runtime.yield() end + return nil, 'closed' + end, + cancel = function (_, reason) req_cancelled = reason or true; return true end, + } + end, + } + + fibers.run(function (scope) + assert(drv:start(scope)) + local waiter = assert(scope:child()) + local result, err + assert(waiter:spawn(function () + result, err = fibers.perform(client.open_exchange_op(drv, { + uri = 'http://example.test/', + method = 'GET', + }, { request_module = request_mod })) + end)) + + yield_until(function () return go_active end, 'request go should become active') + waiter:cancel('stop_waiting') + if not join_child_with_timeout(waiter, 0.25) then + release_go = true + drv:terminate('open exchange detach test bounded failure') + join_child_with_timeout(waiter, 0.25) + error('open_exchange caller did not return after abort; request cleanup must be detached to cqueues job', 2) + end + eq(result, nil) + eq(err, nil) + eq(req_cancelled, false, 'request must not be cancelled from the Fibers abort path') + ok(not drv:is_closed(), 'detached request must not close the driver') + release_go = true + yield_until(function () return req_cancelled == 'aborted' end, 'detached request cleanup should run after cqueues job returns') + ok(not drv:is_closed(), 'request cleanup is owned by cqueues job; driver should survive') + drv:terminate('done') + end) +end + +function M.test_exchange_read_active_abort_detaches_then_cleans_stream_in_cqueues_job() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + local read_active = false + local release_read = false + local stream = { + terminated = false, + get_next_chunk = function (self, timeout) + self.timeout = timeout + read_active = true + while not release_read do runtime.yield() end + return nil + end, + close = function (self) + self.terminated = true + return true + end, + } + local ex = require('services.http.exchange').make(drv, fake_headers(200), stream, { intra_stream_timeout = 0.125 }) + + fibers.run(function (scope) + assert(drv:start(scope)) + local waiter = assert(scope:child()) + local result, err + assert(waiter:spawn(function () result, err = fibers.perform(ex:read_chunk_op()) end)) + yield_until(function () return read_active end, 'exchange read should become active') + eq(stream.timeout, 0.125, 'exchange read should pass intra_stream_timeout to lua-http stream') + waiter:cancel('stop_waiting') + if not join_child_with_timeout(waiter, 0.25) then + release_read = true + drv:terminate('exchange read detach test bounded failure') + join_child_with_timeout(waiter, 0.25) + error('exchange read caller did not return after abort; stream cleanup must be detached to cqueues job', 2) + end + eq(result, nil) + eq(err, nil) + ok(not stream.terminated, 'stream must not be terminated from the Fibers abort path') + ok(ex:is_closed(), 'exchange handle should be marked closed immediately') + ok(not drv:is_closed(), 'detached exchange read must not close driver') + release_read = true + yield_until(function () return stream.terminated end, 'detached stream cleanup should run after cqueues read returns') + ok(not drv:is_closed(), 'exchange cleanup is owned by cqueues job; driver should survive') + drv:terminate('done') + end) +end + +-- The hostile OpenWrt regression is partly an ownership-boundary bug, not just +-- a black-box socket behaviour. Keep these wrappers under the same regression filter +-- substring as the devhost hostile test so a single focused run exercises the +-- unit-level contract that the old hard-close-only fix violated. The payloads +-- run in subprocesses so scheduler wedges are bounded failures rather than +-- stalled test-suite runs. +function M.test_metrics_style_exchange_timeout_regression_abort_detaches_open_exchange_request_cleanup_to_cqueues_job() + local code = run_filtered_child('test_open_exchange_active_abort_detaches_then_cleans_request_in_cqueues_job', + tonumber(os.getenv('HTTP_METRICS_TIMEOUT_UNIT_CHILD_TIMEOUT_S') or '') or 2) + eq(code, 0, 'open_exchange ownership payload should pass in a bounded child process') +end + +function M.test_metrics_style_exchange_timeout_regression_abort_detaches_response_stream_cleanup_to_cqueues_job() + local code = run_filtered_child('test_exchange_read_active_abort_detaches_then_cleans_stream_in_cqueues_job', + tonumber(os.getenv('HTTP_METRICS_TIMEOUT_UNIT_CHILD_TIMEOUT_S') or '') or 2) + eq(code, 0, 'exchange read ownership payload should pass in a bounded child process') +end + +return M diff --git a/tests/unit/http/test_config.lua b/tests/unit/http/test_config.lua new file mode 100644 index 00000000..8d0990c4 --- /dev/null +++ b/tests/unit/http/test_config.lua @@ -0,0 +1,78 @@ +local config = require 'services.http.config' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +function M.test_normalise_supplies_safe_defaults() + local cfg = ok(config.normalise({ schema = config.SCHEMA })) + eq(cfg.id, 'main') + eq(cfg.enabled, true) + eq(cfg.policy.allowed_schemes.http, true) + eq(cfg.policy.allowed_schemes.https, true) + eq(cfg.policy.allowed_schemes.ws, true) + eq(cfg.policy.allowed_schemes.wss, true) + eq(cfg.policy.allowed_response_parsers.strict, true) + eq(cfg.policy.allowed_response_parsers['legacy-http1-close'], nil) + eq(cfg.policy.legacy_http1_close_max_response_bytes, 1024 * 1024) + eq(cfg.observability.status_interval_s, 30) + eq(cfg.observability.request_trace, false) + eq(cfg.observability.success_events, false) + eq(cfg.observability.failure_rate_limit_s, 60) +end + +function M.test_policy_updates_are_copied_and_validated() + local raw = { + schema = config.SCHEMA, + policy = { + allowed_schemes = { https = true }, + allow_loopback = false, + allowed_hosts = { ['example.com'] = true }, + allowed_response_parsers = { strict = true, ['legacy-http1-close'] = true }, + legacy_http1_close_max_response_bytes = 12345, + }, + } + local cfg = ok(config.normalise(raw)) + eq(cfg.policy.allowed_schemes.https, true) + eq(cfg.policy.allowed_schemes.http, nil) + eq(cfg.policy.allow_loopback, false) + eq(cfg.policy.allowed_response_parsers['legacy-http1-close'], true) + eq(cfg.policy.legacy_http1_close_max_response_bytes, 12345) + raw.policy.allowed_hosts['example.com'] = false + eq(cfg.policy.allowed_hosts['example.com'], true, 'normalised config must not alias raw tables') +end + + +function M.test_observability_updates_are_copied_and_validated() + local raw = { + schema = config.SCHEMA, + observability = { + status_interval_s = 10, + request_trace = true, + success_events = true, + failure_rate_limit_s = 20, + }, + } + local cfg = ok(config.normalise(raw)) + eq(cfg.observability.status_interval_s, 10) + eq(cfg.observability.request_trace, true) + eq(cfg.observability.success_events, true) + eq(cfg.observability.failure_rate_limit_s, 20) + raw.observability.status_interval_s = 99 + eq(cfg.observability.status_interval_s, 10, 'normalised config must not alias raw observability') + + local bad, err = config.normalise({ schema = config.SCHEMA, observability = { every_request = true } }) + eq(bad, nil) + ok(tostring(err):find('observability has unknown field', 1, true)) +end + +function M.test_rejects_unknown_fields() + local cfg, err = config.normalise({ schema = config.SCHEMA, backend = {} }) + eq(cfg, nil) + ok(tostring(err):find('unknown field', 1, true)) +end + +return M diff --git a/tests/unit/http/test_headers.lua b/tests/unit/http/test_headers.lua new file mode 100644 index 00000000..6bc81ed9 --- /dev/null +++ b/tests/unit/http/test_headers.lua @@ -0,0 +1,35 @@ +local headers = require 'services.http.headers' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end + +local function ok(v, msg) + if not v then error(msg or 'assertion failed', 2) end + return v +end + +function M.test_status_headers_round_trip_to_table() + local h = ok(headers.status(201, { ['content-type'] = 'text/plain' })) + eq(headers.get_one(h, ':status'), '201') + eq(headers.get_one(h, 'content-type'), 'text/plain') + local t = headers.to_table(h) + eq(t[':status'], '201') + eq(t['content-type'], 'text/plain') +end + +function M.test_request_helpers_set_pseudo_headers_and_repeated_values() + local h = ok(headers.request('POST', '/upload', 'example.test', 'https', { + ['x-test'] = { 'a', 'b' }, + })) + eq(headers.get_one(h, ':method'), 'POST') + eq(headers.get_one(h, ':path'), '/upload') + local all = headers.get_all(h, 'x-test') + eq(#all, 2) + eq(all[1], 'a') + eq(all[2], 'b') +end + +return M diff --git a/tests/unit/http/test_policy.lua b/tests/unit/http/test_policy.lua new file mode 100644 index 00000000..1f9bc1f1 --- /dev/null +++ b/tests/unit/http/test_policy.lua @@ -0,0 +1,160 @@ +local policy = require 'services.http.policy' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end + +local function ok(v, msg) + if not v then error(msg or 'assertion failed', 2) end + return v +end + +function M.test_locality_requires_local_origin_without_fabric_peer_fields() + ok(policy.is_local_origin({ kind = 'local' })) + eq(policy.require_local_origin({ kind = 'fabric', peer_node = 'n1' }), nil) + local allowed, err = policy.require_local_origin({ kind = 'local', link_id = 'l1' }) + eq(allowed, nil) + eq(err, 'not_local') +end + + +function M.test_validate_uri_uses_lua_http_authority_boundary() + local parsed = ok(policy.validate_uri('http://example.test:8080/path?q=1')) + eq(parsed.scheme, 'http') + eq(parsed.authority, 'example.test:8080') + eq(parsed.host, 'example.test') + eq(parsed.port, 8080) + local ws = ok(policy.validate_uri('wss://example.test/ws')) + eq(ws.scheme, 'wss') + eq(ws.host, 'example.test') + local plain_ws = ok(policy.validate_uri('ws://example.test/ws')) + eq(plain_ws.scheme, 'ws') + eq(plain_ws.host, 'example.test') +end + +function M.test_validate_uri_rejects_missing_authority_via_lua_http_boundary() + local parsed, err = policy.validate_uri('http:///missing-host') + eq(parsed, nil) + eq(err, 'invalid_args') +end + +function M.test_validate_exchange_rejects_raw_body_bytes_over_control_plane() + local checked = ok(policy.validate_exchange_args { uri = 'http://example.test/', method = 'GET' }) + eq(checked.method, 'GET') + local bad, err = policy.validate_exchange_args { uri = 'http://example.test/', method = 'POST', body = 'bytes' } + eq(bad, nil) + eq(err, 'invalid_args') + bad, err = policy.validate_exchange_args { uri = 'http://example.test/', method = 'POST', data = 'bytes' } + eq(bad, nil) + eq(err, 'invalid_args') +end + +function M.test_validate_exchange_accepts_body_object_capabilities_and_rejects_remote_reference_tables() + local source = { read_chunk_op = function () end, terminate = function () return true end } + local sink = { write_chunk_op = function () end, terminate = function () return true end } + local checked = ok(policy.validate_exchange_args { uri = 'http://example.test/', method = 'POST', body_source = source, response_sink = sink }) + eq(checked.body_source, source) + eq(checked.response_sink, sink) + + local bad, err = policy.validate_exchange_args { uri = 'http://example.test/', method = 'POST', body_source = { kind = 'remote-ref' } } + eq(bad, nil) + eq(err, 'invalid_args') + bad, err = policy.validate_exchange_args { uri = 'http://example.test/', method = 'POST', source = source } + eq(bad, nil) + eq(err, 'invalid_args') +end + +function M.test_validate_listen_defaults_loopback_ephemeral_port() + local args = ok(policy.validate_listen_args {}) + eq(args.host, '127.0.0.1') + eq(args.port, 0) + eq(args.tls, false) +end + +function M.test_public_payload_validation_rejects_backend_objects() + local listen, lerr = policy.validate_listen_args { host = '127.0.0.1', port = 0, backend_timeout = 2 } + eq(listen, nil) + eq(lerr, 'invalid_args') + local exchange, eerr = policy.validate_exchange_args { uri = 'http://example.test/', request_module = {} } + eq(exchange, nil) + eq(eerr, 'invalid_args') + local ws, werr = policy.validate_connect_ws_args { uri = 'ws://example.test/', websocket_module = {} } + eq(ws, nil) + eq(werr, 'invalid_args') +end + + +function M.test_validate_exchange_accepts_legacy_http1_close_when_policy_allows_it() + local checked = ok(policy.validate_exchange_args({ + uri = 'http://192.168.1.1/cgi/get.cgi?cmd=home_login', + response_parser = 'legacy-http1-close', + timeout_s = 5, + }, { + allowed_response_parsers = { strict = true, ['legacy-http1-close'] = true }, + legacy_http1_close_max_response_bytes = 1024 * 1024, + })) + eq(checked.response_parser, 'legacy-http1-close') + eq(checked.timeout_s, 5) + + local bad, err = policy.validate_exchange_args({ + uri = 'https://192.168.1.1/cgi/get.cgi?cmd=home_login', + response_parser = 'legacy-http1-close', + timeout_s = 5, + }, { allowed_response_parsers = { strict = true, ['legacy-http1-close'] = true } }) + eq(bad, nil) + eq(err, 'unsupported_scheme') +end + +function M.test_validate_exchange_denies_legacy_parser_by_default_and_requires_timeout() + local bad, err = policy.validate_exchange_args { + uri = 'http://192.168.1.1/cgi/get.cgi?cmd=home_login', + response_parser = 'legacy-http1-close', + timeout_s = 5, + } + eq(bad, nil) + eq(err, 'response_parser_denied') + + bad, err = policy.validate_exchange_args({ + uri = 'http://192.168.1.1/cgi/get.cgi?cmd=home_login', + response_parser = 'legacy-http1-close', + }, { allowed_response_parsers = { strict = true, ['legacy-http1-close'] = true } }) + eq(bad, nil) + eq(err, 'timeout_required') +end + + +function M.test_validate_exchange_legacy_http1_close_is_tightly_bounded() + local bad, err = policy.validate_exchange_args({ + uri = 'http://192.168.1.1/cgi/get.cgi?cmd=home_login', + method = 'PUT', + response_parser = 'legacy-http1-close', + timeout_s = 5, + }, { allowed_response_parsers = { strict = true, ['legacy-http1-close'] = true } }) + eq(bad, nil) + eq(err, 'unsupported_method') + + bad, err = policy.validate_exchange_args({ + uri = 'http://192.168.1.1/cgi/get.cgi?cmd=home_login', + response_parser = 'legacy-http1-close', + timeout_s = 5, + max_response_bytes = 2048, + }, { + allowed_response_parsers = { strict = true, ['legacy-http1-close'] = true }, + legacy_http1_close_max_response_bytes = 1024, + }) + eq(bad, nil) + eq(err, 'response_too_large') + + bad, err = policy.validate_exchange_args({ + uri = 'http://192.168.1.1/cgi/get.cgi?cmd=home_login', + response_parser = 'legacy-http1-close', + timeout_s = 5, + headers = { ['X-Bad'] = 'ok\r\nInjected: yes' }, + }, { allowed_response_parsers = { strict = true, ['legacy-http1-close'] = true } }) + eq(bad, nil) + eq(err, 'invalid_args') +end + +return M diff --git a/tests/unit/http/test_registry.lua b/tests/unit/http/test_registry.lua new file mode 100644 index 00000000..07a04a32 --- /dev/null +++ b/tests/unit/http/test_registry.lua @@ -0,0 +1,83 @@ +local registry = require 'services.http.registry' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +local function event_port(events) + return { + emit_required = function (_, ev) + events[#events + 1] = ev + return true, nil + end, + } +end + +function M.test_registry_records_transfer_and_shutdown_termination() + local events = {} + local reg = registry.new({ events_port = event_port(events) }) + local handle = { closed = false, terminate = function (self, reason) self.closed = reason or true; return true end } + local id = ok(reg:register('exchange', handle, { generation = 7 })) + eq(reg:count('exchange'), 1) + ok(reg:mark_transferred(id, 7)) + reg:terminate_all('shutdown') + eq(handle.closed, 'shutdown') + eq(reg:count('exchange'), 0) + eq(events[1].kind, 'exchange_registered') + eq(events[2].kind, 'exchange_transferred') + eq(events[#events].kind, 'exchange_terminated') +end + + +function M.test_registry_can_report_through_service_event_port() + local events = {} + local port = { + emit_required = function (_, ev) + events[#events + 1] = ev + return true, nil + end, + } + local reg = registry.new({ events_port = port }) + local id = ok(reg:register('listener', { terminate = function () return true end }, { generation = 4 })) + ok(reg:mark_transferred(id, 4)) + eq(events[1].kind, 'listener_registered') + eq(events[2].kind, 'listener_transferred') +end + +function M.test_registry_ignores_stale_generation_removal() + local reg = registry.new() + local id = ok(reg:register('listener', { terminate = function () return true end }, { generation = 1 })) + local removed, err = reg:remove(id, 'stale', 2) + eq(removed, false) + eq(err, 'stale_handle') + eq(reg:count('listener'), 1) +end + + +function M.test_registry_reserve_register_transfer_remove_is_stale_safe() + local events = {} + local reg = registry.new({ events_port = event_port(events) }) + local id = ok(reg:reserve('listener', { generation = 3 })) + eq(reg:count('listener'), 0, 'reserved handles are not active') + eq(#events, 0, 'reserve should not emit an active-handle event') + local handle = { terminated = nil, terminate = function (self, reason) self.terminated = reason; return true end } + local rid = ok(reg:register('listener', handle, { id = id, generation = 3 })) + eq(rid, id) + eq(reg:count('listener'), 1) + ok(reg:mark_transferred(id, 3)) + local stale, serr = reg:remove(id, 'stale', 99) + eq(stale, false) + eq(serr, 'stale_handle') + eq(reg:count('listener'), 1) + ok(reg:terminate(id, 'shutdown', 3)) + eq(handle.terminated, 'shutdown') + eq(reg:count('listener'), 0) + eq(events[1].kind, 'listener_registered') + eq(events[2].kind, 'listener_transferred') + eq(events[3].kind, 'listener_terminated') +end + +return M diff --git a/tests/unit/http/test_service.lua b/tests/unit/http/test_service.lua new file mode 100644 index 00000000..7cd8cc13 --- /dev/null +++ b/tests/unit/http/test_service.lua @@ -0,0 +1,1287 @@ +local fibers = require 'fibers' +local runtime = require 'fibers.runtime' +local op = require 'fibers.op' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' +local bus = require 'bus' + +local http_service = require 'services.http.service' +local sdk_mod = require 'services.http.sdk' +local listener_owner = require 'services.http.listener_owner' +local lua_http = require 'services.http.transport.lua_http' +local public_ws = require 'services.http.websocket' +local driver_mod = require 'services.http.transport.cqueues_driver' +local operation_owner = require 'services.http.operation_owner' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end + +local function ok(v, msg) + if not v then error(msg or 'assertion failed', 2) end + return v +end + +local function yield_many(n) + for _ = 1, n do runtime.yield() end +end + + +local function yield_until(pred, msg) + for _ = 1, 80 do + if pred() then return true end + runtime.yield() + end + error(msg or 'condition was not reached', 2) +end + +local function recv_payload(sub, label) + local which, msg, err = fibers.perform(op.named_choice({ + msg = sub:recv_op(), + timeout = sleep.sleep_op(0.1), + })) + if which ~= 'msg' then error(label or 'timed out waiting for message', 2) end + ok(msg, err or label or 'expected message') + return msg.payload +end + +local function recv_payload_where(sub, pred, label) + local deadline = label or 'timed out waiting for matching message' + for _ = 1, 20 do + local which, msg, err = fibers.perform(op.named_choice({ + msg = sub:recv_op(), + timeout = sleep.sleep_op(0.1), + })) + if which ~= 'msg' then error(deadline, 2) end + ok(msg, err or deadline) + local payload = msg.payload + if pred(payload) then return payload end + end + error(deadline, 2) +end + +local function table_empty(t) + for _ in pairs(t or {}) do return false end + return true +end + +local function fake_controller() + local q = {} + return { + wrap = function (self, fn) q[#q + 1] = fn; return self end, + step = function () local fn = table.remove(q, 1); if fn then fn() end; return true end, + pollfd = function () return nil end, + events = function () return '' end, + timeout = function () return nil end, + } +end + +local function fake_driver() + return { + started = false, + terminated = nil, + start = function (self) self.started = true; return true end, + terminate = function (self, reason) self.terminated = reason or true; return true end, + run_op = function (_, _, fn) return fibers.guard(function () return fibers.always(fn()) end) end, + } +end + +function M.test_service_retains_capability_metadata_and_status() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local drv = fake_driver() + local svc = ok(http_service.open_handle(root, { driver = drv, id = 'main' })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local ref = sdk_mod.new_ref(user, 'main') + local rep = ok(fibers.perform(ref:status_op())) + ok(rep.status.ready, 'service should report ready') + svc:terminate('done') + eq(drv.terminated, 'done') + end) +end + + +function M.test_http_failure_recovery_is_visible_and_resets_suppression() + fibers.run(function () + local topics = require 'services.http.topics' + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local warn_sub = root:subscribe(topics.obs_log('main', 'warn'), { queue_len = 10 }) + local info_sub = root:subscribe(topics.obs_log('main', 'info'), { queue_len = 10 }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + yield_many(4) + + local request_id = 'req-recovery' + svc._state.requests[request_id] = { + request_id = request_id, + target = { + method = 'GET', + host = '172.28.100.9', + path = '/cgi/get.cgi', + cmd = 'sys_cpumem', + response_parser = 'legacy-http1-close', + }, + } + local rec = { operation = 'exchange', request_id = request_id } + local ev = { status = 'failed', primary = 'timeout' } + + ok(svc:_record_http_failure(rec, ev)) + local first_failure = recv_payload(warn_sub, 'expected first failure log') + eq(first_failure.what, 'request_failed') + eq(first_failure.reason, 'timeout') + eq(first_failure.cmd, 'sys_cpumem') + + ok(svc:_record_http_failure(rec, ev)) + local saw_suppressed = false + for _, win in pairs(svc._obs.failure_windows) do + if (win.suppressed or 0) == 1 then saw_suppressed = true end + end + ok(saw_suppressed, 'second equivalent failure should be suppressed within the rate limit window') + + ok(svc:_record_http_recovery({ operation = 'exchange' })) + local recovery = recv_payload_where(info_sub, function (payload) return payload and payload.what == 'request_recovered' end, 'expected recovery log') + eq(recovery.what, 'request_recovered') + eq(recovery.operation, 'exchange') + ok(table_empty(svc._obs.failure_windows), 'recovery must clear failure suppression windows') + + ok(svc:_record_http_failure(rec, ev)) + local fresh_failure = recv_payload(warn_sub, 'expected post-recovery failure log') + eq(fresh_failure.what, 'request_failed') + eq(fresh_failure.reason, 'timeout') + + svc:terminate('done') + end) +end + +function M.test_non_local_handle_returning_call_is_rejected_before_backend_work() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local drv = fake_driver() + local svc = ok(http_service.open_handle(root, { driver = drv, id = 'main' })) + local remote = b:connect({ origin_base = { kind = 'local', link_id = 'fabric-link' } }) + local ref = sdk_mod.new_ref(remote, 'main') + local reply, err = fibers.perform(ref:listen_op({ host = '127.0.0.1', port = 0 })) + eq(reply, nil) + eq(err, 'not_local') + svc:terminate('done') + end) +end + + +function M.test_registry_callbacks_do_not_mutate_model_until_coordinator_receives_event() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + yield_many(4) + eq(svc:stats().active_listeners, 0) + eq(svc._emit, nil, 'legacy direct reducer ingress should not exist') + + local real_tx = svc._event_tx + local captured = {} + svc._event_tx = { + send_op = function (_, ev) + captured[#captured + 1] = ev + return fibers.always(true, nil) + end, + } + local id = ok(svc._registry:register('listener', { terminate = function () return true end }, { generation = svc._generation })) + eq(svc:stats().active_listeners, 0, 'registry callback must only enqueue an event') + eq(captured[1].kind, 'listener_registered') + svc._event_tx = real_tx + ok(svc:_submit_event(captured[1])) + yield_many(4) + eq(svc:stats().active_listeners, 1) + svc._registry:remove(id, 'done') + yield_many(4) + eq(svc:stats().active_listeners, 0) + svc:terminate('done') + end) +end + +function M.test_model_fields_are_recomputed_from_identity_records_and_duplicate_termination_is_idempotent() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + yield_many(4) + + ok(svc:_submit_event({ kind = 'listener_registered', handle_id = 'l1', generation = 1 })) + ok(svc:_submit_event({ kind = 'listener_registered', handle_id = 'l2', generation = 1 })) + ok(svc:_submit_event({ kind = 'listener_terminated', handle_id = 'l1', generation = 1 })) + ok(svc:_submit_event({ kind = 'listener_terminated', handle_id = 'l1', generation = 1 })) + yield_many(8) + eq(svc:stats().active_listeners, 1, 'duplicate termination must not decrement a derived count twice') + + ok(svc:_submit_event({ kind = 'exchange_registered', handle_id = 'e1', generation = 1 })) + ok(svc:_submit_event({ kind = 'websocket_registered', handle_id = 'w1', generation = 1 })) + ok(svc:_submit_event({ kind = 'context_admitted', listener_id = 'l2', context_id = 'c1', generation = 1 })) + yield_many(8) + local snap = svc:stats() + eq(snap.active_listeners, 1) + eq(snap.active_exchanges, 1) + eq(snap.active_websockets, 1) + eq(snap.active_contexts, 1) + svc:terminate('done') + end) +end + +function M.test_stale_generation_events_are_ignored() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + yield_many(4) + + ok(svc:_submit_event({ kind = 'listener_registered', handle_id = 'l1', generation = 1 })) + yield_many(4) + eq(svc:stats().active_listeners, 1) + ok(svc:_submit_event({ kind = 'listener_terminated', handle_id = 'l1', generation = 2 })) + yield_many(4) + eq(svc:stats().active_listeners, 1, 'stale handle generation must not terminate the live record') + + svc._state.operations.op_live = { operation_id = 'op_live', generation = 1, operation = 'exchange', state = 'running' } + ok(svc:_submit_event({ kind = 'http_operation_done', operation_id = 'op_live', operation = 'exchange', generation = 2, status = 'ok', result = {} })) + yield_many(4) + eq(svc:stats().completed_exchanges, 0, 'stale operation completion must not be counted') + svc:terminate('done') + end) +end + +function M.test_completed_exchange_prunes_one_shot_request_and_operation_records() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main', max_event_history = 16 })) + yield_many(4) + + svc._state.requests.req1 = { request_id = 'req1', state = 'running', target = { method = 'GET' } } + svc._state.operations.op1 = { operation_id = 'op1', generation = 1, operation = 'exchange', request_id = 'req1', state = 'running' } + ok(svc:_submit_event({ kind = 'http_operation_done', operation_id = 'op1', operation = 'exchange', request_id = 'req1', generation = 1, status = 'ok', result = { status = '200', headers = { large = 'header' } } })) + yield_many(8) + + eq(svc._state.requests.req1, nil) + eq(svc._state.operations.op1, nil) + eq(svc:stats().completed_exchanges, 1) + local last = svc:events()[#svc:events()] + eq(last.kind, 'http_operation_done') + eq(last.result, nil, 'event history should not retain operation result payloads') + svc:terminate('done') + end) +end + +function M.test_http_event_history_is_bounded() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main', max_event_history = 3 })) + for i = 1, 6 do svc:_log_event({ kind = 'synthetic', n = i, raw = { should = 'drop' } }) end + local events = svc:events() + eq(#events, 3) + eq(events[1].n, 4) + eq(events[3].n, 6) + eq(events[3].raw, nil) + svc:terminate('done') + end) +end + +function M.test_cap_request_received_then_service_shutdown_finalises_request_owner() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + local failed + local req = { + payload = {}, + origin = { kind = 'local' }, + reply = function () return true end, + fail = function (_, reason) failed = reason; return true end, + } + local request_id, owner = svc:_next_request_identity('status', req) + ok(svc:_submit_event({ kind = 'cap_request_received', verb = 'status', req = req, request_id = request_id, owner = owner, generation = svc._generation })) + svc:terminate('service_shutdown') + eq(failed, 'service_shutdown') + end) +end + +function M.test_event_queue_overflow_for_completion_fails_observing_service() + fibers.run(function (scope) + local child = ok(scope:child()) + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + ok(child:spawn(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main', event_queue_len = 1 })) + return fibers.perform(tx:send_op(svc)) + end)) + local svc = ok(fibers.perform(rx:recv_op())) + -- Replace the event sender with a would-block sender. This exercises the + -- same required-admission path as a full bounded queue without relying on + -- scheduler timing around a live coordinator. + svc._event_tx = { + send_op = function () return fibers.never() end, + } + local ok_report, err = svc:_submit_event({ kind = 'http_operation_done', operation_id = 'op-missing', generation = 1, status = 'ok' }, 'http_operation_done_report_failed', { fatal = true }) + eq(ok_report, nil) + ok(tostring(err):match('http_operation_done_report_failed'), 'overflow should use the completion report label') + ok(svc._event_admission_failure and tostring(svc._event_admission_failure):match('http_operation_done_report_failed')) + local st, _rep, primary = fibers.perform(child:join_op()) + eq(st, 'failed') + ok(tostring(primary):match('http_operation_done_report_failed')) + end) +end + + +function M.test_bus_request_abandonment_cancels_admitted_http_operation() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + yield_many(4) + + local done_tx, done_rx = mailbox.new(1, { full = 'reject_newest' }) + local req = { + payload = { method = 'GET', url = 'http://example.invalid/' }, + origin = { kind = 'local' }, + reply = function () error('abandoned request must not be replied to', 0) end, + fail = function () error('abandoned request must not be failed visibly', 0) end, + done_op = function () + return done_rx:recv_op():wrap(function (ev) + return ev.status, ev.value, ev.err + end) + end, + } + local request_id, owner = svc:_next_request_identity('exchange', req) + svc._state.requests[request_id] = { request_id = request_id, verb = 'exchange', owner = owner, state = 'received', generation = svc._generation } + + ok(operation_owner.start(svc, 'exchange', req, request_id, owner, function () + fibers.perform(sleep.sleep_op(10.0)) + return { ok = true } + end)) + + local operation_id + for id, rec in pairs(svc._state.operations) do + if rec.request_id == request_id then operation_id = id end + end + ok(operation_id, 'expected operation identity') + + ok(done_tx:send({ status = 'abandoned', err = 'caller_timeout' })) + yield_until(function () + return svc._state.operations[operation_id] == nil + end, 'operation should complete and be pruned after caller abandonment') + + eq(svc._state.requests[request_id], nil) + eq(svc._owned_requests[request_id], nil) + eq(svc:stats().failed_exchanges, 1) + ok(owner:done(), 'owner should be locally abandoned') + + svc:terminate('done') + end) +end + +function M.test_http_sdk_uses_compositional_timeout_by_default() + local seen_opts + local ref = sdk_mod.new_ref({ + call_op = function (_, _topic, _args, opts) + seen_opts = opts + return fibers.always('ok') + end, + }, 'main') + + fibers.run(function () + local v = fibers.perform(ref:exchange_op({ method = 'GET', url = 'http://example.invalid/' })) + eq(v, 'ok') + end) + + ok(seen_opts, 'expected SDK to pass call opts') + eq(seen_opts.timeout, false) +end + +function M.test_service_shutdown_terminates_registry_handles_and_finalises_unresolved_requests() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + local terminated + local id = ok(svc._registry:register('exchange', { terminate = function (_, reason) terminated = reason; return true end }, { generation = svc._generation })) + yield_many(4) + local failed + local req = { fail = function (_, reason) failed = reason; return true end } + local request_id, owner = svc:_next_request_identity('exchange', req) + svc._state.requests[request_id] = { request_id = request_id, verb = 'exchange', owner = owner, state = 'received' } + svc:terminate('shutdown') + eq(terminated, 'shutdown') + eq(failed, 'shutdown') + eq(svc._registry:get(id), nil) + end) +end + + +function M.test_context_transfer_keeps_context_active_after_listener_termination_until_context_terminates() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + yield_many(4) + ok(svc:_submit_event({ kind = 'listener_registered', handle_id = 'l1', generation = 1 })) + ok(svc:_submit_event({ kind = 'context_admitted', listener_id = 'l1', context_id = 'c1', generation = 1 })) + ok(svc:_submit_event({ kind = 'context_transferred', listener_id = 'l1', context_id = 'c1', generation = 1 })) + ok(svc:_submit_event({ kind = 'listener_terminated', handle_id = 'l1', generation = 1, reason = 'listener_closed' })) + yield_many(8) + local snap = svc:stats() + eq(snap.active_listeners, 0) + eq(snap.active_contexts, 1, 'transferred context should remain active after listener termination') + ok(svc:_submit_event({ kind = 'context_terminated', listener_id = 'l1', context_id = 'c1', generation = 1, reason = 'request_done' })) + yield_many(4) + eq(svc:stats().active_contexts, 0) + svc:terminate('done') + end) +end + +local function fake_polling_driver() + local cond = require 'fibers.cond' + local changed = cond.new() + local drv = { + started = false, + terminated = nil, + } + function drv:start() self.started = true; return true end + function drv:terminate(reason) + self.terminated = reason or true + changed:signal() + return true + end + function drv:run_op(_, fn) return fibers.guard(function () return fibers.always(fn()) end) end + function drv:pollable_step_op() + return changed:wait_op():wrap(function () + if self.terminated then return nil, self.terminated end + return true + end) + end + function drv:poke() changed:signal(); return true end + return drv +end + +local function fake_headers(status) + return { + get = function (_, name) if name == ':status' then return tostring(status or 200) end end, + each = function () + local rows = { { ':status', tostring(status or 200) }, { 'content-type', 'text/plain' } } + local i = 0 + return function () + i = i + 1 + local row = rows[i] + if row then return row[1], row[2] end + end + end, + } +end + +local function request_module(body_text, counters) + counters = counters or {} + return { + new_from_uri = function (uri) + counters.request_new_from_uri = (counters.request_new_from_uri or 0) + 1 + return { + uri = uri, + headers = { + set = {}, + upsert = function (self, k, v) self.set[k] = v end, + append = function (self, k, v) self.set[k] = v end, + }, + go = function (self) + local chunks = { body_text or ('body for ' .. self.uri) } + local i = 0 + return fake_headers(204), { + get_next_chunk = function () i = i + 1; return chunks[i] end, + shutdown = function () return true end, + } + end, + } + end, + } +end + +local function websocket_module(counters) + counters = counters or {} + return { + new_from_uri = function (uri) + counters.websocket_new_from_uri = (counters.websocket_new_from_uri or 0) + 1 + return { + uri = uri, + connect = function (self) self.connected = true; return true end, + send = function () return true end, + receive = function () return nil, 'closed' end, + close = function () return true end, + } + end, + } +end + +local function http_server_success(counters) + counters = counters or {} + return { + listen = function (opts) + counters.server_listen_constructed = (counters.server_listen_constructed or 0) + 1 + return { + _opts = opts, + listen = function (self) + self.listen_called = true + counters.server_listen_called = (counters.server_listen_called or 0) + 1 + return true + end, + step = function () return true end, + close = function (self) + self.closed = true + counters.server_closed = (counters.server_closed or 0) + 1 + return true + end, + localname = function () return '127.0.0.1', 12345 end, + } + end, + } +end + + +local function fake_stream_for_context() + return { + terminated = false, + calls = {}, + shutdown = function (self) self.terminated = true; return true end, + get_headers = function () return { ':method', 'GET' } end, + get_next_chunk = function () return nil end, + write_headers = function () return true end, + write_chunk = function () return true end, + } +end + +local function event_seen(events, kind, pred) + for _, ev in ipairs(events or {}) do + if ev.kind == kind and (not pred or pred(ev)) then return ev end + end + return nil +end + +function M.test_backend_failure_terminates_service_owned_handles() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + yield_many(4) + + local terminated = {} + local function handle(name) + return { + terminate = function (_, reason) + terminated[name] = reason or true + return true + end, + close_op = function () error('graceful close_op must not be used by registry termination', 0) end, + } + end + + svc._registry:register('listener', handle('listener'), { id = 'l1', generation = svc._generation }) + svc._registry:register('exchange', handle('exchange'), { id = 'e1', generation = svc._generation }) + svc._registry:register('websocket', handle('websocket'), { id = 'w1', generation = svc._generation }) + yield_many(8) + local before = svc:stats() + eq(before.active_listeners, 1) + eq(before.active_exchanges, 1) + eq(before.active_websockets, 1) + + ok(svc:_submit_event({ kind = 'backend_failed', reason = 'backend_boom', generation = svc._generation })) + yield_many(12) + local after = svc:stats() + eq(after.backend, 'failed') + eq(after.ready, false) + eq(after.last_error, 'backend_boom') + eq(after.active_listeners, 0) + eq(after.active_exchanges, 0) + eq(after.active_websockets, 0) + eq(terminated.listener, 'backend_boom') + eq(terminated.exchange, 'backend_boom') + eq(terminated.websocket, 'backend_boom') + svc:terminate('done') + end) +end + +function M.test_listener_setup_failure_leaves_no_live_ownership() + fibers.run(function () + local counters = {} + local failing_server = { + listen = function () + counters.constructed = (counters.constructed or 0) + 1 + return { + listen = function () counters.listen_called = (counters.listen_called or 0) + 1; return nil, 'listen_failed' end, + close = function () counters.closed = (counters.closed or 0) + 1; return true end, + step = function () return true end, + } + end, + } + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_polling_driver(), id = 'main', http_server = failing_server })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local ref = sdk_mod.new_ref(user, 'main') + + local reply, err = fibers.perform(ref:listen_op({ host = '127.0.0.1', port = 0 })) + eq(reply, nil) + eq(err, 'listen_failed') + yield_many(12) + eq(counters.constructed, 1) + eq(counters.listen_called, 1) + eq(counters.closed, 1) + eq(svc:stats().active_listeners, 0) + eq(next(svc._registry:snapshot()), nil, 'failed listener setup must leave no active registry record') + svc:terminate('done') + end) +end + +function M.test_handle_returning_rpcs_return_public_wrappers() + fibers.run(function () + local counters = {} + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = fake_polling_driver(), + id = 'main', + http_server = http_server_success(counters), + request_module = request_module('body', counters), + websocket_module = websocket_module(counters), + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local ref = sdk_mod.new_ref(user, 'main') + + local listen_reply = ok(fibers.perform(ref:listen_op({ host = '127.0.0.1', port = 0 }))) + local exchange_reply = ok(fibers.perform(ref:open_exchange_op({ uri = 'http://example.test/', method = 'GET' }))) + local websocket_reply = ok(fibers.perform(ref:connect_ws_op({ uri = 'ws://example.test/ws' }))) + + local listener_mod = require 'services.http.listener' + local exchange_mod = require 'services.http.exchange' + local websocket_mod = require 'services.http.websocket' + eq(getmetatable(listen_reply.listener), listener_mod.HttpListener, 'listen should return public HttpListener wrapper') + eq(getmetatable(exchange_reply.exchange), exchange_mod.HttpExchange, 'open_exchange should return public HttpExchange wrapper') + eq(getmetatable(websocket_reply.websocket), websocket_mod.ClientWebSocket, 'connect_ws should return public ClientWebSocket wrapper') + ok(not tostring(getmetatable(listen_reply.listener)):match('transport'), 'listener wrapper should not expose transport type') + ok(not tostring(getmetatable(exchange_reply.exchange)):match('transport'), 'exchange wrapper should not expose transport type') + ok(not tostring(getmetatable(websocket_reply.websocket)):match('transport'), 'websocket wrapper should not expose transport type') + svc:terminate('done') + end) +end + +function M.test_each_local_handle_rpc_rejects_non_local_before_backend_admission() + fibers.run(function () + local counters = {} + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = fake_polling_driver(), + id = 'main', + http_server = http_server_success(counters), + request_module = request_module('body', counters), + websocket_module = websocket_module(counters), + })) + local remote = b:connect({ origin_base = { kind = 'local', link_id = 'fabric-link' } }) + local ref = sdk_mod.new_ref(remote, 'main') + + local listen_reply, listen_err = fibers.perform(ref:listen_op({ host = '127.0.0.1', port = 0 })) + local exchange_reply, exchange_err = fibers.perform(ref:open_exchange_op({ uri = 'http://example.test/', method = 'GET' })) + local ws_reply, ws_err = fibers.perform(ref:connect_ws_op({ uri = 'ws://example.test/ws' })) + eq(listen_reply, nil) + eq(exchange_reply, nil) + eq(ws_reply, nil) + eq(listen_err, 'not_local') + eq(exchange_err, 'not_local') + eq(ws_err, 'not_local') + eq(counters.server_listen_constructed or 0, 0, 'non-local listen must reject before backend admission') + eq(counters.request_new_from_uri or 0, 0, 'non-local open_exchange must reject before request construction') + eq(counters.websocket_new_from_uri or 0, 0, 'non-local connect_ws must reject before websocket construction') + eq(svc:stats().rejected_requests, 3) + svc:terminate('done') + end) +end + + +function M.test_initial_retained_status_is_not_available_before_backend_ready_event_is_reduced() + fibers.run(function () + local topics = require 'services.http.topics' + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + + local view = root:retained_view(topics.status('main')) + local msg = ok(view:get(topics.status('main')), 'status should be retained immediately') + eq(msg.payload.available, false, 'cap status must not advertise availability before backend_ready is reduced') + eq(msg.payload.state, 'starting') + svc:terminate('done') + end) +end + +function M.test_listener_owner_reports_identity_completion_when_listener_runtime_ends() + fibers.run(function () + local counters = {} + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = fake_polling_driver(), + id = 'main', + http_server = http_server_success(counters), + max_event_history = 32, + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local ref = sdk_mod.new_ref(user, 'main') + + local rep = ok(fibers.perform(ref:listen_op({ host = '127.0.0.1', port = 0 }))) + local listener = ok(rep.listener) + local handle_id = ok(rep.handle_id) + listener:terminate('listener_done_for_test') + yield_many(12) + + local saw_done = false + for _, ev in ipairs(svc:events()) do + if ev.kind == 'listener_done' and ev.handle_id == handle_id then + saw_done = true + eq(ev.generation, svc._generation) + eq(ev.status, 'ok') + end + end + ok(saw_done, 'listener owner scope should report an identity-bearing completion') + svc:terminate('done') + end) +end + + +function M.test_onstream_context_admission_emits_service_event_without_reducing_in_callback() + fibers.run(function () + local counters = {} + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = fake_polling_driver(), + id = 'main', + http_server = http_server_success(counters), + max_event_history = 32, + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local ref = sdk_mod.new_ref(user, 'main') + local rep = ok(fibers.perform(ref:listen_op({ host = '127.0.0.1', port = 0 }))) + local listener = ok(rep.listener) + local raw_listener = ok(listener:_raw_listener()) + local raw_ctx = lua_http._new_context_for_test(raw_listener, raw_listener:_raw_server_for_test(), fake_stream_for_context()) + + local admitted, err = raw_listener:_admit_context(raw_ctx) + eq(admitted, true) + eq(err, nil) + eq(svc:stats().active_contexts, 0, 'onstream admission must enqueue, not run the service reducer synchronously') + + yield_many(8) + eq(svc:stats().active_contexts, 1, 'queued, unaccepted context should be counted as listener-owned after coordinator observes admission') + ok(event_seen(svc:events(), 'context_admitted', function (ev) + return ev.listener_id == rep.handle_id and ev.context_id == raw_ctx:id() + end), 'context_admitted event should carry listener/context identity') + ok(not event_seen(svc:events(), 'context_transferred', function (ev) + return ev.context_id == raw_ctx:id() + end), 'onstream admission must not imply caller ownership transfer') + svc:terminate('done') + end) +end + +function M.test_unaccepted_context_is_listener_owned_until_accept() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = fake_polling_driver(), + id = 'main', + http_server = http_server_success({}), + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local rep = ok(fibers.perform(sdk_mod.new_ref(user, 'main'):listen_op({ host = '127.0.0.1', port = 0 }))) + local raw_listener = rep.listener:_raw_listener() + local raw_ctx = lua_http._new_context_for_test(raw_listener, raw_listener:_raw_server_for_test(), fake_stream_for_context()) + ok(raw_listener:_admit_context(raw_ctx)) + yield_many(8) + + local snap = svc:stats() + eq(snap.active_contexts, 1) + local rec = snap.handles['ctx' .. tostring(raw_ctx:id())] + ok(rec, 'unaccepted context should have a registry record') + eq(rec.kind, 'context') + eq(rec.owner, 'listener') + eq(rec.state, 'registered') + + local accepted = ok(fibers.perform(rep.listener:accept_op())) + eq(accepted:id(), raw_ctx:id()) + yield_many(8) + local after = svc:stats().handles['ctx' .. tostring(raw_ctx:id())] + ok(after, 'accepted context should remain registered as a service shutdown backstop') + eq(after.owner, 'caller_after_handoff') + eq(after.state, 'transferred') + svc:terminate('done') + end) +end + +function M.test_listener_termination_terminates_unaccepted_context_and_emits_context_terminated() + fibers.run(function () + local terminated_reason + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = fake_polling_driver(), + id = 'main', + http_server = http_server_success({}), + context_terminator = function (_, reason) terminated_reason = reason end, + max_event_history = 32, + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local rep = ok(fibers.perform(sdk_mod.new_ref(user, 'main'):listen_op({ host = '127.0.0.1', port = 0 }))) + local raw_listener = rep.listener:_raw_listener() + local raw_ctx = lua_http._new_context_for_test(raw_listener, raw_listener:_raw_server_for_test(), fake_stream_for_context()) + ok(raw_listener:_admit_context(raw_ctx)) + yield_many(8) + + rep.listener:terminate('listener_closed') + yield_many(12) + ok(raw_ctx:is_closed(), 'unaccepted context should be terminated with its listener') + eq(terminated_reason, 'listener_closed') + ok(event_seen(svc:events(), 'context_terminated', function (ev) + return ev.context_id == raw_ctx:id() and ev.reason == 'listener_closed' + end), 'listener-owned context termination should emit context_terminated') + eq(svc:stats().active_contexts, 0) + svc:terminate('done') + end) +end + +function M.test_accepted_context_survives_listener_termination_until_context_terminates() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = fake_polling_driver(), + id = 'main', + http_server = http_server_success({}), + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local rep = ok(fibers.perform(sdk_mod.new_ref(user, 'main'):listen_op({ host = '127.0.0.1', port = 0 }))) + local raw_listener = rep.listener:_raw_listener() + local raw_ctx = lua_http._new_context_for_test(raw_listener, raw_listener:_raw_server_for_test(), fake_stream_for_context()) + ok(raw_listener:_admit_context(raw_ctx)) + local ctx = ok(fibers.perform(rep.listener:accept_op())) + yield_many(8) + + rep.listener:terminate('listener_closed') + yield_many(8) + ok(not ctx:is_closed(), 'accepted context should be owned by caller/request scope after accept') + eq(svc:stats().active_contexts, 1) + ctx:terminate('request_done') + yield_many(8) + eq(svc:stats().active_contexts, 0) + svc:terminate('done') + end) +end + +function M.test_cancel_listen_start_while_waiting_for_ready_cleans_up_setup_listener() + fibers.run(function (scope) + local closed_reason + local server = { + listen = function () return true end, + close = function (_, reason) closed_reason = reason or true; return true end, + step = function () return true end, + } + local drv = fake_driver() + drv.run_op = function () return fibers.never() end + local lifetime = ok(scope:child()) + local caller = ok(scope:child()) + local result, err + ok(caller:spawn(function () + result, err = listener_owner.start({ + lifetime_scope = lifetime, + listen_opts = { driver = drv, server = server }, + handle_id = 'listener-pending', + generation = 1, + }) + end)) + yield_many(8) + caller:cancel('caller_cancelled') + yield_many(12) + eq(result, nil) + ok(closed_reason ~= nil, 'cancelled listen setup should terminate the unreturned listener immediately') + lifetime:cancel('done') + end) +end + +function M.test_server_side_websocket_upgrade_registers_and_deregisters_service_websocket_record() + fibers.run(function (scope) + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = fake_polling_driver(), + id = 'main', + http_server = http_server_success({}), + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local rep = ok(fibers.perform(sdk_mod.new_ref(user, 'main'):listen_op({ host = '127.0.0.1', port = 0 }))) + local raw_listener = rep.listener:_raw_listener() + local raw_ctx = lua_http._new_context_for_test(raw_listener, raw_listener:_raw_server_for_test(), fake_stream_for_context()) + ok(raw_listener:_admit_context(raw_ctx)) + local ctx = ok(fibers.perform(rep.listener:accept_op())) + local fake_raw_ws = { + accept = function () return true end, + receive = function () return nil, 'closed' end, + send = function () return true end, + close = function () return true end, + } + local module = { new_from_stream = function () return fake_raw_ws end } + local ws, werr + ok(scope:spawn(function () + ws, werr = fibers.perform(public_ws.from_context_op(ctx, {}, { websocket_module = module })) + end)) + yield_many(3) + ok(raw_ctx:_drain_one_for_test()) + yield_many(8) + ok(ws, werr or 'server websocket should be returned') + eq(svc:stats().active_websockets, 1, 'server-side websocket upgrade should be visible in service summary') + ws:terminate('ws_done') + yield_many(8) + eq(svc:stats().active_websockets, 0) + svc:terminate('done') + end) +end + +function M.test_service_shutdown_terminates_server_side_upgraded_websocket() + fibers.run(function (scope) + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = fake_polling_driver(), + id = 'main', + http_server = http_server_success({}), + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local rep = ok(fibers.perform(sdk_mod.new_ref(user, 'main'):listen_op({ host = '127.0.0.1', port = 0 }))) + local raw_listener = rep.listener:_raw_listener() + local raw_ctx = lua_http._new_context_for_test(raw_listener, raw_listener:_raw_server_for_test(), fake_stream_for_context()) + ok(raw_listener:_admit_context(raw_ctx)) + local ctx = ok(fibers.perform(rep.listener:accept_op())) + local fake_raw_ws = { close = function () return true end, receive = function () return nil, 'closed' end, send = function () return true end } + local module = { new_from_stream = function () return fake_raw_ws end } + local ws + ok(scope:spawn(function () ws = fibers.perform(public_ws.from_context_op(ctx, {}, { websocket_module = module })) end)) + yield_many(3) + ok(raw_ctx:_drain_one_for_test()) + yield_many(8) + ok(ws) + svc:terminate('service_shutdown') + yield_many(4) + ok(ws:is_closed(), 'service shutdown should terminate registered server-side websocket transport handles') + eq(ws:why(), 'service_shutdown') + end) +end + + +function M.test_public_context_read_cancellation_terminates_context_and_service_record() + fibers.run(function (scope) + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = fake_polling_driver(), + id = 'main', + http_server = http_server_success({}), + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local rep = ok(fibers.perform(sdk_mod.new_ref(user, 'main'):listen_op({ host = '127.0.0.1', port = 0 }))) + local raw_listener = rep.listener:_raw_listener() + local stream = fake_stream_for_context() + local read_active = false + stream.get_next_chunk = function (self) + read_active = true + while not self.terminated do runtime.yield() end + return nil, 'closed' + end + local raw_ctx = lua_http._new_context_for_test(raw_listener, raw_listener:_raw_server_for_test(), stream) + ok(raw_listener:_admit_context(raw_ctx)) + local ctx = ok(fibers.perform(rep.listener:accept_op())) + yield_many(8) + eq(svc:stats().active_contexts, 1) + + local waiter = ok(scope:child()) + ok(waiter:spawn(function () fibers.perform(ctx:read_chunk_op(1024)) end)) + local runner = ok(scope:child()) + ok(runner:spawn(function () raw_ctx:_drain_one_for_test() end)) + yield_until(function () return read_active end, 'public context read should be active') + + waiter:cancel('caller_cancelled') + fibers.perform(waiter:join_op()) + yield_until(function () return svc:stats().active_contexts == 0 end, + 'context termination should be reported through service-owned events') + ok(ctx:is_closed(), 'cancelling a public context read should terminate the context owner') + eq(ctx:why(), 'aborted') + runner:cancel('done') + fibers.perform(runner:join_op()) + svc:terminate('done') + end) +end + +function M.test_public_server_websocket_receive_cancellation_terminates_websocket_and_registry_record() + fibers.run(function (scope) + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = fake_polling_driver(), + id = 'main', + http_server = http_server_success({}), + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local rep = ok(fibers.perform(sdk_mod.new_ref(user, 'main'):listen_op({ host = '127.0.0.1', port = 0 }))) + local raw_listener = rep.listener:_raw_listener() + local raw_ctx = lua_http._new_context_for_test(raw_listener, raw_listener:_raw_server_for_test(), fake_stream_for_context()) + ok(raw_listener:_admit_context(raw_ctx)) + local ctx = ok(fibers.perform(rep.listener:accept_op())) + + local receive_active = false + local fake_raw_ws = { + accept = function () return true end, + receive = function () + receive_active = true + while not raw_ctx:is_closed() do runtime.yield() end + return nil, 'closed' + end, + send = function () return true end, + close = function (self, _code, reason) self.closed_reason = reason or true; return true end, + } + local module = { new_from_stream = function () return fake_raw_ws end } + local ws + ok(scope:spawn(function () ws = fibers.perform(public_ws.from_context_op(ctx, {}, { websocket_module = module })) end)) + yield_many(3) + ok(raw_ctx:_drain_one_for_test()) + yield_many(8) + ok(ws, 'server websocket should be returned before receive test') + eq(svc:stats().active_websockets, 1) + + local waiter = ok(scope:child()) + ok(waiter:spawn(function () fibers.perform(ws:receive_op()) end)) + local runner = ok(scope:child()) + ok(runner:spawn(function () raw_ctx:_drain_one_for_test() end)) + yield_until(function () return receive_active end, 'public websocket receive should be active') + + waiter:cancel('caller_cancelled') + fibers.perform(waiter:join_op()) + yield_until(function () return svc:stats().active_websockets == 0 end, + 'websocket termination should deregister the service record') + ok(ws:is_closed(), 'cancelling a public websocket receive should terminate the websocket') + eq(ws:why(), 'aborted') + runner:cancel('done') + fibers.perform(runner:join_op()) + svc:terminate('done') + end) +end + +function M.test_public_open_exchange_handle_read_cancellation_keeps_service_backend_usable() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + local first_stream + local first_read_active = false + local request_mod = { + new_from_uri = function (uri) + return { + uri = uri, + headers = { + upsert = function () end, + append = function () end, + }, + go = function (self) + local headers = fake_headers(200) + if tostring(self.uri):match('slow') then + first_stream = { + terminated = false, + release_read = false, + get_next_chunk = function (stream) + first_read_active = true + while not stream.release_read do runtime.yield() end + return nil, 'closed' + end, + shutdown = function (stream) stream.terminated = true; return true end, + } + return headers, first_stream + end + local chunks = { 'ok' } + local i = 0 + return headers, { + get_next_chunk = function () i = i + 1; return chunks[i] end, + shutdown = function () return true end, + } + end, + shutdown = function () return true end, + } + end, + } + + fibers.run(function (scope) + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local svc = ok(http_service.open_handle(root, { + driver = drv, + id = 'main', + request_module = request_mod, + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local ref = sdk_mod.new_ref(user, 'main') + + local rep = ok(fibers.perform(ref:open_exchange_op({ uri = 'http://example.test/slow', method = 'GET' }))) + local ex = ok(rep.exchange) + yield_many(8) + eq(svc:stats().active_exchanges, 1) + + local waiter = ok(scope:child()) + ok(waiter:spawn(function () fibers.perform(ex:read_chunk_op(1024)) end)) + yield_until(function () return first_read_active end, 'slow exchange read should be active') + -- The driver pump runs the active read job. Once the caller loses interest, + -- the public exchange handle should detach and close for the caller, but the + -- lua-http/cqueues-owned stream cleanup must wait until the cqueues job + -- returns. The fake read is therefore released explicitly after caller + -- cancellation, modelling a cqueues-side timeout/error return. + waiter:cancel('caller_cancelled') + fibers.perform(waiter:join_op()) + yield_until(function () return svc:stats().active_exchanges == 0 end, + 'exchange termination should deregister the service record') + + ok(not first_stream.terminated, 'caller cancellation must not synchronously terminate cqueues-owned stream state') + ok(ex:is_closed(), 'public exchange handle should be closed after active read abort') + first_stream.release_read = true + yield_until(function () return first_stream.terminated end, + 'detached read cleanup should terminate the stream after the cqueues job returns') + ok(not drv:is_closed(), 'the exchange is the narrow owner; the HTTP backend should stay usable') + + local second = ok(fibers.perform(ref:exchange_op({ uri = 'http://example.test/fast', method = 'GET' }))) + eq(second.result.status, '200') + eq(svc:stats().completed_exchanges, 1, 'a subsequent public exchange should complete after active read cancellation') + svc:terminate('done') + end) +end + +function M.test_retained_http_config_updates_policy_generation_and_policy() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + root:retain({ 'cfg', 'http' }, { + rev = 7, + data = { + schema = require('services.http.config').SCHEMA, + policy = { allow_loopback = false, allowed_schemes = { https = true } }, + }, + }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + yield_until(function () return svc:stats().policy_generation == 7 end, 'retained cfg/http was not applied') + eq(svc._opts.policy.allow_loopback, false) + eq(svc._opts.policy.allowed_schemes.https, true) + eq(svc._opts.policy.allowed_schemes.http, nil) + svc:terminate('done') + end) +end + +function M.test_invalid_retained_http_config_marks_service_degraded_without_replacing_policy() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + root:retain({ 'cfg', 'http' }, { rev = 3, data = { schema = 'wrong' } }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + yield_until(function () return svc:stats().last_error ~= nil end, 'invalid cfg/http was not reported') + ok(tostring(svc:stats().last_error):find('schema', 1, true)) + eq(svc:stats().policy_generation, 1, 'invalid config must not advance policy generation') + svc:terminate('done') + end) +end + +function M.test_refresh_model_does_not_publish_when_snapshot_is_unchanged() + local model_mod = require 'services.http.model' + local conn = { retains = {}, retain = function(self, topic, payload) self.retains[#self.retains + 1] = { topic = topic, payload = payload }; return true end } + local svc = setmetatable({ + _conn = conn, + _id = 'main', + _closed = false, + _model = model_mod.new(), + _opts = {}, + _obs = { last_status_key = nil, last_status_emit_at = nil, failure_windows = {}, had_failure = false }, + _state = { + service_state = 'starting', backend = 'starting', ready = false, last_error = nil, + policy_generation = 1, completed_exchanges = 0, failed_exchanges = 0, rejected_requests = 0, + listeners = {}, contexts = {}, exchanges = {}, websockets = {}, config = { observability = { status_interval_s = 30, stats_interval_s = 30 } }, + }, + }, http_service.HttpService) + ok(svc:_refresh_model({ kind = 'noop' })) + eq(#conn.retains, 0) +end + +function M.test_http_retained_stats_publish_for_handle_activity_without_status_retain() + local model_mod = require 'services.http.model' + local topics = require 'services.http.topics' + local conn = { retains = {}, retain = function(self, topic, payload) self.retains[#self.retains + 1] = { topic = table.concat(topic, '/'), payload = payload }; return true end } + local svc = setmetatable({ + _conn = conn, + _id = 'main', + _closed = false, + _model = model_mod.new(), + _opts = {}, + _obs = { last_status_key = nil, last_status_emit_at = nil, failure_windows = {}, had_failure = false }, + _retained_status_cache = {}, + _retained_stats_cache = {}, + _state = { + service_state = 'ready', backend = 'ready', ready = true, last_error = nil, + policy_generation = 1, completed_exchanges = 0, failed_exchanges = 0, rejected_requests = 0, + listeners = {}, contexts = {}, exchanges = {}, websockets = {}, config = { observability = { status_interval_s = 30, stats_interval_s = 30 } }, + }, + }, http_service.HttpService) + ok(svc:_refresh_model({ kind = 'backend_ready' })) + local status_topic = table.concat(topics.status('main'), '/') + local stats_topic = table.concat(topics.state('main', 'stats'), '/') + local status_count = 0 + for _, rec in ipairs(conn.retains) do if rec.topic == status_topic then status_count = status_count + 1 end end + local before_retains = #conn.retains + svc._state.listeners.l1 = { state = 'registered' } + ok(svc:_refresh_model({ kind = 'listener_started' })) + local saw_stats = false + for i = before_retains + 1, #conn.retains do + if conn.retains[i].topic == stats_topic then saw_stats = true end + end + ok(saw_stats, 'active listener change should refresh retained stats promptly') + local after_status_count = 0 + for _, rec in ipairs(conn.retains) do if rec.topic == status_topic then after_status_count = after_status_count + 1 end end + eq(after_status_count, status_count, 'active listener change must not republish cap status') +end + +function M.test_http_retained_status_is_not_republished_for_stats_only_changes() + local model_mod = require 'services.http.model' + local topics = require 'services.http.topics' + local conn = { retains = {}, retain = function(self, topic, payload) self.retains[#self.retains + 1] = { topic = table.concat(topic, '/'), payload = payload }; return true end } + local svc = setmetatable({ + _conn = conn, + _id = 'main', + _closed = false, + _model = model_mod.new(), + _opts = {}, + _obs = { last_status_key = nil, last_status_emit_at = nil, failure_windows = {}, had_failure = false }, + _retained_status_cache = {}, + _retained_stats_cache = {}, + _state = { + service_state = 'ready', backend = 'ready', ready = true, last_error = nil, + policy_generation = 1, completed_exchanges = 0, failed_exchanges = 0, rejected_requests = 0, + listeners = {}, contexts = {}, exchanges = {}, websockets = {}, config = { observability = { status_interval_s = 30, stats_interval_s = 30 } }, + }, + }, http_service.HttpService) + ok(svc:_refresh_model({ kind = 'backend_ready' })) + local first_count = #conn.retains + ok(first_count >= 2, 'initial semantic change should publish status and stats') + svc._state.completed_exchanges = 1 + ok(svc:_refresh_model({ kind = 'http_operation_done', status = 'ok' })) + eq(#conn.retains, first_count, 'stats-only changes should not republish retained cap state immediately') + svc._retained_stats_emit_at = -1000000 + svc._state.completed_exchanges = 2 + ok(svc:_refresh_model({ kind = 'http_operation_done', status = 'ok' })) + eq(conn.retains[#conn.retains].topic, table.concat(topics.state('main', 'stats'), '/')) +end + +return M diff --git a/tests/unit/http/test_service_architecture.lua b/tests/unit/http/test_service_architecture.lua new file mode 100644 index 00000000..508c817b --- /dev/null +++ b/tests/unit/http/test_service_architecture.lua @@ -0,0 +1,135 @@ +local fibers = require 'fibers' +local runtime = require 'fibers.runtime' +local bus = require 'bus' + +local http_service = require 'services.http.service' +local sdk_mod = require 'services.http.sdk' +local blob = require 'devicecode.blob_source' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +local function yield_many(n) + for _ = 1, n do runtime.yield() end +end + +local function fake_driver() + return { + started = false, + terminated = nil, + start = function (self) self.started = true; return true end, + terminate = function (self, reason) self.terminated = reason or true; return true end, + run_op = function (_, _, fn) return fibers.guard(function () return fibers.always(fn()) end) end, + } +end + +local function failing_run_driver() + return { + terminated = nil, + run = function () return nil, 'backend_boom' end, + terminate = function (self, reason) self.terminated = reason or true; return true end, + run_op = function (_, _, fn) return fibers.guard(function () return fibers.always(fn()) end) end, + } +end + +local function fake_headers(status) + return { + get = function (_, name) if name == ':status' then return tostring(status or 200) end end, + each = function () + local rows = { { ':status', tostring(status or 200) }, { 'content-type', 'text/plain' } } + local i = 0 + return function () + i = i + 1 + local row = rows[i] + if row then return row[1], row[2] end + end + end, + } +end + +local function request_module(body) + return { + new_from_uri = function (uri) + return { + uri = uri, + headers = { + set = {}, + upsert = function (self, k, v) self.set[k] = v end, + append = function (self, k, v) self.set[k] = v end, + }, + go = function () + local chunks = { body or 'ok' } + local i = 0 + return fake_headers(200), { + get_next_chunk = function () i = i + 1; return chunks[i] end, + shutdown = function () return true end, + } + end, + } + end, + } +end + +function M.test_backend_run_is_named_component_with_identity_completion() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local drv = failing_run_driver() + local svc = ok(http_service.open_handle(root, { driver = drv, id = 'main', max_event_history = 32 })) + yield_many(10) + local done + for _, ev in ipairs(svc:events()) do + if ev.kind == 'backend_done' then done = ev end + end + ok(done, 'backend_done completion should exist') + eq(done.component, 'backend') + eq(done.component_id, 'http_backend') + eq(done.status, 'failed') + eq(done.primary, 'backend_boom') + local snap = svc:stats() + eq(snap.backend, 'failed') + eq(snap.last_error, 'backend_boom') + svc:terminate('done') + end) +end + +function M.test_exchange_is_named_scoped_work_with_identity_completion_and_event_driven_summary() + fibers.run(function () + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local sink = blob.to_memory() + local svc = ok(http_service.open_handle(root, { + driver = fake_driver(), + id = 'main', + request_module = request_module('hello'), + max_event_history = 32, + })) + local user = b:connect({ origin_base = { kind = 'local' } }) + local ref = sdk_mod.new_ref(user, 'main') + local rep = ok(fibers.perform(ref:exchange_op({ uri = 'http://example.test/', method = 'GET', response_sink = sink }))) + eq(rep.result.status, '200') + eq(rep.result.body, nil) + eq(rep.result.response_sink.bytes, 5) + eq(sink:result(), 'hello') + yield_many(8) + local events = svc:events() + local started, done + for _, ev in ipairs(events) do + if ev.kind == 'operation_started' and ev.operation == 'exchange' then started = ev end + if ev.kind == 'http_operation_done' and ev.operation == 'exchange' then done = ev end + end + ok(started, 'operation_started event should exist') + ok(done, 'identity-bearing completion should exist') + eq(done.operation_id, started.operation_id) + eq(done.status, 'ok') + local status = ok(fibers.perform(ref:status_op())) + eq(status.status.completed_exchanges, 1) + svc:terminate('done') + end) +end + +return M diff --git a/tests/unit/http/test_structure.lua b/tests/unit/http/test_structure.lua new file mode 100644 index 00000000..4fd3e200 --- /dev/null +++ b/tests/unit/http/test_structure.lua @@ -0,0 +1,116 @@ +local M = {} + +local function ok(v, msg) + if not v then error(msg or 'assertion failed', 2) end +end + +local function read_file(path) + local f = assert(io.open(path, 'r')) + local s = f:read('*a') + f:close() + return s +end + +function M.test_only_http_transport_requires_lua_http_or_cqueues() + local p = io.popen("find ../src/services/http -name '*.lua' | sort") + for file in p:lines() do + local s = read_file(file) + local has_backend_require = s:find("require 'cqueues", 1, true) + or s:find("require 'http.", 1, true) + or s:find("pcall(require, 'cqueues", 1, true) + or s:find("pcall(require, 'http.", 1, true) + if has_backend_require then + ok(file:find('/transport/', 1, true), 'backend require outside transport: ' .. file) + end + end + p:close() +end + + +function M.test_http_sources_use_termination_vocabulary() + local cmd = [[find ../src/services/http ../src/services/ui \( -name '*.lua' -o -name '*.md' \) | sort]] + local p = assert(io.popen(cmd)) + + for file in p:lines() do + local s = read_file(file) + local old_terminate = 'terminate' .. '_now' + local old_close = 'close' .. '_now' + + ok(not s:find(old_terminate, 1, true), old_terminate .. ' used in ' .. file) + ok(not s:find(old_close, 1, true), old_close .. ' used in ' .. file) + end + + p:close() +end + + +function M.test_http_service_wires_components_with_service_event_ports() + local service = read_file('../src/services/http/service.lua') + ok(service:find('devicecode.support.service_events', 1, true), 'http service does not use service event helper') + ok(service:find('_event_port', 1, true), 'http service lacks event-port helper') + ok(service:find('events_port = self:_event_port', 1, true), 'http backend/registry are not wired through event ports') + + local ops = read_file('../src/services/http/operations.lua') + ok(ops:find('events_port = service_events.port', 1, true), 'http listener operation does not pass event port') + ok(not ops:find('on_context_admitted = function', 1, true), 'http operation still passes context admission callback') + ok(not ops:find('on_context_transferred = function', 1, true), 'http operation still passes context transfer callback') + ok(not ops:find('on_server_websocket = function', 1, true), 'http operation still passes server websocket callback') +end + +function M.test_http_service_code_does_not_use_perform_raw() + local p = io.popen("find ../src/services/http -name '*.lua' | sort") + for file in p:lines() do + local s = read_file(file) + ok(not s:find('perform_raw', 1, true), 'perform_raw used in ' .. file) + end + p:close() +end + + +function M.test_http_service_and_support_do_not_use_private_op_internals() + local files = {} + local p1 = io.popen("find ../src/services/http -name '*.lua' | sort") + for file in p1:lines() do + if not file:find('/transport/', 1, true) then files[#files + 1] = file end + end + p1:close() + files[#files + 1] = '../src/devicecode/support/queue.lua' + files[#files + 1] = '../src/devicecode/support/priority_event.lua' + + local forbidden = { + 'try_fn', + 'wait_fn', + 'new_primitive', + } + + for _, file in ipairs(files) do + local s = read_file(file) + for _, needle in ipairs(forbidden) do + ok(not s:find(needle, 1, true), ('private Op internals %s used in %s'):format(needle, file)) + end + end +end + + +function M.test_http_service_start_is_foreground_entry_not_handle_constructor() + local service = read_file('../src/services/http/service.lua') + ok(service:find('function M.run%(scope, params%)'), 'http service does not expose run(scope, params)') + ok(service:find('function M.start%(conn, opts%)'), 'http service does not expose start(conn, opts)') + ok(service:find("require 'devicecode.service_base'", 1, true), 'http start does not use service_base lifecycle') + ok(service:find('function M.open_handle%(conn, opts%)'), 'http handle-returning helper is not explicit') + ok(not service:find('function M.start%(conn, opts%)[%s%S]-return self[%s%S]-end'), 'http start returns a service handle') + ok(service:find('http service returned unexpectedly', 1, true), 'http start does not fail if run returns') +end + +function M.test_http_unit_tests_use_explicit_handle_helper() + local needle = 'http_service.' .. 'start' + local cmd = "find ../tests/unit/http ../tests/integration/devhost -name '*.lua' | sort" + local p = assert(io.popen(cmd)) + for file in p:lines() do + local s = read_file(file) + ok(not s:find(needle, 1, true), 'http tests still use start as a handle constructor: ' .. file) + end + p:close() +end + +return M diff --git a/tests/unit/http/test_websocket_client.lua b/tests/unit/http/test_websocket_client.lua new file mode 100644 index 00000000..c10f08c7 --- /dev/null +++ b/tests/unit/http/test_websocket_client.lua @@ -0,0 +1,79 @@ +local fibers = require 'fibers' +local websocket = require 'services.http.websocket' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +local function fake_driver() + return { + run_op = function (_, _, fn) + return fibers.guard(function () return fibers.always(fn()) end) + end, + } +end + +function M.test_client_connect_and_send_are_driver_jobs() + fibers.run(function () + local raw = { + connected = false, + sent = nil, + connect = function (self) self.connected = true; return true end, + send = function (self, data, opcode) self.sent = { data, opcode }; return true end, + receive = function () return 'reply', 'text' end, + close = function () return true end, + } + local module = { + new_from_uri = function (uri) + eq(uri, 'ws://example.test/ws') + return raw + end, + } + local ws = ok(fibers.perform(websocket.connect_op(fake_driver(), { + uri = 'ws://example.test/ws', + }, { websocket_module = module }))) + ok(raw.connected, 'connect should run') + ok(fibers.perform(ws:send_op('hello', 'text'))) + eq(raw.sent[1], 'hello') + eq(raw.sent[2], 'text') + local data, opcode = fibers.perform(ws:receive_op()) + eq(data, 'reply') + eq(opcode, 'text') + end) +end + + +function M.test_client_connect_applies_validated_headers_before_connect() + fibers.run(function () + local raw = { + headers = { + values = {}, + upsert = function (self, k, v) self.values[k] = v end, + append = function (self, k, v) self.values[k] = v end, + }, + connect = function (self) + eq(self.headers.values['x-auth'], 'token', 'headers must be applied before websocket connect') + return true + end, + send = function () return true end, + receive = function () return nil, 'closed' end, + close = function () return true end, + } + local module = { + new_from_uri = function (uri) + eq(uri, 'ws://example.test/ws') + return raw + end, + } + local ws = ok(fibers.perform(websocket.connect_op(fake_driver(), { + uri = 'ws://example.test/ws', + headers = { ['x-auth'] = 'token' }, + }, { websocket_module = module }))) + ok(ws) + end) +end + +return M diff --git a/tests/unit/http/transport/test_cqueues_driver.lua b/tests/unit/http/transport/test_cqueues_driver.lua new file mode 100644 index 00000000..6cc00e39 --- /dev/null +++ b/tests/unit/http/transport/test_cqueues_driver.lua @@ -0,0 +1,454 @@ +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local runtime = require 'fibers.runtime' +local driver_mod = require 'services.http.transport.cqueues_driver' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end + +local function ok(v, msg) + if not v then error(msg or 'assertion failed', 2) end +end + +local function yield_once() + runtime.yield() +end + +local function yield_many(n) + for _ = 1, n do yield_once() end +end + +local function join_child_with_timeout(child, timeout_s) + local which = fibers.perform(fibers.named_choice { + joined = child:join_op(), + timeout = sleep.sleep_op(timeout_s or 1), + }) + return which == 'joined' +end + +local function yield_until(pred, msg) + for _ = 1, 50 do + if pred() then return true end + yield_once() + end + error(msg or 'condition was not reached', 2) +end + +local function shell_quote(s) + s = tostring(s or '') + return "'" .. s:gsub("'", "'\\''") .. "'" +end + +local function shell_exit_status(a, b, c) + if type(a) == 'number' then + if a >= 256 then return math.floor(a / 256) end + return a + end + if a == true then return 0 end + if b == 'exit' and type(c) == 'number' then return c end + if type(c) == 'number' then return c end + return 1 +end + +local function run_filtered_child(filter, timeout_s) + local cmd = ('timeout %s env TEST_FILTER=%s luajit run.lua'):format( + tostring(timeout_s or 2), shell_quote(filter)) + return shell_exit_status(os.execute(cmd)) +end + +local function fake_controller() + local q = {} + return { + wraps = 0, + steps = 0, + closed = false, + wrap = function (self, fn) + self.wraps = self.wraps + 1 + q[#q + 1] = fn + return self + end, + step = function (self, timeout) + eq(timeout, 0, 'driver must step with timeout 0') + self.steps = self.steps + 1 + local fn = table.remove(q, 1) + if fn then fn() end + return true + end, + pollfd = function () return nil end, + events = function () return '' end, + timeout = function () return nil end, + queued = function () return #q end, + } +end + +function M.test_event_wants_maps_cqueues_event_strings() + local rd, wr = driver_mod.event_wants('r') + ok(rd, 'r should want read') + ok(not wr, 'r should not want write') + + rd, wr = driver_mod.event_wants('w') + ok(not rd, 'w should not want read') + ok(wr, 'w should want write') + + rd, wr = driver_mod.event_wants('p') + ok(rd, 'priority event should map to read-side wake') + ok(not wr, 'p should not want write') + + rd, wr = driver_mod.event_wants('rw') + ok(rd and wr, 'rw should want both directions') +end + +function M.test_pollable_ready_op_uses_poke_as_step_hint() + local drv = assert(driver_mod.new { create_controller = false }) + local pollable = { + pollfd = function () return nil end, + events = function () return '' end, + timeout = function () return nil end, + } + + fibers.run(function (scope) + local reason, err + assert(scope:spawn(function () + reason, err = fibers.perform(drv:pollable_ready_op(pollable)) + end)) + yield_many(3) + drv:poke() + yield_many(3) + eq(reason, 'poke') + eq(err, nil) + drv:terminate('done') + end) +end + +function M.test_pollable_ready_op_reports_due_timeout_immediately() + local drv = assert(driver_mod.new { create_controller = false }) + local pollable = { timeout = function () return 0 end } + + fibers.run(function () + local reason, err = fibers.perform(drv:pollable_ready_op(pollable)) + eq(reason, 'timeout') + eq(err, nil) + drv:terminate('done') + end) +end + +function M.test_pollable_step_op_treats_spurious_timeout_step_as_nonfatal() + local drv = assert(driver_mod.new { create_controller = false }) + local pollable = { + timeout = function () return 0 end, + step = function (_, timeout) + eq(timeout, 0) + return nil, 'timeout' + end, + } + + fibers.run(function () + local ok1, err = fibers.perform(drv:pollable_step_op(pollable)) + eq(ok1, true) + eq(err, nil) + drv:terminate('done') + end) +end + +function M.test_run_op_completes_inside_driver_pump() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + + fibers.run(function (scope) + assert(drv:start(scope)) + local a, b = fibers.perform(drv:run_op('unit.job', function () + return 'done', 42 + end)) + eq(a, 'done') + eq(b, 42) + ok(ctl.wraps >= 1, 'controller.wrap should be used') + ok(ctl.steps >= 1, 'controller.step should be used') + drv:terminate('test complete') + end) +end + +function M.test_run_op_reports_lua_errors_as_failed_results() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + + fibers.run(function (scope) + assert(drv:start(scope)) + local v, err = fibers.perform(drv:run_op('unit.error', function () + error('boom', 0) + end)) + eq(v, nil) + ok(tostring(err):find('boom', 1, true) ~= nil, 'error should be reported') + drv:terminate('test complete') + end) +end + +function M.test_losing_run_op_aborts_without_running_job() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + local aborted = false + local ran = false + + fibers.run(function (scope) + assert(drv:start(scope)) + + local which = fibers.perform(fibers.named_choice { + winner = fibers.always('now'), + job = drv:run_op('unit.loser', function () + ran = true + return 'late' + end, { + on_abort = function () aborted = true end, + }), + }) + + eq(which, 'winner') + ok(aborted, 'losing job should be abandoned') + ctl:step(0) + ok(not ran, 'abandoned job body must not run') + drv:terminate('test complete') + end) +end + +function M.test_losing_active_run_op_calls_active_abort_without_closing_driver() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + local active = false + local release = false + local active_abort + + fibers.run(function (scope) + assert(drv:start(scope)) + + local waiter = assert(scope:child()) + local result, err + assert(waiter:spawn(function () + result, err = fibers.perform(drv:run_op('unit.active_loser', function () + active = true + while not release do runtime.yield() end + return 'late' + end, { + on_active_abort = function (reason) + active_abort = reason + release = true + end, + })) + end)) + + yield_until(function () return active end, 'job should become active') + waiter:cancel('stop_waiting') + fibers.perform(waiter:join_op()) + eq(active_abort, 'aborted') + ok(not drv:is_closed(), 'active abort with a narrow owner must not close the driver') + release = true + yield_many(3) + eq(result, nil) + eq(err, nil) + drv:terminate('done') + end) +end + + +function M.test_losing_active_run_op_can_detach_and_cleanup_after_cqueues_completion() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + local active = false + local release = false + local detach_reason + local cleanup_reason + local cleanup_result + + fibers.run(function (scope) + assert(drv:start(scope)) + + local waiter = assert(scope:child()) + local result, err + assert(waiter:spawn(function () + result, err = fibers.perform(drv:run_op('unit.active_detached', function () + active = true + while not release do runtime.yield() end + return 'late_result' + end, { + detach_on_abort = true, + on_detach = function (reason) detach_reason = reason end, + on_detached_complete = function (reason, ok, packed) + cleanup_reason = reason + if ok and packed then cleanup_result = packed[1] end + end, + })) + end)) + + yield_until(function () return active end, 'job should become active') + waiter:cancel('stop_waiting') + if not join_child_with_timeout(waiter, 0.25) then + release = true + drv:terminate('detached test bounded failure') + join_child_with_timeout(waiter, 0.25) + error('detached run_op caller did not return after abort; active cqueues jobs must detach caller cleanup', 2) + end + eq(result, nil) + eq(err, nil) + eq(detach_reason, 'aborted') + ok(not drv:is_closed(), 'detached active abort must not close driver') + release = true + yield_until(function () return cleanup_reason == 'aborted' end, 'detached cleanup should run after cqueues job returns') + eq(cleanup_result, 'late_result') + ok(not drv:is_closed(), 'detached cleanup must not close driver') + drv:terminate('done') + end) +end + +function M.test_losing_active_run_op_without_owner_terminates_driver() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + local active = false + + fibers.run(function (scope) + assert(drv:start(scope)) + + local waiter = assert(scope:child()) + assert(waiter:spawn(function () + fibers.perform(drv:run_op('unit.active_loser_no_owner', function () + active = true + while not drv:is_closed() do runtime.yield() end + return 'late' + end)) + end)) + + yield_until(function () return active end, 'job should become active') + waiter:cancel('stop_waiting') + fibers.perform(waiter:join_op()) + ok(drv:is_closed(), 'active abort with no narrower owner should close the driver') + eq(drv:why(), 'aborted') + end) +end + +function M.test_active_aborted_job_late_completion_is_ignored() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + local active = false + local release = false + local active_abort = 0 + + fibers.run(function (scope) + assert(drv:start(scope)) + + local waiter = assert(scope:child()) + local result, err + assert(waiter:spawn(function () + result, err = fibers.perform(drv:run_op('unit.active_late_completion', function () + active = true + while not release do runtime.yield() end + return 'late_result' + end, { + on_active_abort = function () + active_abort = active_abort + 1 + end, + })) + end)) + + yield_until(function () return active end, 'job should become active') + waiter:cancel('stop_waiting') + fibers.perform(waiter:join_op()) + eq(active_abort, 1) + release = true + yield_many(5) + eq(result, nil) + eq(err, nil) + ok(not drv:is_closed(), 'owned active abort should not close driver') + drv:terminate('done') + end) +end + +function M.test_terminate_wakes_pending_run_op_without_pump() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + + fibers.run(function (scope) + local result, err + assert(scope:spawn(function () + result, err = fibers.perform(drv:run_op('unit.never', function () + return 'should not matter' + end)) + end)) + + yield_many(3) + drv:terminate('stopping') + yield_many(3) + + eq(result, nil) + eq(err, 'stopping') + end) +end + +function M.test_multiple_concurrent_run_ops_are_completed_by_one_pump() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + + fibers.run(function (scope) + local a, b + assert(drv:start(scope)) + assert(scope:spawn(function () a = fibers.perform(drv:run_op('a', function () return 'A' end)) end)) + assert(scope:spawn(function () b = fibers.perform(drv:run_op('b', function () return 'B' end)) end)) + + for _ = 1, 5 do yield_once() end + eq(a, 'A') + eq(b, 'B') + ok(ctl.steps >= 2, 'pump should step enough to complete queued jobs') + drv:terminate('done') + end) +end + +function M.test_driver_scope_finaliser_terminates_driver_after_scope_cancel() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + + fibers.run(function (scope) + local child = assert(scope:child()) + assert(child:spawn(function (s) + assert(drv:start(s)) + -- Keep the component scope alive until cancellation. + fibers.perform(sleep.sleep_op(60)) + end)) + yield_once() + child:cancel('stop') + fibers.perform(child:join_op()) + ok(drv:is_closed(), 'driver should be closed by scope finaliser') + end) +end + + +function M.test_driver_start_can_target_started_parent_scope_from_child_scope() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + + fibers.run(function (parent) + local child = assert(parent:child()) + local started, start_err + + assert(child:spawn(function () + started, start_err = drv:start(parent) + end)) + + yield_many(3) + eq(started, true) + eq(start_err, nil) + drv:terminate('done') + end) +end + + +-- Keep this ownership-boundary regression under the same focused regression filter as the +-- hostile real-HTTP test. Run the raw payload in a subprocess: the old +-- hard-close-only implementation can wedge the Fibers scheduler so an +-- in-process watchdog cannot fire reliably. The subprocess timeout turns that +-- into a bounded, clear test failure. +function M.test_metrics_style_exchange_timeout_regression_cqueues_driver_detaches_active_job_cleanup_to_cqueues_job() + local code = run_filtered_child('test_losing_active_run_op_can_detach_and_cleanup_after_cqueues_completion', + tonumber(os.getenv('HTTP_METRICS_TIMEOUT_UNIT_CHILD_TIMEOUT_S') or '') or 2) + eq(code, 0, 'detached cqueues-driver ownership payload should pass in a bounded child process') +end + +return M diff --git a/tests/unit/http/transport/test_legacy_http1_close.lua b/tests/unit/http/transport/test_legacy_http1_close.lua new file mode 100644 index 00000000..f0ff9193 --- /dev/null +++ b/tests/unit/http/transport/test_legacy_http1_close.lua @@ -0,0 +1,97 @@ +local fibers = require 'fibers' +local op = require 'fibers.op' +local legacy = require 'services.http.transport.legacy_http1_close' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +function M.test_parse_response_requires_status_line_and_keeps_body_bytes() + local headers, stream = legacy._test.parse_response('HTTP/1.1 200 OK\r\nServer: Hydra/0.1.8\r\nConnection: close\r\n\r\n{"data":{"ok":true}}') + ok(headers) + eq(headers:get(':status'), '200') + eq(headers:get('server'), 'Hydra/0.1.8') + eq(stream:get_body_as_string(), '{"data":{"ok":true}}') + + local bad, _, err = legacy._test.parse_response('not http before json {"data":{"ok":true}}') + eq(bad, nil) + eq(err, 'no_header_separator') +end + +function M.test_render_request_uses_http10_connection_close_and_content_length() + local chunks = { 'abc' } + local req = legacy._test.render_request({ + method = 'POST', + _uri = { path = '/cgi/set.cgi?cmd=home_loginAuth', authority = '192.168.1.1' }, + headers = { ['content-type'] = 'application/x-www-form-urlencoded' }, + }, 'abc') + ok(req:find('POST /cgi/set.cgi?cmd=home_loginAuth HTTP/1.0\r\n', 1, true)) + ok(req:find('Host: 192.168.1.1\r\n', 1, true)) + ok(req:find('Connection: close\r\n', 1, true)) + ok(req:find('Content-Length: 3\r\n', 1, true)) + ok(req:find('\r\n\r\nabc', 1, true)) +end + +function M.test_read_response_bounded_stops_at_content_length_without_eof() + fibers.run(function () + local reads = { + 'HTTP/1.1 200 OK\r\nContent-Length: 7\r\nConnection: keep-alive\r\n\r\n{', + '"x":1}', + } + local stream = { + read_some_op = function (_, _want) + local chunk = table.remove(reads, 1) + if chunk == nil then return op.always(nil, 'unexpected_extra_read') end + return op.always(chunk, nil) + end, + } + local headers, body_stream, err = legacy._test.read_response_bounded(stream, 1024, 'GET') + ok(headers, err) + eq(headers:get(':status'), '200') + eq(body_stream:get_body_as_string(), '{"x":1}') + eq(#reads, 0) + end) +end + + +function M.test_render_request_preserves_query_from_original_uri() + local request = ok(legacy._test.render_request({ + method = 'GET', + _uri = { + uri = 'http://192.168.1.1/cgi/get.cgi?cmd=panel_info&dummy=123', + scheme = 'http', + host = '192.168.1.1', + port = 80, + path = '/cgi/get.cgi', + authority = '192.168.1.1', + }, + })) + ok(request:find('GET /cgi/get.cgi?cmd=panel_info&dummy=123 HTTP/1.0', 1, true)) +end + + +function M.test_parse_response_rejects_chunked_transfer_encoding() + local headers, stream, err = legacy._test.parse_response( + 'HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n{"x":1}\r\n0\r\n\r\n' + ) + eq(headers, nil) + eq(stream, nil) + eq(err, 'unsupported_transfer_encoding') +end + +function M.test_read_response_bounded_rejects_oversize_headers() + fibers.run(function () + local stream = { + read_some_op = function () return op.always('HTTP/1.1 200 OK\r\nX-Long: abcdef\r\n', nil) end, + } + local headers, body_stream, err = legacy._test.read_response_bounded(stream, 16, 'GET') + eq(headers, nil) + eq(body_stream, nil) + eq(err, 'response_too_large') + end) +end + +return M diff --git a/tests/unit/http/transport/test_lua_http.lua b/tests/unit/http/transport/test_lua_http.lua new file mode 100644 index 00000000..74207e1d --- /dev/null +++ b/tests/unit/http/transport/test_lua_http.lua @@ -0,0 +1,403 @@ +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local runtime = require 'fibers.runtime' +local driver_mod = require 'services.http.transport.cqueues_driver' +local lua_http = require 'services.http.transport.lua_http' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end + +local function ok(v, msg) + if not v then error(msg or 'assertion failed', 2) end +end + +local function yield_once() + runtime.yield() +end + +local function yield_many(n) + for _ = 1, n do yield_once() end +end + +local function yield_until(pred, msg) + for _ = 1, 50 do + if pred() then return true end + yield_once() + end + error(msg or 'condition was not reached', 2) +end + +local function fake_condition() + return { + signals = 0, + waits = 0, + signal = function (self) self.signals = self.signals + 1; return true end, + wait = function (self) self.waits = self.waits + 1; return true end, + } +end + +local function fake_server() + return { + steps = 0, + closed = false, + listened = false, + pollfd = function () return nil end, + events = function () return '' end, + timeout = function () return nil end, + step = function (self, timeout) + eq(timeout, 0) + self.steps = self.steps + 1 + return true + end, + listen = function (self, timeout) + self.listened = timeout or true + return true + end, + close = function (self) + self.closed = true + return true + end, + pause = function (self) self.paused = true; return true end, + resume = function (self) self.resumed = true; return true end, + } +end + +local function fake_stream() + return { + terminated = false, + calls = {}, + shutdown = function (self) self.terminated = true; return true end, + get_headers = function (self, timeout) self.calls[#self.calls+1] = {'headers', timeout}; return { ':method', 'GET' } end, + get_next_chunk = function (self, timeout) self.calls[#self.calls+1] = {'chunk', timeout}; return 'chunk', false end, + get_body_chars = function (self, n, timeout) self.calls[#self.calls+1] = {'chars', n, timeout}; return string.rep('x', n) end, + get_body_as_string = function (self, timeout) self.calls[#self.calls+1] = {'body', timeout}; return 'body' end, + write_headers = function (self, headers, end_stream, timeout) self.calls[#self.calls+1] = {'write_headers', headers, end_stream, timeout}; return true end, + write_chunk = function (self, chunk, end_stream, timeout) self.calls[#self.calls+1] = {'write_chunk', chunk, end_stream, timeout}; return true end, + write_body_from_string = function (self, body, timeout) self.calls[#self.calls+1] = {'write_body', body, timeout}; return true end, + } +end + +local function listener_with(opts) + opts = opts or {} + local drv = opts.driver or assert(driver_mod.new { create_controller = false }) + return assert(lua_http.listen { + server = opts.server or fake_server(), + driver = drv, + max_accept_queue = opts.max_accept_queue, + condition_factory = function () return fake_condition() end, + context_terminator = opts.context_terminator, + intra_stream_timeout = opts.intra_stream_timeout, + on_context = opts.on_context, + }) +end + +function M.test_listener_admits_context_and_accept_op_returns_it() + local listener = listener_with() + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), fake_stream()) + + fibers.run(function () + assert(listener:_admit_context(ctx)) + local got, err = fibers.perform(listener:accept_op()) + eq(got, ctx) + eq(err, nil) + listener:terminate('done') + end) +end + +function M.test_listener_accept_queue_overflow_terminates_context() + local terminated_reason + local listener = listener_with { + max_accept_queue = 0, + context_terminator = function (_, reason) terminated_reason = reason end, + } + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), fake_stream()) + local ok1, err = listener:_admit_context(ctx) + eq(ok1, nil) + eq(err, 'accept_queue_full') + eq(terminated_reason, 'accept_queue_full') + listener:terminate('done') +end + +function M.test_listener_terminate_wakes_accept_waiter() + local listener = listener_with() + + fibers.run(function (scope) + local got, err + assert(scope:spawn(function () + got, err = fibers.perform(listener:accept_op()) + end)) + yield_many(3) + listener:terminate('closed') + yield_many(3) + eq(got, nil) + eq(err, 'closed') + end) +end + +function M.test_http_context_stream_ops_run_on_command_loop() + local listener = listener_with() + local stream = fake_stream() + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), stream) + + fibers.run(function (scope) + local ok_write, err + assert(scope:spawn(function () + ok_write, err = fibers.perform(ctx:write_chunk_op('abc', true)) + end)) + yield_many(3) + ok(ctx:_drain_one_for_test(), 'command should be queued') + yield_many(3) + eq(ok_write, true) + eq(err, nil) + eq(stream.calls[1][1], 'write_chunk') + eq(stream.calls[1][2], 'abc') + eq(stream.calls[1][3], true) + eq(stream.calls[1][4], nil) + ctx:terminate('done') + listener:terminate('done') + end) +end + +function M.test_http_context_read_helpers_preserve_http_semantics() + local listener = listener_with() + local stream = fake_stream() + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), stream) + + fibers.run(function (scope) + local body + assert(scope:spawn(function () + body = fibers.perform(ctx:read_body_as_string_op()) + end)) + yield_many(3) + ok(ctx:_drain_one_for_test()) + yield_many(3) + eq(body, 'body') + eq(stream.calls[1][1], 'body') + eq(stream.calls[1][2], nil) + ctx:terminate('done') + listener:terminate('done') + end) +end + +function M.test_http_context_stream_ops_pass_intra_stream_timeout() + local listener = listener_with { intra_stream_timeout = 0.25 } + local stream = fake_stream() + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), stream) + + fibers.run(function (scope) + assert(scope:spawn(function () fibers.perform(ctx:read_chunk_op()) end)) + yield_many(3) + ok(ctx:_drain_one_for_test()) + yield_many(3) + eq(stream.calls[1][1], 'chunk') + eq(stream.calls[1][2], 0.25, 'server context should pass intra_stream_timeout to lua-http stream read') + ctx:terminate('done') + listener:terminate('done') + end) +end + +function M.test_losing_stream_command_is_abandoned_and_not_run() + local listener = listener_with() + local stream = fake_stream() + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), stream) + local ran = false + local aborted = false + + fibers.run(function () + local which = fibers.perform(fibers.named_choice { + winner = fibers.always('now'), + cmd = ctx:run_stream_op('loser', function () ran = true; return true end, { + on_abort = function () aborted = true end, + }), + }) + eq(which, 'winner') + ok(aborted, 'abort hook should run') + ctx:_drain_one_for_test() + ok(not ran, 'abandoned command should not run') + ctx:terminate('done') + listener:terminate('done') + end) +end + +function M.test_losing_active_stream_command_terminates_context() + local terminated_reason + local listener = listener_with { + context_terminator = function (_, reason) terminated_reason = reason end, + } + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), fake_stream()) + local active = false + local release = false + + fibers.run(function (scope) + local waiter = assert(scope:child()) + assert(waiter:spawn(function () + fibers.perform(ctx:run_stream_op('active_loser', function () + active = true + while not release do runtime.yield() end + return true + end)) + end)) + + assert(scope:spawn(function () ctx:_drain_one_for_test() end)) + yield_until(function () return active end, 'stream command should become active') + waiter:cancel('stop_waiting') + fibers.perform(waiter:join_op()) + eq(terminated_reason, 'aborted') + ok(ctx:is_closed(), 'active command abort should close the context') + release = true + listener:terminate('done') + end) +end + +function M.test_losing_active_write_command_terminates_context() + local terminated_reason + local stream = fake_stream() + stream.write_chunk = function () + stream.write_active = true + while not stream.terminated do runtime.yield() end + return true + end + local listener = listener_with { + context_terminator = function (s, reason) + terminated_reason = reason + s.terminated = true + end, + } + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), stream) + + fibers.run(function (scope) + local waiter = assert(scope:child()) + assert(waiter:spawn(function () + fibers.perform(ctx:write_chunk_op('payload', false)) + end)) + + assert(scope:spawn(function () ctx:_drain_one_for_test() end)) + yield_until(function () return stream.write_active end, 'write command should become active') + waiter:cancel('stop_waiting') + fibers.perform(waiter:join_op()) + eq(terminated_reason, 'aborted') + ok(ctx:is_closed(), 'active write abort should close the context') + listener:terminate('done') + end) +end + +function M.test_context_terminate_wakes_pending_stream_command() + local listener = listener_with() + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), fake_stream()) + + fibers.run(function (scope) + local result, err + assert(scope:spawn(function () + result, err = fibers.perform(ctx:run_stream_op('pending', function () return 'late' end)) + end)) + yield_many(3) + ctx:terminate('gone') + yield_many(3) + eq(result, nil) + eq(err, 'gone') + listener:terminate('done') + end) +end + +function M.test_listener_terminate_terminates_open_contexts_and_server() + local server = fake_server() + local terminated = 0 + local listener = listener_with { + server = server, + context_terminator = function () terminated = terminated + 1 end, + } + local ctx = lua_http._new_context_for_test(listener, server, fake_stream()) + assert(listener:_admit_context(ctx)) + listener:terminate('service_down') + ok(server.closed, 'server close should be called') + eq(terminated, 1) + ok(ctx:is_closed(), 'context should be closed') +end + + +function M.test_listener_terminate_does_not_terminate_accepted_context_after_transfer() + local server = fake_server() + local terminated = 0 + local listener = listener_with { + server = server, + context_terminator = function () terminated = terminated + 1 end, + } + local ctx = lua_http._new_context_for_test(listener, server, fake_stream()) + + fibers.run(function () + assert(listener:_admit_context(ctx)) + local got = fibers.perform(listener:accept_op()) + eq(got, ctx) + + listener:terminate('listener_down') + ok(server.closed, 'server close should be called') + eq(terminated, 0) + ok(not ctx:is_closed(), 'accepted context should be owned by the caller after accept') + ctx:terminate('request_done') + end) +end + +function M.test_listener_listen_op_runs_via_driver() + local server = fake_server() + local ctl = { + q = {}, + wrap = function (self, fn) self.q[#self.q+1] = fn end, + step = function (self) local fn = table.remove(self.q, 1); if fn then fn() end; return true end, + pollfd = function () return nil end, + events = function () return '' end, + timeout = function () return nil end, + } + local drv = assert(driver_mod.new { controller = ctl }) + local listener = listener_with { server = server, driver = drv } + + fibers.run(function (scope) + assert(drv:start(scope)) + local ok_listen = fibers.perform(listener:listen_op()) + eq(ok_listen, true) + eq(server.listened, true) + listener:terminate('done') + drv:terminate('done') + end) +end + + +function M.test_listener_start_can_target_started_parent_scope_from_child_scope() + local listener = listener_with() + + fibers.run(function (parent) + local child = assert(parent:child()) + local started, start_err + + assert(child:spawn(function () + started, start_err = listener:start(parent) + end)) + + yield_many(3) + eq(started, true) + eq(start_err, nil) + listener:terminate('done') + end) +end + + +function M.test_onstream_admission_does_not_call_service_hook_synchronously() + local called = 0 + local listener = listener_with { + on_context = function () + called = called + 1 + error('service hook must not run from the backend onstream path', 0) + end, + } + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), fake_stream()) + + local admitted, err = listener:_admit_context(ctx) + eq(admitted, true) + eq(err, nil) + eq(called, 0, 'backend context admission must only wake a Fibers-owned event source') + listener:terminate('done') +end + +return M diff --git a/tests/unit/http/transport/test_terminate.lua b/tests/unit/http/transport/test_terminate.lua new file mode 100644 index 00000000..c84610b1 --- /dev/null +++ b/tests/unit/http/transport/test_terminate.lua @@ -0,0 +1,101 @@ +local terminate = require 'services.http.transport.terminate' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +function M.test_terminate_stream_takes_and_closes_underlying_socket_before_graceful_methods() + local socket_closed = false + local stream_close_called = false + local stream_shutdown_called = false + local stream = { + connection = { + take_socket = function () + return { close = function () socket_closed = true; return true end } + end, + }, + close = function () stream_close_called = true; return true end, + shutdown = function () stream_shutdown_called = true; return true end, + } + ok(terminate.terminate_stream(stream, 'drop')) + eq(socket_closed, true) + eq(stream_close_called, false) + eq(stream_shutdown_called, false) +end + +function M.test_terminate_stream_prefers_terminate_then_close_over_shutdown_when_socket_take_unavailable() + local calls = {} + local stream = { + terminate = function (_, reason) calls[#calls + 1] = 'terminate:' .. tostring(reason); return true end, + close = function () calls[#calls + 1] = 'close'; return true end, + shutdown = function () calls[#calls + 1] = 'shutdown'; return true end, + } + ok(terminate.terminate_stream(stream, 'done')) + eq(calls[1], 'terminate:done') + eq(calls[2], nil) +end + +function M.test_terminate_stream_falls_back_to_close_before_shutdown() + local calls = {} + local stream = { + close = function () calls[#calls + 1] = 'close'; return true end, + shutdown = function () calls[#calls + 1] = 'shutdown'; return true end, + } + ok(terminate.terminate_stream(stream, 'done')) + eq(calls[1], 'close') + eq(calls[2], nil) +end + +function M.test_terminate_server_uses_immediate_termination() + local closed + ok(terminate.terminate_server({ close = function () closed = true; return true end }, 'down')) + eq(closed, true) +end + +function M.test_terminate_websocket_uses_abnormal_immediate_termination() + local code, reason, timeout + ok(terminate.terminate_websocket({ close = function (_, c, r, t) code, reason, timeout = c, r, t; return true end }, 'drop')) + eq(code, 1006) + eq(reason, 'drop') + eq(timeout, 0) +end + +function M.test_terminate_request_takes_and_closes_underlying_socket_when_present() + local socket_closed = false + local request = { + connection = { + take_socket = function () return { close = function () socket_closed = true; return true end } end, + }, + cancel = function () error('cancel must not be used when socket take succeeds', 0) end, + close = function () error('close must not be used when socket take succeeds', 0) end, + shutdown = function () error('shutdown must not be used when socket take succeeds', 0) end, + } + ok(terminate.terminate_request(request, 'drop')) + eq(socket_closed, true) +end + +function M.test_terminate_request_prefers_terminate_when_available() + local calls = {} + ok(terminate.terminate_request({ + terminate = function (_, reason) calls[#calls + 1] = 'terminate:' .. tostring(reason); return true end, + cancel = function () calls[#calls + 1] = 'cancel'; return true end, + }, 'drop')) + eq(calls[1], 'terminate:drop') + eq(calls[2], nil) +end + +function M.test_terminate_request_falls_back_to_cancel_before_graceful_methods() + local calls = {} + ok(terminate.terminate_request({ + cancel = function (_, reason) calls[#calls + 1] = 'cancel:' .. tostring(reason); return true end, + close = function () calls[#calls + 1] = 'close'; return true end, + shutdown = function () calls[#calls + 1] = 'shutdown'; return true end, + }, 'drop')) + eq(calls[1], 'cancel:drop') + eq(calls[2], nil) +end + +return M diff --git a/tests/unit/http/transport/test_websocket.lua b/tests/unit/http/transport/test_websocket.lua new file mode 100644 index 00000000..700bbbb7 --- /dev/null +++ b/tests/unit/http/transport/test_websocket.lua @@ -0,0 +1,287 @@ +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local runtime = require 'fibers.runtime' +local driver_mod = require 'services.http.transport.cqueues_driver' +local lua_http = require 'services.http.transport.lua_http' +local websocket = require 'services.http.transport.websocket' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end + +local function ok(v, msg) + if not v then error(msg or 'assertion failed', 2) end +end + +local function yield_once() + runtime.yield() +end + +local function yield_many(n) + for _ = 1, n do yield_once() end +end + +local function yield_until(pred, msg) + for _ = 1, 50 do + if pred() then return true end + yield_once() + end + error(msg or 'condition was not reached', 2) +end + +local function fake_controller() + local q = {} + return { + wrap = function (self, fn) q[#q + 1] = fn; return self end, + step = function () local fn = table.remove(q, 1); if fn then fn() end; return true end, + pollfd = function () return nil end, + events = function () return '' end, + timeout = function () return nil end, + } +end + +local function fake_condition() + return { + signal = function () return true end, + wait = function () return true end, + } +end + +local function fake_server() + return { + pollfd = function () return nil end, + events = function () return '' end, + timeout = function () return nil end, + step = function () return true end, + close = function () return true end, + } +end + +local function listener_with(term) + return assert(lua_http.listen { + server = fake_server(), + driver = assert(driver_mod.new { create_controller = false }), + condition_factory = function () return fake_condition() end, + context_terminator = term, + }) +end + +local function fake_ws() + return { + calls = {}, + accept = function (self, options, timeout) + self.calls[#self.calls+1] = {'accept', options, timeout} + return true + end, + receive = function (self, timeout) + self.calls[#self.calls+1] = {'receive', timeout} + return 'hello', 'text' + end, + send = function (self, data, opcode, timeout) + self.calls[#self.calls+1] = {'send', data, opcode, timeout} + return true + end, + send_ping = function (self, data, timeout) + self.calls[#self.calls+1] = {'ping', data, timeout} + return true + end, + send_pong = function (self, data, timeout) + self.calls[#self.calls+1] = {'pong', data, timeout} + return true + end, + close = function (self, code, reason, timeout) + self.calls[#self.calls+1] = {'close', code, reason, timeout} + return true + end, + } +end + +function M.test_from_context_op_constructs_websocket_inside_context() + local listener = listener_with() + local stream = { id = 'stream' } + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), stream) + local raw = fake_ws() + local module = { + new_from_stream = function (s, headers) + eq(s, stream) + eq(headers.token, 'h') + return raw + end, + } + + fibers.run(function (scope) + local ws, err + assert(scope:spawn(function () + ws, err = fibers.perform(websocket.from_context_op(ctx, { token = 'h' }, { + websocket_module = module, + })) + end)) + yield_many(3) + ok(ctx:_drain_one_for_test()) + yield_many(3) + ok(ws, 'websocket should be returned') + eq(err, nil) + eq(ws:_raw_websocket_for_test(), raw) + ws:terminate('done') + listener:terminate('done') + end) +end + +function M.test_websocket_receive_and_send_are_context_commands() + local listener = listener_with() + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), {}) + local raw = fake_ws() + local ws = websocket.WebSocket and setmetatable({ _ctx = ctx, _ws = raw }, websocket.WebSocket) or nil + -- Avoid reaching through the metatable from production callers; this is a unit seam. + if not ws then error('missing WebSocket class') end + + fibers.run(function (scope) + local data, opcode, sent + assert(scope:spawn(function () data, opcode = fibers.perform(ws:receive_op()) end)) + yield_many(3) + ok(ctx:_drain_one_for_test()) + yield_many(3) + eq(data, 'hello') + eq(opcode, 'text') + + assert(scope:spawn(function () sent = fibers.perform(ws:send_op('payload', 'text')) end)) + yield_many(3) + ok(ctx:_drain_one_for_test()) + yield_many(3) + eq(sent, true) + eq(raw.calls[2][1], 'send') + eq(raw.calls[2][2], 'payload') + eq(raw.calls[2][3], 'text') + eq(raw.calls[2][4], nil) + + ws:terminate('done') + listener:terminate('done') + end) +end + +function M.test_websocket_close_op_marks_wrapper_closed() + local listener = listener_with() + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), {}) + local raw = fake_ws() + local ws = setmetatable({ _ctx = ctx, _ws = raw }, websocket.WebSocket) + + fibers.run(function (scope) + local closed_ok + assert(scope:spawn(function () closed_ok = fibers.perform(ws:close_op(1000, 'bye')) end)) + yield_many(3) + ok(ctx:_drain_one_for_test()) + yield_many(3) + eq(closed_ok, true) + ok(ws:is_closed(), 'wrapper should be closed after close_op') + eq(ws:why(), 'bye') + eq(raw.calls[1][1], 'close') + eq(raw.calls[1][2], 1000) + eq(raw.calls[1][3], 'bye') + listener:terminate('done') + end) +end + +function M.test_websocket_terminate_terminates_context_without_protocol_close() + local terminated + local listener = listener_with(function (_, reason) terminated = reason end) + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), {}) + local raw = fake_ws() + local ws = setmetatable({ _ctx = ctx, _ws = raw }, websocket.WebSocket) + + ws:terminate('drop') + ok(ws:is_closed(), 'websocket should be marked closed') + eq(terminated, 'drop') + eq(#raw.calls, 0, 'terminate must not perform graceful websocket close_op') + listener:terminate('done') +end + +function M.test_server_websocket_active_receive_abort_terminates_context_and_wrapper() + local terminated_reason + local listener = listener_with(function (_, reason) terminated_reason = reason end) + local ctx = lua_http._new_context_for_test(listener, listener:_raw_server_for_test(), {}) + local raw = fake_ws() + raw.receive = function (self) + self.receive_active = true + while terminated_reason == nil do runtime.yield() end + return nil, 'closed' + end + local ws = setmetatable({ _ctx = ctx, _ws = raw }, websocket.WebSocket) + ctx._http_transport_websocket = ws + + fibers.run(function (scope) + local waiter = assert(scope:child()) + assert(waiter:spawn(function () fibers.perform(ws:receive_op()) end)) + assert(scope:spawn(function () ctx:_drain_one_for_test() end)) + yield_until(function () return raw.receive_active end, 'server websocket receive should become active') + waiter:cancel('stop_waiting') + fibers.perform(waiter:join_op()) + eq(terminated_reason, 'aborted') + ok(ctx:is_closed(), 'active server websocket abort should close context') + ok(ws:is_closed(), 'transport websocket wrapper should be marked closed') + listener:terminate('done') + end) +end + +function M.test_client_websocket_active_receive_abort_terminates_websocket() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + local raw = fake_ws() + raw.receive = function (self) + self.receive_active = true + while not self.closed do runtime.yield() end + return nil, 'closed' + end + raw.close = function (self, code, reason, timeout) + self.closed = true + self.calls[#self.calls+1] = {'close', code, reason, timeout} + return true + end + local ws = setmetatable({ _driver = drv, _ws = raw }, websocket.ClientWebSocket) + + fibers.run(function (scope) + assert(drv:start(scope)) + local waiter = assert(scope:child()) + assert(waiter:spawn(function () fibers.perform(ws:receive_op()) end)) + yield_until(function () return raw.receive_active end, 'client websocket receive should become active') + waiter:cancel('stop_waiting') + fibers.perform(waiter:join_op()) + ok(raw.closed, 'active client websocket abort should close raw websocket') + ok(ws:is_closed(), 'client websocket wrapper should be marked closed') + ok(not drv:is_closed(), 'websocket is the narrow owner; driver should survive') + drv:terminate('done') + end) +end + +function M.test_client_websocket_active_send_abort_terminates_websocket() + local ctl = fake_controller() + local drv = assert(driver_mod.new { controller = ctl }) + local raw = fake_ws() + raw.send = function (self) + self.send_active = true + while not self.closed do runtime.yield() end + return true + end + raw.close = function (self, code, reason, timeout) + self.closed = true + self.calls[#self.calls+1] = {'close', code, reason, timeout} + return true + end + local ws = setmetatable({ _driver = drv, _ws = raw }, websocket.ClientWebSocket) + + fibers.run(function (scope) + assert(drv:start(scope)) + local waiter = assert(scope:child()) + assert(waiter:spawn(function () fibers.perform(ws:send_op('payload', 'text')) end)) + yield_until(function () return raw.send_active end, 'client websocket send should become active') + waiter:cancel('stop_waiting') + fibers.perform(waiter:join_op()) + ok(raw.closed, 'active client websocket send abort should close raw websocket') + ok(ws:is_closed(), 'client websocket wrapper should be marked closed') + ok(not drv:is_closed(), 'websocket is the narrow owner; driver should survive') + drv:terminate('done') + end) +end + +return M diff --git a/tests/unit/main/service_spec.lua b/tests/unit/main/service_spec.lua new file mode 100644 index 00000000..a0bd146e --- /dev/null +++ b/tests/unit/main/service_spec.lua @@ -0,0 +1,47 @@ +-- tests/main_spec.lua + +local mainmod = require 'devicecode.main' +local runfibers = require 'tests.support.run_fibers' +local busmod = require 'bus' + +local safe = require 'coxpcall' + +local T = {} + +function T.main_rejects_duplicate_service_names() + local ok, err = safe.pcall(function() + runfibers.run(function(scope) + mainmod.run(scope, { + env = 'dev', + bus = busmod.new(), + services_csv = 'hal,hal', + service_loader = function(name) + return { start = function() end } + end, + }) + end) + end) + + assert(ok == false) + assert(tostring(err):match('duplicate service name')) +end + +function T.main_fails_boot_when_service_load_fails() + local ok, err = safe.pcall(function() + runfibers.run(function(scope) + mainmod.run(scope, { + env = 'dev', + bus = busmod.new(), + services_csv = 'hal', + service_loader = function(name) + error('boom') + end, + }) + end) + end) + + assert(ok == false) + assert(tostring(err):match('boot failed')) +end + +return T diff --git a/tests/unit/metrics/config_spec.lua b/tests/unit/metrics/config_spec.lua new file mode 100644 index 00000000..bc25bd53 --- /dev/null +++ b/tests/unit/metrics/config_spec.lua @@ -0,0 +1,140 @@ +-- tests/unit/metrics/config_spec.lua +-- +-- Pure unit tests for services.metrics.config. +-- No fibers needed; all tests are synchronous function calls. + +local conf = require 'services.metrics.config' + +local T = {} + +function T.validate_http_config_valid() + local ok, err = conf.validate_http_config({ + url = 'http://cloud.example.com', + thing_key = 'key', + channels = { { id = 'ch1', name = 'data' } }, + }) + assert(ok == true, 'expected valid http config, got err=' .. tostring(err)) + assert(err == nil, 'expected no error, got ' .. tostring(err)) +end + +function T.validate_http_config_nil() + local ok, err = conf.validate_http_config(nil) + assert(ok == false, 'expected invalid for nil config') + assert(err ~= nil, 'expected error message') +end + +function T.validate_http_config_missing_url() + local ok, err = conf.validate_http_config({ thing_key = 'k', channels = {} }) + assert(ok == false, 'expected invalid config missing url') + assert(err ~= nil, 'expected error message') +end + +function T.merge_config() + local merged = conf.merge_config( + { a = 1, nested = { x = 10, y = 20 } }, + { b = 2, nested = { y = 99, z = 30 } } + ) + assert(merged.a == 1, 'expected merged.a=1') + assert(merged.b == 2, 'expected merged.b=2') + assert(merged.nested.x == 10, 'expected nested.x=10') + assert(merged.nested.y == 99, 'expected nested.y=99 (overridden)') + assert(merged.nested.z == 30, 'expected nested.z=30 (added)') +end + +function T.apply_config_builds_pipeline() + local map, period = conf.apply_config({ + data = { + schema = 'devicecode.config/metrics/1', + publish_period = 30, + pipelines = { + rx_bytes = { + protocol = 'log', + process = { { type = 'DeltaValue' } }, + }, + }, + }, + }) + assert(period == 30, 'expected period=30, got ' .. tostring(period)) + assert(map.rx_bytes ~= nil, 'expected pipeline entry for rx_bytes') + assert(map.rx_bytes.protocol == 'log', 'expected protocol=log') + assert(map.rx_bytes.pipeline ~= nil, 'expected pipeline object') +end + +function T.validate_config_rejects_bad_period() + local ok, _, err = conf.validate_config({ + data = { + schema = 'devicecode.config/metrics/1', + publish_period = -1, + pipelines = { sim = { protocol = 'log', process = {} } }, + }, + }) + assert(ok == false, 'expected invalid config with period=-1') + assert(err ~= nil, 'expected error message') +end + +function T.validate_config_warns_bad_protocol() + local ok, warns = conf.validate_config({ + data = { + schema = 'devicecode.config/metrics/1', + publish_period = 10, + pipelines = { sim = { protocol = 'invalid' } }, + }, + }) + assert(ok == true, 'expected ok=true (warnings, not fatal)') + assert(#warns > 0, 'expected at least one warning for invalid protocol') +end + +function T.validate_config_propagates_invalid_template_to_pipeline() + local ok, warns, err = conf.validate_config({ + data = { + schema = 'devicecode.config/metrics/1', + publish_period = 10, + templates = { + bad_template = { + protocol = 'invalid', + }, + }, + pipelines = { + sim = { + template = 'bad_template', + }, + }, + }, + }) + + assert(ok == true, 'expected ok=true') + assert(err == nil, 'expected no fatal error') + + local saw_template_invalid = false + local saw_metric_uses_invalid_template = false + local saw_metric_invalid_protocol = false + + for _, w in ipairs(warns) do + if w.type == 'template' + and w.endpoint == 'bad_template' + and string.find(w.msg, "invalid protocol 'invalid'", 1, true) + then + saw_template_invalid = true + end + + if w.type == 'metric' + and w.endpoint == 'sim' + and string.find(w.msg, 'uses invalid template [bad_template]', 1, true) + then + saw_metric_uses_invalid_template = true + end + + if w.type == 'metric' + and w.endpoint == 'sim' + and string.find(w.msg, "invalid protocol 'invalid'", 1, true) + then + saw_metric_invalid_protocol = true + end + end + + assert(saw_template_invalid, 'expected warning: template bad_template has invalid protocol') + assert(saw_metric_uses_invalid_template, 'expected warning: sim uses invalid template [bad_template]') + assert(saw_metric_invalid_protocol, "expected warning: sim inherited invalid protocol 'invalid'") +end + +return T diff --git a/tests/unit/metrics/http_spec.lua b/tests/unit/metrics/http_spec.lua new file mode 100644 index 00000000..a03df85c --- /dev/null +++ b/tests/unit/metrics/http_spec.lua @@ -0,0 +1,83 @@ +-- tests/unit/metrics/http_spec.lua +-- +-- Unit test for the HTTP publisher module. +-- Supplies a stub http_ref whose exchange_op records what it receives so no +-- real network traffic is made. Uses runfibers + virtual_time for +-- deterministic timing. + +local fibers = require 'fibers' +local op = require 'fibers.op' +local perform = fibers.perform +local time_harness = require 'tests.support.time_harness' +local virtual_time = require 'tests.support.virtual_time' +local runfibers = require 'tests.support.run_fibers' + +local T = {} + +-- Build a stub http_ref whose exchange_op immediately resolves with the given +-- HTTP status, and records what it was called with. +local function make_fake_ref(captured, reply_status) + local ref = {} + function ref:exchange_op(args) + captured.method = args.method + captured.uri = args.uri + captured.auth = args.headers and args.headers.authorization + captured.content_type = args.headers and args.headers['content-type'] + captured.body_source = args.body_source + return op.always({ result = { status = reply_status or '202', headers = {} } }) + end + return ref +end + +function T.start_http_publisher_sends_expected_request() + local original_http_module = package.loaded['services.metrics.http'] + + local captured = {} + + -- Force a fresh load. + package.loaded['services.metrics.http'] = nil + + local ok_run, run_err = pcall(function() + runfibers.run(function(scope) + local clock = virtual_time.install({ monotonic = 0, realtime = 1700000000 }) + scope:finally(function() clock:restore() end) + + local http_mod = require 'services.metrics.http' + local fake_ref = make_fake_ref(captured, '202') + local worker_scope = scope:child() + + local spawn_ok, spawn_err = worker_scope:spawn(function() + local ch = http_mod.start_http_publisher(fake_ref) + + perform(ch:put_op({ + uri = 'http://localhost:18080/http/channels/ch-data/messages', + auth = 'Thing test-thing-key', + body = '[{"n":"sim","vs":"present"}]', + })) + end) + assert(spawn_ok, tostring(spawn_err)) + + time_harness.flush_ticks(20) + + assert(captured.uri == 'http://localhost:18080/http/channels/ch-data/messages', + 'unexpected uri: ' .. tostring(captured.uri)) + assert(captured.method == 'POST', + 'expected method=POST, got ' .. tostring(captured.method)) + assert(captured.auth == 'Thing test-thing-key', + 'unexpected auth: ' .. tostring(captured.auth)) + assert(captured.content_type == 'application/senml+json', + 'unexpected content-type: ' .. tostring(captured.content_type)) + assert(captured.body_source ~= nil, + 'expected body_source to be set') + + worker_scope:cancel('test done') + perform(worker_scope:join_op()) + end, { timeout = 2.0 }) + end) + + package.loaded['services.metrics.http'] = original_http_module + + assert(ok_run, tostring(run_err)) +end + +return T diff --git a/tests/unit/metrics/processing_spec.lua b/tests/unit/metrics/processing_spec.lua new file mode 100644 index 00000000..4abf75e1 --- /dev/null +++ b/tests/unit/metrics/processing_spec.lua @@ -0,0 +1,120 @@ +-- tests/unit/metrics/processing_spec.lua +-- +-- Pure unit tests for services.metrics.processing blocks. +-- No fibers needed; all tests are synchronous function calls. + +local processing = require 'services.metrics.processing' + +local T = {} + +function T.diff_trigger_absolute() + local trigger = processing.DiffTrigger.new({ + threshold = 5, + diff_method = 'absolute', + initial_val = 10, + }) + local state = trigger:new_state() + + local val, short, err = trigger:run(12, state) + assert(err == nil, tostring(err)) + assert(short == true, 'diff=2 < threshold 5, expected short-circuit') + + val, short, err = trigger:run(16, state) + assert(err == nil, tostring(err)) + assert(short == false, 'diff=6 >= threshold 5, expected pass') + assert(val == 16, 'expected val=16, got ' .. tostring(val)) +end + +function T.diff_trigger_percent() + local trigger = processing.DiffTrigger.new({ + threshold = 10, + diff_method = 'percent', + initial_val = 100, + }) + local state = trigger:new_state() + + local val, short, err = trigger:run(105, state) + assert(err == nil, tostring(err)) + assert(short == true, '5% < 10% threshold, expected short-circuit') + + val, short, err = trigger:run(115, state) + assert(err == nil, tostring(err)) + assert(short == false, '15% >= 10% threshold, expected pass') + assert(val == 115, 'expected val=115, got ' .. tostring(val)) +end + +function T.diff_trigger_any_change() + local trigger = processing.DiffTrigger.new({ + diff_method = 'any-change', + initial_val = 10, + }) + local state = trigger:new_state() + + local val, short, err = trigger:run(10, state) + assert(err == nil, tostring(err)) + assert(short == true, 'same value, expected short-circuit') + + val, short, err = trigger:run(10.1, state) + assert(err == nil, tostring(err)) + assert(short == false, 'value changed, expected pass') + assert(val == 10.1, 'expected val=10.1, got ' .. tostring(val)) +end + +function T.delta_value() + local block = processing.DeltaValue.new({ initial_val = 10 }) + local state = block:new_state() + + local val, short, err = block:run(15, state) + assert(err == nil, tostring(err)) + assert(short == false, 'expected no short-circuit') + assert(val == 5, 'expected delta=5 (15-10), got ' .. tostring(val)) + + block:reset(state) -- simulate publish: last_val = 15 + + val, short, err = block:run(20, state) + assert(err == nil, tostring(err)) + assert(val == 5, 'expected delta=5 (20-15), got ' .. tostring(val)) +end + +function T.pipeline_run_and_reset() + local pipeline, err = processing.new_process_pipeline() + assert(err == nil, tostring(err)) + pipeline:add(processing.DeltaValue.new({ initial_val = 10 })) + + local state = pipeline:new_state() + + local val, short + val, short, err = pipeline:run(20, state) + assert(err == nil, tostring(err)) + assert(short == false, 'expected no short-circuit') + assert(val == 10, 'expected delta=10 (20-10), got ' .. tostring(val)) + + pipeline:reset(state) -- last_val = 20 + + val, short, err = pipeline:run(25, state) + assert(err == nil, tostring(err)) + assert(val == 5, 'expected delta=5 (25-20), got ' .. tostring(val)) +end + +function T.pipeline_short_circuit() + local pipeline, err = processing.new_process_pipeline() + assert(err == nil, tostring(err)) + pipeline:add(processing.DiffTrigger.new({ + diff_method = 'absolute', threshold = 5, initial_val = 10, + })) + pipeline:add(processing.DeltaValue.new({ initial_val = 10 })) + + local state = pipeline:new_state() + + local val, short + val, short, err = pipeline:run(20, state) -- diff=10, passes DiffTrigger + assert(err == nil, tostring(err)) + assert(short == false, 'expected pass through pipeline') + assert(val == 10, 'expected DeltaValue delta=10, got ' .. tostring(val)) + + val, short, err = pipeline:run(22, state) -- diff=2 from last(20), short-circuits + assert(err == nil, tostring(err)) + assert(short == true, 'expected short-circuit (diff=2 < threshold 5)') +end + +return T diff --git a/tests/unit/metrics/senml_spec.lua b/tests/unit/metrics/senml_spec.lua new file mode 100644 index 00000000..ba97d59f --- /dev/null +++ b/tests/unit/metrics/senml_spec.lua @@ -0,0 +1,55 @@ +-- tests/unit/metrics/senml_spec.lua +-- +-- Pure unit tests for services.metrics.senml encoder. +-- No fibers needed; all tests are synchronous function calls. + +local senml = require 'services.metrics.senml' + +local T = {} + +function T.encode_number() + local rec, err = senml.encode('cpu', 42.5) + assert(err == nil, tostring(err)) + assert(rec.n == 'cpu', 'expected n=cpu, got ' .. tostring(rec.n)) + assert(rec.v == 42.5, 'expected v=42.5, got ' .. tostring(rec.v)) +end + +function T.encode_string() + local rec, err = senml.encode('status', 'ok') + assert(err == nil, tostring(err)) + assert(rec.vs == 'ok', 'expected vs=ok, got ' .. tostring(rec.vs)) +end + +function T.encode_boolean() + local rec, err = senml.encode('flag', true) + assert(err == nil, tostring(err)) + assert(rec.vb == true, 'expected vb=true') +end + +function T.encode_with_time() + local rec, err = senml.encode('t', 1, 1000) + assert(err == nil, tostring(err)) + assert(rec.t == 1000, 'expected t=1000, got ' .. tostring(rec.t)) +end + +function T.encode_invalid_value() + local rec, err = senml.encode('k', {}) + assert(rec == nil, 'expected nil record for invalid value') + assert(err ~= nil, 'expected error for invalid value type') +end + +function T.encode_r_flat() + local recs, err = senml.encode_r('dev', { temp = 23.5, status = 'on' }) + assert(err == nil, tostring(err)) + assert(#recs == 2, 'expected 2 records, got ' .. tostring(#recs)) + + local names = {} + for _, r in ipairs(recs) do names[r.n] = r end + + assert(names['dev.temp'] ~= nil, 'expected record dev.temp') + assert(names['dev.temp'].v == 23.5, 'expected dev.temp.v=23.5') + assert(names['dev.status'] ~= nil, 'expected record dev.status') + assert(names['dev.status'].vs == 'on', 'expected dev.status.vs=on') +end + +return T diff --git a/tests/unit/monitor/test_service.lua b/tests/unit/monitor/test_service.lua new file mode 100644 index 00000000..75d1adeb --- /dev/null +++ b/tests/unit/monitor/test_service.lua @@ -0,0 +1,169 @@ +local busmod = require 'bus' +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' + +local run_fibers = require 'tests.support.run_fibers' +local probe = require 'tests.support.bus_probe' +local monitor = require 'services.monitor' + +local T = {} + +local function assert_eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end + +local function assert_true(v, msg) + if not v then error(msg or 'assertion failed', 2) end + return v +end + +local function monitor_rpc(method) + return { 'cap', 'monitor', 'main', 'rpc', method } +end + +local function log_topic(service) + return { 'obs', 'v1', service, 'event', 'log' } +end + +local function start_monitor(scope, bus, opts) + local root = bus:connect({ origin_base = { service = 'monitor-test-root' } }) + local ok, err = scope:spawn(function () monitor.start(root, opts or { env = 'test' }) end) + assert_true(ok, tostring(err)) + local reader = bus:connect({ origin_base = { service = 'monitor-test-reader' } }) + probe.wait_retained_payload(reader, { 'cap', 'monitor', 'main' }, { timeout = 0.5 }) + return root, reader +end + +local function publish_log(conn, service, level, what, summary) + conn:publish(log_topic(service), { + service = service, + level = level or 'info', + what = what, + summary = summary or what, + }) +end + +local function wait_summary_field(conn, pred, label) + return probe.wait_until(function () + local p = conn:call(monitor_rpc('query-logs'), { limit = 0 }, { timeout = 0.05 }) + return p and p.summary and pred(p.summary) + end, { timeout = 0.8, interval = 0.01 }) or error(label or 'summary condition not reached', 2) +end + +function T.query_logs_respects_configured_boot_and_ring_buffers() + run_fibers.run(function (scope) + local b = busmod.new() + local admin = b:connect({ origin_base = { service = 'monitor-test-admin' } }) + admin:retain({ 'cfg', 'monitor' }, { data = { storage = { boot_records = 5, ring_records = 3, boot_seconds = 0 } }, rev = 1 }) + local _, reader = start_monitor(scope, b, { env = 'test' }) + + wait_summary_field(reader, function (s) return s.boot_max_records == 5 and s.ring_max_records == 3 and s.storage_source == 'config' end, + 'monitor storage config was not applied') + + for i = 1, 6 do publish_log(admin, 'alpha', 'info', 'record_' .. i, 'record ' .. i) end + probe.wait_until(function () + local rep = reader:call(monitor_rpc('query-logs'), { service = 'alpha', limit = 10 }, { timeout = 0.05 }) + return rep and rep.count == 3 + end, { timeout = 0.8, interval = 0.01 }) + + local ring = assert_true(reader:call(monitor_rpc('query-logs'), { service = 'alpha', limit = 10 }, { timeout = 0.2 })) + assert_eq(ring.count, 3) + assert_eq(ring.records[1].what, 'record_4') + assert_eq(ring.records[3].what, 'record_6') + + -- The boot buffer stores the first N log records globally. Because this + -- test configures monitor through cfg/monitor before publishing alpha + -- records, the monitor's own storage-configuration log legitimately + -- consumes one boot-buffer slot. Verify both the global cap and the + -- filtered alpha view. + local boot_all = assert_true(reader:call(monitor_rpc('query-logs'), { boot = true, limit = 10 }, { timeout = 0.2 })) + assert_eq(boot_all.count, 5) + local boot = assert_true(reader:call(monitor_rpc('query-logs'), { boot = true, service = 'alpha', limit = 10 }, { timeout = 0.2 })) + assert_eq(boot.count, 4) + assert_eq(boot.records[1].what, 'record_1') + assert_eq(boot.records[4].what, 'record_4') + end) +end + +function T.query_logs_filters_by_level_service_since_and_text() + run_fibers.run(function (scope) + local b = busmod.new() + local admin = b:connect({ origin_base = { service = 'monitor-test-admin' } }) + local _, reader = start_monitor(scope, b, { env = 'test' }) + + publish_log(admin, 'alpha', 'debug', 'debug_one', 'debug detail') + publish_log(admin, 'alpha', 'info', 'info_one', 'contains needle') + publish_log(admin, 'beta', 'warn', 'warn_one', 'other warning') + probe.wait_until(function () + local rep = reader:call(monitor_rpc('query-logs'), { service = 'alpha', min_level = 'debug', limit = 10 }, { timeout = 0.05 }) + return rep and rep.count == 2 + end, { timeout = 0.8, interval = 0.01 }) + + local alpha_info = assert_true(reader:call(monitor_rpc('query-logs'), { service = 'alpha', min_level = 'info', limit = 10 }, { timeout = 0.2 })) + assert_eq(alpha_info.count, 1) + assert_eq(alpha_info.records[1].what, 'info_one') + + local text = assert_true(reader:call(monitor_rpc('query-logs'), { min_level = 'debug', contains = 'needle', limit = 10 }, { timeout = 0.2 })) + assert_eq(text.count, 1) + assert_eq(text.records[1].service, 'alpha') + + local since = assert_true(reader:call(monitor_rpc('query-logs'), { min_level = 'debug', since_id = alpha_info.records[1].id, limit = 10 }, { timeout = 0.2 })) + assert_eq(since.count, 1) + assert_eq(since.records[1].what, 'warn_one') + end) +end + +function T.follow_logs_replays_then_streams_matching_records() + run_fibers.run(function (scope) + local b = busmod.new() + local admin = b:connect({ origin_base = { service = 'monitor-test-admin' } }) + local _, reader = start_monitor(scope, b, { env = 'test' }) + + publish_log(admin, 'alpha', 'info', 'before_follow', 'before follow') + probe.wait_until(function () + local rep = reader:call(monitor_rpc('query-logs'), { service = 'alpha', limit = 1 }, { timeout = 0.05 }) + return rep and rep.count == 1 + end, { timeout = 0.8, interval = 0.01 }) + + local rep = assert_true(reader:call(monitor_rpc('follow-logs'), { service = 'alpha', min_level = 'info', limit = 5, replay = true }, { timeout = 0.2 })) + assert_eq(rep.ok, true) + assert_true(rep.feed, 'expected follow feed') + + local first = fibers.perform(rep.feed:recv_op()) + assert_eq(first.kind, 'log') + assert_eq(first.replay, true) + assert_eq(first.record.what, 'before_follow') + local ready = fibers.perform(rep.feed:recv_op()) + assert_eq(ready.kind, 'ready') + + publish_log(admin, 'alpha', 'info', 'after_follow', 'after follow') + local live = fibers.perform(rep.feed:recv_op()) + assert_eq(live.kind, 'log') + assert_eq(live.replay, false) + assert_eq(live.record.what, 'after_follow') + rep.feed:close('test_done') + end) +end + +function T.set_profile_updates_retained_summary_and_raw_requires_restart() + run_fibers.run(function (scope) + local b = busmod.new() + local _, reader = start_monitor(scope, b, { env = 'test' }) + + local ok_rep = assert_true(reader:call(monitor_rpc('set-profile'), { profile = 'debug' }, { timeout = 0.2 })) + assert_eq(ok_rep.ok, true) + assert_eq(ok_rep.profile, 'debug') + assert_eq(ok_rep.min_level, 'debug') + + probe.wait_until(function () + local rep = reader:call(monitor_rpc('query-logs'), { limit = 0 }, { timeout = 0.05 }) + return rep and rep.summary and rep.summary.profile == 'debug' and rep.summary.min_level == 'debug' + end, { timeout = 0.8, interval = 0.01 }) + + local raw_rep = assert_true(reader:call(monitor_rpc('set-profile'), { profile = 'raw' }, { timeout = 0.2 })) + assert_eq(raw_rep.ok, false) + assert_eq(raw_rep.err, 'raw_profile_requires_restart') + end) +end + +return T diff --git a/tests/unit/net/test_architecture.lua b/tests/unit/net/test_architecture.lua new file mode 100644 index 00000000..9bab4583 --- /dev/null +++ b/tests/unit/net/test_architecture.lua @@ -0,0 +1,110 @@ +-- tests/unit/net/test_architecture.lua + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end + +local function read_file(path) + local f = assert(io.open(path, 'r')) + local s = f:read('*a') + f:close() + return s +end + +local function list_net_files() + local p = io.popen("find ../src/services/net -type f -name '*.lua' | sort") + local out = {} + for line in p:lines() do out[#out + 1] = line end + p:close() + out[#out + 1] = '../src/services/net.lua' + return out +end + +function tests.test_net_service_code_does_not_use_perform_raw() + for _, path in ipairs(list_net_files()) do + local s = read_file(path) + if s:find('perform_raw', 1, true) then fail('perform_raw found in ' .. path) end + end +end + +function tests.test_net_service_code_does_not_call_join_op() + for _, path in ipairs(list_net_files()) do + local s = read_file(path) + if s:find('join_op', 1, true) then fail('join_op found in ' .. path) end + end +end + +function tests.test_net_does_not_require_platform_backend_modules() + for _, path in ipairs(list_net_files()) do + local s = read_file(path) + if s:find("services.hal.backends", 1, true) then + fail('net code must not require HAL backend modules: ' .. path) + end + end +end + +function tests.test_net_uses_priority_event_and_scoped_apply_work() + local events = read_file('../src/services/net/events.lua') + if not events:find("devicecode.support.priority_event", 1, true) then + fail('net events should use priority_event') + end + local apply = read_file('../src/services/net/apply_runtime.lua') + if not apply:find('scoped_work.start', 1, true) then + fail('net apply runtime should use scoped_work') + end +end + +function tests.test_net_config_rejects_legacy_shape_in_code_and_tests() + local cfg = read_file('../src/services/net/config.lua') + if cfg:find('legacy', 1, true) then fail('legacy compatibility found in net config boundary') end + if cfg:find('raw.network', 1, true) then fail('old network shape found in net config boundary') end + if cfg:find('raw.networks', 1, true) then fail('old networks fallback found in net config boundary') end +end + + +function tests.test_net_service_uses_named_runtime_components() + local svc = read_file('../src/services/net/service.lua') + for _, mod in ipairs({ + "devicecode.support.capability_dependencies", + "services.net.observer_manager", + "services.net.wan_manager", + "services.net.drift", + }) do + if not svc:find(mod, 1, true) then fail('net service should use ' .. mod) end + end +end + +function tests.test_net_publisher_has_dirty_publication_path() + local pub = read_file('../src/services/net/publisher.lua') + if not pub:find('publish_dirty_now', 1, true) then fail('net publisher should expose publish_dirty_now') end + if not pub:find('new_dirty_state', 1, true) then fail('net publisher should expose dirty state') end +end + + +function tests.test_net_wan_runtime_uses_explicit_timeout_races_not_bus_timeouts() + local rt = read_file('../src/services/net/wan_runtime.lua') + if not rt:find('sleep.sleep_op', 1, true) or not rt:find('op.named_choice', 1, true) then + fail('net WAN runtime should express timeouts as explicit Op races') + end + if rt:find('timeout = request.max_duration_s', 1, true) or rt:find('timeout = spec.timeout_s', 1, true) then + fail('net WAN runtime should not pass positive hidden bus timeouts to HAL calls') + end + if not rt:find('timeout = false', 1, true) then + fail('net WAN runtime should disable hidden HAL bus timeout on inner calls') + end +end + +function tests.test_hal_network_manager_uses_canonical_control_loop() + local mgr = read_file('../src/services/hal/managers/network.lua') + if not mgr:find("services.hal.support.control_loop", 1, true) then + fail('network manager should use the canonical HAL control loop') + end + if mgr:find('req.reply_ch:put_op', 1, true) then + fail('network manager should not reply directly; use control_loop reply/cancellation path') + end + if mgr:find('fibers.perform(driver_op)', 1, true) then + fail('network manager should return driver Ops rather than performing them inline') + end +end + +return tests diff --git a/tests/unit/net/test_backhaul_model.lua b/tests/unit/net/test_backhaul_model.lua new file mode 100644 index 00000000..300729d2 --- /dev/null +++ b/tests/unit/net/test_backhaul_model.lua @@ -0,0 +1,204 @@ +-- tests/unit/net/test_backhaul_model.lua + +local backhaul = require 'services.net.backhaul_model' + +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function eq(a, b, msg) if a ~= b then fail((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function ok(v, msg) if not v then fail(msg or 'expected truthy') end return v end + +function tests.test_reduces_hal_multiwan_facts_to_semantic_backhaul() + local model = backhaul.reduce({ + segments = { + wan = { kind = 'wan', vlan = { id = 4 } }, + }, + wan = { configured_members = { wan = { interface = 'wan', metric = 10 } } }, + observed = { + snapshot = { + multiwan = { + backend = 'openwrt', + source = 'mwan3', + interfaces_by_semantic = { + wan = { + interface = 'wan', + ifname = 'eth0.2', + state = 'online', + usable = true, + uptime_s = 123, + age_s = 4, + metric = 10, + }, + }, + }, + live = { + interfaces = { + wan = { ipv4 = { { address = '203.0.113.10', mask = 24 } } }, + }, + }, + }, + }, + }, { now = 42 }) + + eq(model.state, 'ok') + local wan = ok(model.uplinks.wan, 'wan uplink expected') + eq(wan.state, 'online') + eq(wan.usable, true) + eq(wan.uptime_s, 123) + eq(wan.source.kind, 'host-multiwan') + eq(wan.source.tool, 'mwan3') + eq(wan.status_source.tool, 'mwan3') + eq(wan.link.kind, 'wired') + eq(wan.link.vlan, 4) + eq(wan.path_address.address, '203.0.113.10') +end + + +function tests.test_backhaul_uses_mwan3_for_status_and_endpoint_for_device_name() + local model = backhaul.reduce({ + interfaces = { + wan = { endpoint = { ifname = 'vl-wan' } }, + }, + wan = { configured_members = { wan = { interface = 'wan', metric = 10 } } }, + observed = { + snapshot = { + multiwan = { + backend = 'openwrt', + source = 'mwan3', + interfaces_by_semantic = { + wan = { interface = 'wan', state = 'online', usable = true }, + }, + }, + }, + }, + }, { now = 42 }) + + local wan = ok(model.uplinks.wan, 'wan uplink expected') + eq(wan.state, 'online') + eq(wan.usable, true) + eq(wan.ifname, 'vl-wan') + eq(wan.source.tool, 'mwan3') + eq(wan.status_source.tool, 'mwan3') +end + +function tests.test_gsm_uplink_uses_mwan3_status_not_gsm_connection_state() + local model = backhaul.reduce({ + wan = { + configured_members = { + gsm_primary = { interface = 'modem_primary', source = { kind = 'gsm-uplink', id = 'primary' } }, + }, + }, + sources = { + gsm_uplinks = { + primary = { + state = 'sim_absent', + connected = false, + linux = { ifname = 'wwan0' }, + }, + }, + }, + observed = { + snapshot = { + multiwan = { + backend = 'openwrt', + source = 'mwan3', + interfaces_by_semantic = { + modem_primary = { + interface = 'modem_primary', + ifname = 'wwan0', + state = 'online', + usable = true, + }, + }, + }, + live = { interfaces = { wwan0 = { ipv4 = { { address = '10.1.2.3' } } } } }, + }, + }, + }, { now = 42 }) + + local uplink = ok(model.uplinks.gsm_primary, 'gsm uplink expected') + eq(uplink.state, 'online') + eq(uplink.usable, true) + eq(uplink.observed, true) + eq(uplink.source.kind, 'gsm-uplink') + eq(uplink.source.id, 'primary') + eq(uplink.status_source.kind, 'host-multiwan') + eq(uplink.status_source.tool, 'mwan3') + eq(uplink.link.kind, 'cellular') + eq(uplink.ifname, 'wwan0') + eq(uplink.path_address.address, '10.1.2.3') + eq(uplink.gsm, nil) +end + +function tests.test_gsm_uplink_remains_offline_when_mwan3_is_offline_even_if_gsm_connected() + local model = backhaul.reduce({ + wan = { + configured_members = { + gsm_primary = { interface = 'modem_primary', source = { kind = 'gsm-uplink', id = 'primary' } }, + }, + }, + sources = { + gsm_uplinks = { + primary = { + state = 'connected', + connected = true, + linux = { ifname = 'wwan0' }, + }, + }, + }, + observed = { + snapshot = { + multiwan = { + backend = 'openwrt', + source = 'mwan3', + interfaces_by_semantic = { + modem_primary = { + interface = 'modem_primary', + ifname = 'wwan0', + state = 'offline', + usable = false, + }, + }, + }, + }, + }, + }, { now = 42 }) + + local uplink = ok(model.uplinks.gsm_primary, 'gsm uplink expected') + eq(uplink.state, 'offline') + eq(uplink.usable, false) + eq(uplink.ifname, 'wwan0') +end + +function tests.test_gsm_member_stays_present_without_gsm_details() + local model = backhaul.reduce({ + wan = { + configured_members = { + gsm_primary = { interface = 'modem_primary', source = { kind = 'gsm-uplink', id = 'primary' } }, + }, + }, + observed = { + snapshot = { + multiwan = { + backend = 'openwrt', + source = 'mwan3', + interfaces_by_semantic = { + modem_primary = { + interface = 'modem_primary', + ifname = 'wwan0', + state = 'offline', + usable = false, + }, + }, + }, + }, + }, + }, { now = 42 }) + + local uplink = ok(model.uplinks.gsm_primary, 'gsm uplink expected') + eq(uplink.state, 'offline') + eq(uplink.usable, false) + eq(uplink.source.id, 'primary') + eq(uplink.ifname, 'wwan0') +end + +return tests diff --git a/tests/unit/net/test_config.lua b/tests/unit/net/test_config.lua new file mode 100644 index 00000000..b07a1642 --- /dev/null +++ b/tests/unit/net/test_config.lua @@ -0,0 +1,237 @@ +-- tests/unit/net/test_config.lua + +local config = require 'services.net.config' + +local tests = {} + +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end +local function eq(a, b, msg) if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end end + +local function sample_cfg() + return { + schema = config.SCHEMA, + version = 1, + product = 'bigbox', + segments = { + lan = { + kind = 'lan', + vlan = { id = 10 }, + addressing = { ipv4 = { mode = 'static', cidr = '172.28.10.1/24' } }, + dhcp = { enabled = true, start = 10, limit = 100, lease_time = '12h' }, + firewall = { zone = 'lan' }, + }, + guest = { + kind = 'guest', + vlan = 30, + firewall = { zone = 'guest', isolation = 'internet_only' }, + }, + }, + interfaces = { + lan_bridge = { + kind = 'bridge', + role = 'lan', + segment = 'lan', + members = { 'cm5_lan' }, + }, + wan_modem_a = { + kind = 'cellular', + role = 'wan', + endpoint = { selector = 'modem.primary' }, + }, + }, + wan = { + load_balancing = { policy = 'balanced' }, + rules = { https = { family = 'ipv4', proto = 'tcp', dest_port = '443', policy = 'balanced', sticky = true } }, + members = { + gsm_a = { interface = 'wan_modem_a', weight = 70, mwan_metric = 1 }, + }, + }, + firewall = { + zones = { lan = {}, guest = {}, wan = {} }, + policies = { guest_to_wan = { from = 'guest', to = 'wan', action = 'allow' } }, + }, + routing = { + routes = {}, + rules = {}, + }, + dns = { + upstreams = { '1.1.1.1', '8.8.8.8' }, + host_files = { base_dir = '/data/devicecode/dns/hosts', sources = { ads = { file = 'ads.hosts' } } }, + }, + dhcp = { + defaults = { lease_time = '12h' }, + reservations = {}, + }, + vpn = { + enabled = true, + tunnels = { management = { kind = 'wireguard' } }, + }, + diagnostics = { + reflectors = { cloud = { address = '1.1.1.1' } }, + }, + runtime = { + apply = { debounce_s = 0.25 }, + observe = { interval_s = 5 }, + }, + } +end + +function tests.test_accepts_only_current_cfg_net_schema() + local intent = ok(config.normalise(sample_cfg(), { rev = 7, generation = 3 })) + eq(intent.schema, config.INTENT_SCHEMA) + eq(intent.config_schema, config.SCHEMA) + eq(intent.rev, 7) + eq(intent.generation, 3) + eq(intent.version, 1) + ok(intent.segments.lan, 'lan segment expected') + ok(intent.interfaces.lan_bridge, 'lan interface expected') + eq(intent.segments.guest.vlan.id, 30) + eq(intent.stats.segments, 2) + eq(intent.stats.interfaces, 2) + eq(intent.stats.wan_members, 1) + eq(intent.wan.members.gsm_a.metric, 1) + eq(intent.wan.members.gsm_a.mwan_metric, nil) + eq(intent.wan.rules.https.proto, 'tcp') + eq(intent.wan.rules.https.dest_port, '443') + eq(intent.wan.rules.https.policy, 'balanced') + eq(intent.wan.rules.https.sticky, true) + eq(intent.stats.vpn_tunnels, 1) +end + +function tests.test_accepts_config_service_record_shape_without_legacy_migration() + local intent = ok(config.normalise({ rev = 12, data = sample_cfg() }, { generation = 4 })) + eq(intent.rev, 12) + eq(intent.generation, 4) + eq(intent.wan.policy, nil) +end + + + + + +function tests.test_rejects_top_level_shaping() + local cfg = sample_cfg() + cfg.shaping = { enabled = true } + local intent, err = config.normalise(cfg, { rev = 1 }) + if intent ~= nil then error('expected top-level shaping to be rejected', 2) end + ok(err and err:find('cfg.net.shaping', 1, true), 'top-level shaping rejection error expected') +end + +function tests.test_rejects_low_level_segment_shaping_fields() + local cfg = sample_cfg() + cfg.segments.lan.shaping = { egress = { enabled = true } } + local intent, err = config.normalise(cfg, { rev = 1 }) + if intent ~= nil then error('expected segment shaping egress to be rejected', 2) end + ok(err and err:find('net.segments.lan.shaping.egress', 1, true), 'segment shaping egress rejection error expected') +end + +function tests.test_rejects_low_level_wan_shaping_fields() + local cfg = sample_cfg() + cfg.wan.members.gsm_a.shaping = { download_limit = '80mbit' } + local intent, err = config.normalise(cfg, { rev = 1 }) + if intent ~= nil then error('expected wan member shaping download_limit to be rejected', 2) end + ok(err and err:find('net.wan.members.gsm_a.shaping.download_limit', 1, true), 'wan shaping download_limit rejection error expected') +end + +function tests.test_rejects_implicit_static_route_without_kind() + local cfg = sample_cfg() + cfg.routing.routes.legacy = { target = '192.168.100.1', interface = 'wan_modem_a' } + local intent, err = config.normalise(cfg, { rev = 1 }) + if intent ~= nil then error('expected route without kind to be rejected', 2) end + ok(err and err:find('net.routing.routes.legacy.kind', 1, true), 'route kind error expected') +end + +function tests.test_rejects_subnet_route_without_netmask() + local cfg = sample_cfg() + cfg.routing.routes.bad_subnet = { kind = 'subnet', target = '192.168.100.0', interface = 'wan_modem_a' } + local intent, err = config.normalise(cfg, { rev = 1 }) + if intent ~= nil then error('expected subnet route without netmask to be rejected', 2) end + ok(err and err:find('net.routing.routes.bad_subnet.netmask', 1, true), 'route netmask error expected') +end + +function tests.test_rejects_unknown_route_interface() + local cfg = sample_cfg() + cfg.routing.routes.starlink_admin = { kind = 'host', target = '192.168.100.1', interface = 'missing' } + local intent, err = config.normalise(cfg, { rev = 1 }) + if intent ~= nil then error('expected unknown route interface to be rejected', 2) end + ok(err and err:find('net.routing.routes.starlink_admin.interface', 1, true), 'route interface error expected') +end + +function tests.test_rejects_mwan_rule_for_unknown_policy() + local cfg = sample_cfg() + cfg.wan.rules.https.policy = 'not_declared' + local intent, err = config.normalise(cfg, { rev = 1 }) + eq(intent, nil) + ok(err and err:find('wan%.rules%.https%.policy', 1, false), 'policy error expected') +end + +function tests.test_rejects_missing_or_wrong_schema() + local intent, err = config.normalise({ segments = {} }, { rev = 1 }) + if intent ~= nil then error('expected config without schema to be rejected', 2) end + ok(err and err:find('devicecode.config/net/1', 1, true), 'schema error expected') + + intent, err = config.normalise({ schema = 'legacy', network = {} }, { rev = 1 }) + if intent ~= nil then error('expected legacy network shape to be rejected', 2) end + ok(err and err:find('devicecode.config/net/1', 1, true), 'legacy shape must be rejected') +end + +function tests.test_rejects_arrays_for_core_maps() + local cfg = sample_cfg() + cfg.segments = { { id = 'lan', kind = 'lan' } } + local intent, err = config.normalise(cfg, { rev = 1 }) + if intent ~= nil then error('expected array segments to be rejected', 2) end + ok(err and err:find('map keyed by id', 1, true), 'map keyed by id error expected') +end + + +local function read_project_file(rel) + local candidates = { rel, '../' .. rel } + for i = 1, #candidates do + local f = io.open(candidates[i], 'rb') + if f then local data = f:read('*a'); f:close(); return data end + end + return nil, 'unable to read ' .. rel +end + +function tests.test_bigbox_config_uses_clean_segment_authority_shape() + local cjson = require 'cjson.safe' + local text = ok(read_project_file('src/configs/bigbox-v1-cm-2.json')) + local doc = ok(cjson.decode(text), 'bigbox config must decode') + local intent = ok(config.normalise(doc.net, { generation = 1 })) + eq(intent.dhcp.pools, nil, 'top-level dhcp pools must not be authoritative') + eq(intent.segments.jan.dhcp.enabled, true) + eq(intent.segments.jan.dns.host_files[1], 'ads') + eq(intent.segments.jan.dns.host_files[2], 'adult') + eq(intent.segments.jan.shaping.host_default.download.sustained_rate, '500kbit') + eq(intent.segments.jan.shaping.host_default.download.peak_rate, '2mbit') + eq(intent.segments.jan.shaping.host_default.upload.sustained_rate, '250kbit') + eq(intent.segments.jan.shaping.host_default.upload.peak_rate, '1mbit') + eq(intent.segments.jan.shaping.download, nil) + eq(intent.segments.jan.shaping.upload, nil) + eq(intent.dns.records['config.bigbox.home'].address, '172.28.8.1') + eq(intent.routing.routes.starlink_admin.kind, 'host') + eq(intent.routing.routes.starlink_admin.interface, 'wan') + eq(intent.routing.routes.starlink_admin.target, '192.168.100.1') + eq(intent.routing.routes.starlink_admin.netmask, '255.255.255.255') + ok(intent.routing.routes.starlink_admin.description and intent.routing.routes.starlink_admin.description:find('Starlink', 1, true), 'Starlink route should carry context') + eq(intent.firewall.rules.Allow_DNS_queries_RST.dest_port, '53') +end + + +function tests.test_rejects_cross_domain_unknown_segment_reference() + local cfg = sample_cfg() + cfg.interfaces.bad = { kind = 'bridge', segment = 'missing' } + local intent, err = config.normalise(cfg, { rev = 1 }) + if intent ~= nil then error('expected unknown segment reference to be rejected', 2) end + ok(err and err:find('unknown segment', 1, true), 'unknown segment error expected') +end + +function tests.test_rejects_unknown_wan_interface_when_interface_catalogue_present() + local cfg = sample_cfg() + cfg.wan.members.bad = { interface = 'missing' } + local intent, err = config.normalise(cfg, { rev = 1 }) + if intent ~= nil then error('expected unknown WAN interface to be rejected', 2) end + ok(err and err:find('unknown interface', 1, true), 'unknown interface error expected') +end + +return tests diff --git a/tests/unit/net/test_gsm_uplink_watch.lua b/tests/unit/net/test_gsm_uplink_watch.lua new file mode 100644 index 00000000..cfd3953e --- /dev/null +++ b/tests/unit/net/test_gsm_uplink_watch.lua @@ -0,0 +1,35 @@ +-- tests/unit/net/test_gsm_uplink_watch.lua + +local watch = require 'services.net.gsm_uplink_watch' + +local tests = {} +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end +local function eq(a, b, msg) if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end end + +function tests.test_maps_canonical_retained_event() + local ev = watch._test.map_event({ op = 'retain', topic = { 'state', 'gsm', 'uplink', 'primary' }, payload = { connected = true, linux = { ifname = 'wwan1' } } }) + eq(ev.kind, 'gsm_uplink_changed') + eq(ev.role, 'primary') + eq(ev.payload.schema, 'devicecode.gsm.uplink/1') + eq(ev.payload.linux.ifname, 'wwan1') +end + +function tests.test_maps_unretain_to_unavailable_state() + local ev = watch._test.map_event({ op = 'unretain', topic = { 'state', 'gsm', 'uplink', 'secondary' } }) + eq(ev.kind, 'gsm_uplink_changed') + eq(ev.role, 'secondary') + eq(ev.payload.state, 'unavailable') + eq(ev.payload.connected, false) +end + +function tests.test_maps_replay_done() + local ev = watch._test.map_event({ op = 'replay_done' }) + eq(ev.kind, 'gsm_uplink_replay_done') +end + +function tests.test_malformed_topic_is_unknown() + local ev = watch._test.map_event({ op = 'retain', topic = { 'state', 'gsm', 'modem', 'primary', 'uplink' }, payload = {} }) + eq(ev.kind, 'gsm_uplink_unknown') +end + +return tests diff --git a/tests/unit/net/test_hal_client.lua b/tests/unit/net/test_hal_client.lua new file mode 100644 index 00000000..171d1d91 --- /dev/null +++ b/tests/unit/net/test_hal_client.lua @@ -0,0 +1,123 @@ +-- tests/unit/net/test_hal_client.lua + +local fibers = require 'fibers' +local op = require 'fibers.op' + +local hal_client = require 'services.net.hal_client' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function ok(v, msg) if not v then fail(msg) end return v end +local function eq(a, b, msg) if a ~= b then fail((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function contains(s, needle, msg) if type(s) ~= 'string' or not s:find(needle, 1, true) then fail(msg or ('expected ' .. tostring(s) .. ' to contain ' .. tostring(needle))) end end + +function tests.test_missing_hal_fails_by_default() + fibers.run(function () + local client = hal_client.new(nil, {}) + local result = fibers.perform(client:apply_intent_op({ rev = 1 }, {})) + eq(result.ok, false) + contains(result.err, 'network-config HAL capability not configured') + ok(result.reason and result.reason.code == 'missing_network_config_hal', 'structured missing-hal reason expected') + end) +end + +function tests.test_missing_hal_succeeds_only_when_explicit_dry_run() + fibers.run(function () + local client = hal_client.new(nil, { dry_run = true }) + local result = fibers.perform(client:apply_intent_op({ rev = 1 }, {})) + eq(result.ok, true) + eq(result.dry_run, true) + eq(result.applied, false) + end) +end + +function tests.test_structured_failure_reason_is_preserved() + fibers.run(function () + local cap = { + call_control_op = function () + return op.always({ + ok = false, + code = 409, + reason = { + code = 'backend_failed', + err = 'backend said no', + detail = { field = 'segments.lan' }, + }, + }) + end, + } + local client = hal_client.new(nil, { network_config_cap = cap }) + local result = fibers.perform(client:apply_intent_op({ rev = 1 }, {})) + eq(result.ok, false) + eq(result.err, 'backend said no') + eq(result.code, 409) + eq(result.reason.code, 'backend_failed') + eq(result.reason.detail.field, 'segments.lan') + end) +end + + +function tests.test_network_state_watch_and_subscription_are_exposed() + fibers.run(function () + local called = false + local cap = { + call_control_op = function (_, method, args, opts) + called = true + eq(method, 'watch') + ok(type(args) == 'table') + ok(type(opts) == 'table') + return op.always({ ok = true, reason = { ok = true, watching = true } }) + end, + get_event_sub = function (_, name, opts) + eq(name, 'observed') + return { name = name, opts = opts } + end, + } + local client = hal_client.new({}, { network_state_cap = cap }) + local sub = client:open_observed_subscription({ queue_len = 7 }) + eq(sub.name, 'observed') + eq(sub.opts.queue_len, 7) + local result = fibers.perform(client:start_observation_op({ debounce_s = 0.01 })) + eq(result.ok, true) + eq(result.watching, true) + ok(called, 'watch should have been called') + end) +end + + +function tests.test_live_weight_shaping_and_speedtest_capabilities_are_exposed() + fibers.run(function () + local config_calls = {} + local diag_calls = {} + local config_cap = { + call_control_op = function (_, method, args, opts) + config_calls[#config_calls + 1] = { method = method, args = args, opts = opts } + return op.always({ ok = true, reason = { ok = true, method = method } }) + end, + } + local diag_cap = { + call_control_op = function (_, method, args, opts) + diag_calls[#diag_calls + 1] = { method = method, args = args, opts = opts } + return op.always({ ok = true, reason = { ok = true, peak_mbps = 12.5 } }) + end, + } + local client = hal_client.new({}, { network_config_cap = config_cap, network_diagnostics_cap = diag_cap }) + local r1 = fibers.perform(client:apply_live_weights_op({ members = {} }, { timeout = 1 })) + eq(r1.ok, true) + eq(config_calls[1].method, 'apply_live_weights') + local r2 = fibers.perform(client:apply_shaping_op({ links = {} }, { timeout = 1 })) + eq(r2.ok, true) + eq(config_calls[2].method, 'apply_shaping') + local r3 = fibers.perform(client:speedtest_op({ interface = 'wan_a' }, { timeout = 1 })) + eq(r3.ok, true) + eq(r3.peak_mbps, 12.5) + eq(diag_calls[1].method, 'speedtest') + local r4 = fibers.perform(client:read_counters_op({ interfaces = { 'wan_a' } }, { timeout = 1 })) + eq(r4.ok, true) + eq(diag_calls[2].method, 'read_counters') + eq(diag_calls[2].args.interfaces[1], 'wan_a') + end) +end + +return tests diff --git a/tests/unit/net/test_intent_realiser.lua b/tests/unit/net/test_intent_realiser.lua new file mode 100644 index 00000000..5f0507c9 --- /dev/null +++ b/tests/unit/net/test_intent_realiser.lua @@ -0,0 +1,60 @@ +-- tests/unit/net/test_intent_realiser.lua + +local config = require 'services.net.config' +local realiser = require 'services.net.intent_realiser' + +local tests = {} +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end +local function eq(a, b, msg) if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end end + +local function base_cfg() + return { + schema = config.SCHEMA, + version = 1, + segments = { + wan = { kind = 'wan', vlan = { id = 4 }, firewall = { zone = 'wan' }, addressing = { ipv4 = { mode = 'dhcp', peerdns = false } } }, + }, + interfaces = {}, + firewall = { zones = { wan = { masq = true } }, policies = {} }, + routing = {}, dns = {}, dhcp = {}, vpn = {}, diagnostics = {}, + wan = { + enabled = true, + members = { + wan = { interface = 'wan', mwan_metric = 1, weight = 1 }, + modem_primary = { interface = 'modem_primary', mwan_metric = 1, weight = 1, source = { kind = 'gsm-uplink', id = 'primary' } }, + modem_secondary = { interface = 'modem_secondary', mwan_metric = 1, weight = 1, source = { kind = 'gsm-uplink', id = 'secondary' } }, + }, + }, + } +end + +function tests.test_unavailable_gsm_uplinks_are_not_realised() + local intent = ok(config.normalise(base_cfg(), { rev = 1, generation = 1 })) + local realised = realiser.realise(intent, { gsm_uplinks = {} }) + eq(realised.wan.members.wan.interface, 'wan') + eq(realised.wan.members.modem_primary, nil) + eq(realised.wan.members.modem_secondary, nil) + eq(realised.interfaces.modem_primary, nil) + eq(realised.segments.wan.addressing.ipv4.metric, 11) +end + +function tests.test_gsm_uplink_ifname_realises_modem_interface() + local intent = ok(config.normalise(base_cfg(), { rev = 1, generation = 1 })) + local realised = realiser.realise(intent, { gsm_uplinks = { primary = { linux = { ifname = 'wwan1' } } } }) + local iface = ok(realised.interfaces.modem_primary) + eq(iface.role, 'wan') + eq(iface.endpoint.ifname, 'wwan1') + eq(iface.addressing.ipv4.metric, 12) + eq(realised.wan.members.modem_primary.interface, 'modem_primary') + eq(realised.wan.members.modem_secondary, nil) +end + +function tests.test_rejects_non_gsm_uplink_source() + local cfg = base_cfg() + cfg.wan.members.modem_primary.source = { kind = 'unsupported-source', id = 'primary' } + local intent, err = config.normalise(cfg, { rev = 1, generation = 1 }) + eq(intent, nil) + ok(err and err:find('gsm%-uplink'), 'gsm-uplink error expected') +end + +return tests diff --git a/tests/unit/net/test_service_behaviour.lua b/tests/unit/net/test_service_behaviour.lua new file mode 100644 index 00000000..9b070430 --- /dev/null +++ b/tests/unit/net/test_service_behaviour.lua @@ -0,0 +1,920 @@ +-- tests/unit/net/test_service_behaviour.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local busmod = require 'bus' + +local service = require 'services.net.service' +local cfg_mod = require 'services.net.config' +local topics = require 'services.net.topics' +local probe = require 'tests.support.bus_probe' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function ok(v, msg) if not v then fail(msg) end return v end +local function eq(a, b, msg) if a ~= b then fail((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) end end +local function contains(s, needle, msg) if type(s) ~= 'string' or not s:find(needle, 1, true) then fail(msg or ('expected ' .. tostring(s) .. ' to contain ' .. tostring(needle))) end end + +local function cfg() + return { + schema = cfg_mod.SCHEMA, + version = 1, + segments = { + lan = { kind = 'lan', vlan = 10, addressing = { ipv4 = { mode = 'static', cidr = '172.28.10.1/24' } } }, + }, + interfaces = { + lan_bridge = { kind = 'bridge', role = 'lan', segment = 'lan', members = { 'eth0' } }, + }, + wan = { members = {} }, + } +end + +local function success_hal(calls) + return { + available = function () return true end, + apply_intent_op = function (_, intent, opts) + calls[#calls + 1] = { intent = intent, opts = opts } + return op.always({ + ok = true, + applied = true, + changed = true, + backend = 'test', + intent_rev = intent.rev, + }) + end, + } +end + + +local function network_config_status_topic() + return { 'cap', 'network-config', 'main', 'status' } +end + +local function network_config_apply_topic() + return { 'cap', 'network-config', 'main', 'rpc', 'apply' } +end + +local function network_state_status_topic() + return { 'cap', 'network-state', 'main', 'status' } +end + +local function network_state_watch_topic() + return { 'cap', 'network-state', 'main', 'rpc', 'watch' } +end + +local function retain_network_config_status_payload(conn, payload) + return conn:retain(network_config_status_topic(), payload) +end + +local function retain_network_state_status_payload(conn, payload) + return conn:retain(network_state_status_topic(), payload) +end + +local function retain_network_config_status(conn, status) + return conn:retain(network_config_status_topic(), { + schema = 'devicecode.cap.status/1', + state = status, + available = status == 'available' or status == 'running', + }) +end + +local function bind_network_config_apply(scope, conn, calls, reply_fn) + local ep = assert(conn:bind(network_config_apply_topic(), { queue_len = 8 })) + local spawned, err = scope:spawn(function () + while true do + local req = fibers.perform(ep:recv_op()) + if not req then return end + local payload = req.payload or {} + calls[#calls + 1] = payload + local reply = reply_fn and reply_fn(req, payload) or { + ok = true, + reason = { + ok = true, + applied = true, + changed = true, + backend = 'test-cap', + intent_rev = payload.intent and payload.intent.rev, + }, + } + req:reply(reply) + end + end) + ok(spawned, err) + return ep +end + +local function bind_network_state_watch(scope, conn, calls, reply_fn) + local ep = assert(conn:bind(network_state_watch_topic(), { queue_len = 8 })) + local spawned, err = scope:spawn(function () + while true do + local req = fibers.perform(ep:recv_op()) + if not req then return end + local payload = req.payload or {} + calls[#calls + 1] = payload + local reply = reply_fn and reply_fn(req, payload) or { + ok = true, + reason = { ok = true, watching = true, backend = 'test-cap' }, + } + req:reply(reply) + end + end) + ok(spawned, err) + return ep +end + +local function start_service(scope, conn, params) + local child = ok(scope:child()) + local spawned, err = child:spawn(function () + service.run(child, params) + end) + ok(spawned, err) + return child +end + +function tests.test_initial_config_starts_apply_and_publishes_running_state() + fibers.run(function (scope) + local b = busmod.new() + local conn = b:connect() + local reader = b:connect() + local calls = {} + + retain_network_config_status(conn, 'available') + + local child = start_service(scope, conn, { + conn = conn, + config = cfg(), + rev = 17, + hal = success_hal(calls), + }) + + local view = reader:retained_view(topics.summary()) + local summary = probe.wait_versioned_until('net running summary', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + return msg and msg.payload and msg.payload.state == 'running' and msg.payload or nil + end, + { timeout = 0.5 }) + eq(summary.service, 'net') + eq(summary.state, 'running') + eq(summary.ready, true) + eq(summary.generation, 1) + eq(summary.apply.state, 'applied') + eq(summary.apply.last_applied_rev, 17) + eq(summary.dependencies.network_config.status, 'available') + eq(summary.counts.segments, 1) + eq(#calls, 1) + eq(calls[1].intent.rev, 17) + eq(calls[1].opts.generation, 1) + eq(calls[1].opts.apply_id, 1) + + local seg_msg = view:get(topics.segment('lan')) + local seg = seg_msg and seg_msg.payload or probe.wait_retained_payload(reader, topics.segment('lan'), { timeout = 0.2 }) + eq(seg.kind, 'lan') + view:close() + + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + + + +function tests.test_backhaul_publishes_configured_gsm_members_even_when_unrealised() + fibers.run(function (scope) + local b = busmod.new() + local conn = b:connect() + local reader = b:connect() + local calls = {} + + local c = cfg() + c.segments.wan = { kind = 'wan', vlan = { id = 4 }, firewall = { zone = 'wan' }, addressing = { ipv4 = { mode = 'dhcp' } } } + c.wan = { + members = { + wan = { interface = 'wan', weight = 1 }, + modem_primary = { interface = 'modem_primary', source = { kind = 'gsm-uplink', id = 'primary' }, weight = 1 }, + modem_secondary = { interface = 'modem_secondary', source = { kind = 'gsm-uplink', id = 'secondary' }, weight = 1 }, + }, + } + + retain_network_config_status(conn, 'available') + local child = start_service(scope, conn, { + conn = conn, + config = c, + rev = 32, + hal = success_hal(calls), + observe = false, + }) + + local summary_view = reader:retained_view(topics.summary()) + probe.wait_versioned_until('net running summary with configured gsm wan members', + function () return summary_view:version() end, + function (seen) return summary_view:changed_op(seen) end, + function () + local msg = summary_view:get(topics.summary()) + return msg and msg.payload and msg.payload.state == 'running' and msg.payload or nil + end, + { timeout = 0.5 }) + + local wan_domain = probe.wait_retained_payload(reader, topics.domain('wan'), { timeout = 0.2 }) + ok(wan_domain.configured_members and wan_domain.configured_members.wan, 'configured WAN catalogue expected') + ok(wan_domain.configured_members.modem_primary, 'configured primary modem member expected in NET model') + ok(wan_domain.realised_members and wan_domain.realised_members.wan, 'realised wired WAN member expected') + eq(wan_domain.realised_members.modem_primary, nil) + eq(wan_domain.members, nil) + + local backhaul = probe.wait_retained_payload(reader, topics.domain('backhaul'), { timeout = 0.2 }) + ok(backhaul.uplinks.wan, 'wired WAN member expected') + ok(backhaul.uplinks.modem_primary, 'configured primary modem WAN member expected') + ok(backhaul.uplinks.modem_secondary, 'configured secondary modem WAN member expected') + eq(backhaul.uplinks.modem_primary.state, 'unknown') + eq(backhaul.uplinks.modem_primary.observed, false) + eq(backhaul.uplinks.modem_primary.source.kind, 'gsm-uplink') + eq(backhaul.uplinks.modem_primary.source.id, 'primary') + -- HAL apply still receives only the realised member set: without GSM ifnames, + -- modem interfaces are not written to OpenWrt. + eq(calls[1].intent.wan.members.modem_primary, nil) + eq(calls[1].intent.wan.members.modem_secondary, nil) + + summary_view:close() + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + +function tests.test_counter_metrics_publish_generic_topics_with_member_namespaces() + fibers.run(function (scope) + local b = busmod.new() + local conn = b:connect() + local sub_conn = b:connect() + local calls = {} + local counter_calls = {} + local sub = sub_conn:subscribe({ 'obs', 'v1', 'net', 'metric', '+' }, { queue_len = 256, full = 'drop_oldest' }) + + local hal = success_hal(calls) + hal.read_counters_op = function (_, req) + counter_calls[#counter_calls + 1] = req + local counters = {} + local bases = { adm = 1000, jan = 2000, wan = 3000, modem_primary = 4000, modem_secondary = 5000 } + for _, iface in ipairs(req.interfaces or {}) do + local stats = {} + local base = bases[iface] or 0 + for i, stat in ipairs(req.stats or {}) do stats[stat] = base + i end + counters[iface] = { statistics = stats } + end + return op.always({ ok = true, backend = 'test', counters = counters }) + end + + local c = cfg() + c.segments.adm = { kind = 'lan', addressing = { ipv4 = { mode = 'static', cidr = '172.28.8.1/24' } } } + c.segments.jan = { kind = 'lan', addressing = { ipv4 = { mode = 'static', cidr = '172.28.32.1/24' } } } + c.segments.wan = { kind = 'wan', addressing = { ipv4 = { mode = 'dhcp' } }, dhcp = { enabled = false } } + c.wan = { + members = { + modem_primary = { interface = 'modem_primary', source = { kind = 'gsm-uplink', id = 'primary' } }, + modem_secondary = { interface = 'modem_secondary', source = { kind = 'gsm-uplink', id = 'secondary' } }, + }, + } + + conn:retain(topics.gsm_uplink('primary'), { connected = true, available = true, linux = { ifname = 'wwan0' } }) + conn:retain(topics.gsm_uplink('secondary'), { connected = true, available = true, linux = { ifname = 'wwan1' } }) + retain_network_config_status(conn, 'available') + local child = start_service(scope, conn, { + conn = conn, + config = c, + rev = 31, + hal = hal, + observe = false, + counter_poll_interval_s = 0.05, + }) + + local view = sub_conn:retained_view(topics.summary()) + probe.wait_versioned_until('net counter metrics running summary', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + return msg and msg.payload and msg.payload.state == 'running' and msg.payload or nil + end, + { timeout = 0.5 }) + view:close() + + local expected = { + ['net.adm.rx_bytes'] = 1001, + ['net.jan.tx_errors'] = 2008, + ['net.wan.rx_packets'] = 3002, + ['net.modem_primary.rx_dropped'] = 4003, + ['net.modem_secondary.tx_packets'] = 5006, + } + local seen = {} + local deadline = fibers.now() + 2.0 + while fibers.now() < deadline do + local which, msg = fibers.perform(op.named_choice({ + msg = sub:recv_op(), + timeout = sleep.sleep_op(0.05), + })) + if which == 'msg' and msg and msg.payload then + local payload = msg.payload + local key = table.concat(payload.namespace or {}, '.') + if expected[key] ~= nil then + local fields = 0 + for _ in pairs(payload) do fields = fields + 1 end + eq(fields, 2, 'metric payload must only contain value and namespace') + eq(msg.topic[5], payload.namespace[3], 'generic metric topic should match stat class') + seen[key] = payload.value + end + local complete = true + for key in pairs(expected) do if seen[key] == nil then complete = false end end + if complete then break end + end + end + local requested = counter_calls[1] and table.concat(counter_calls[1].interfaces or {}, ',') or 'none' + for key, value in pairs(expected) do eq(seen[key], value, 'missing counter metric ' .. key .. ' requested=' .. requested) end + ok(#counter_calls >= 1, 'counter poll expected') + + sub:unsubscribe() + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + +function tests.test_config_waits_for_network_config_capability() + fibers.run(function (scope) + local b = busmod.new() + local conn = b:connect() + local reader = b:connect() + + local child = start_service(scope, conn, { + conn = conn, + config = cfg(), + rev = 18, + observe = false, + }) + + local view = reader:retained_view(topics.summary()) + local summary = probe.wait_versioned_until('net waiting for network-config capability', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'waiting_for_hal' and payload or nil + end, + { timeout = 0.5 }) + eq(summary.service, 'net') + eq(summary.state, 'waiting_for_hal') + eq(summary.ready, false) + eq(summary.reason, 'network_config_unavailable') + eq(summary.apply.state, 'waiting_for_hal') + eq(summary.stats.apply_started, 0) + ok(summary.dependencies and summary.dependencies.network_config, 'network_config dependency expected') + eq(summary.dependencies.network_config.status, 'configured') + eq(summary.dependencies.network_config.available, false) + view:close() + + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + +function tests.test_network_config_available_false_does_not_start_apply() + fibers.run(function (scope) + local b = busmod.new() + local conn = b:connect() + local reader = b:connect() + local calls = {} + + local apply_ep = bind_network_config_apply(scope, conn, calls) + retain_network_config_status_payload(conn, { + schema = 'devicecode.cap.status/1', + state = 'available', + available = false, + }) + + local child = start_service(scope, conn, { + conn = conn, + config = cfg(), + rev = 24, + observe = false, + }) + + local view = reader:retained_view(topics.summary()) + local summary = probe.wait_versioned_until('net does not apply when network-config is not effectively available', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'waiting_for_hal' + and payload.stats.apply_started == 0 + and payload or nil + end, + { timeout = 0.5 }) + eq(summary.reason, 'network_config_unavailable') + eq(#calls, 0) + fibers.perform(sleep.sleep_op(0.05)) + eq(#calls, 0) + view:close() + + apply_ep:close() + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + +function tests.test_network_state_running_available_false_does_not_start_observer() + fibers.run(function (scope) + local b = busmod.new() + local conn = b:connect() + local reader = b:connect() + local calls = {} + + local watch_ep = bind_network_state_watch(scope, conn, calls) + retain_network_state_status_payload(conn, { + schema = 'devicecode.cap.status/1', + state = 'running', + available = false, + }) + + local child = start_service(scope, conn, { + conn = conn, + observe = true, + }) + + local view = reader:retained_view(topics.summary()) + local summary = probe.wait_versioned_until('net records network-state status without starting observer', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + local payload = msg and msg.payload + local dep = payload and payload.dependencies and payload.dependencies.network_state + return dep and dep.status == 'running' and payload or nil + end, + { timeout = 0.5 }) + eq(summary.dependencies.network_state.status, 'running') + eq(#calls, 0) + fibers.perform(sleep.sleep_op(0.05)) + eq(#calls, 0) + view:close() + + watch_ep:close() + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + +function tests.test_network_config_available_starts_pending_apply() + fibers.run(function (scope) + local b = busmod.new() + local conn = b:connect() + local reader = b:connect() + local calls = {} + + local child = start_service(scope, conn, { + conn = conn, + config = cfg(), + rev = 21, + observe = false, + }) + + local view = reader:retained_view(topics.summary()) + probe.wait_versioned_until('net pending apply before capability is available', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'waiting_for_hal' and payload or nil + end, + { timeout = 0.5 }) + + local apply_ep = bind_network_config_apply(scope, conn, calls) + retain_network_config_status(conn, 'available') + + local summary = probe.wait_versioned_until('net apply after network-config becomes available', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'running' and payload.apply.state == 'applied' and payload or nil + end, + { timeout = 0.5 }) + eq(#calls, 1) + eq(calls[1].intent.rev, 21) + eq(calls[1].opts.generation, 1) + eq(calls[1].opts.apply_id, 1) + eq(summary.apply.last_applied_rev, 21) + eq(summary.dependencies.network_config.status, 'available') + ok(summary.dependencies and summary.dependencies.network_config, 'network_config dependency expected') + eq(summary.dependencies.network_config.available, true) + view:close() + + apply_ep:close() + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + +function tests.test_newer_pending_config_wins_when_capability_arrives() + fibers.run(function (scope) + local b = busmod.new() + local conn = b:connect() + local writer = b:connect() + local reader = b:connect() + local calls = {} + + local child = start_service(scope, conn, { conn = conn, observe = false }) + local view = reader:retained_view(topics.summary()) + probe.wait_versioned_until('net ready to receive config', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'waiting_for_config' and payload or nil + end, + { timeout = 0.5 }) + + writer:retain(topics.config(), { rev = 31, data = cfg() }) + local c2 = cfg() + c2.segments.lan.vlan = 20 + writer:retain(topics.config(), { rev = 32, data = c2 }) + + probe.wait_versioned_until('net records newest pending config before capability is available', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'waiting_for_hal' + and payload.config and payload.config.rev == 32 + and payload.stats.apply_started == 0 + and payload or nil + end, + { timeout = 0.5 }) + + local apply_ep = bind_network_config_apply(scope, conn, calls) + retain_network_config_status(conn, 'available') + + local summary = probe.wait_versioned_until('net applies newest pending config', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'running' and payload.apply.state == 'applied' and payload or nil + end, + { timeout = 0.5 }) + eq(#calls, 1) + eq(calls[1].intent.rev, 32) + eq(calls[1].opts.generation, 2) + eq(summary.generation, 2) + eq(summary.apply.last_applied_rev, 32) + view:close() + + apply_ep:close() + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + +function tests.test_no_route_apply_returns_to_pending_not_degraded() + fibers.run(function (scope) + local b = busmod.new() + local conn = b:connect() + local reader = b:connect() + local calls = {} + + retain_network_config_status(conn, 'available') + local child = start_service(scope, conn, { + conn = conn, + config = cfg(), + rev = 22, + observe = false, + }) + + local view = reader:retained_view(topics.summary()) + local pending = probe.wait_versioned_until('net returns no_route apply to pending', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'waiting_for_hal' + and payload.apply.state == 'waiting_for_hal' + and payload.stats.apply_started == 1 + and payload or nil + end, + { timeout = 0.5 }) + eq(pending.ready, false) + eq(pending.reason, 'network_config_unavailable') + + local apply_ep = bind_network_config_apply(scope, conn, calls) + retain_network_config_status(conn, 'available') + + local summary = probe.wait_versioned_until('net retries pending apply after route returns', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'running' and payload.apply.state == 'applied' and payload or nil + end, + { timeout = 0.5 }) + eq(#calls, 1) + eq(calls[1].intent.rev, 22) + eq(summary.apply.last_applied_rev, 22) + view:close() + + apply_ep:close() + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + +function tests.test_network_config_backend_failure_still_degrades() + fibers.run(function (scope) + local b = busmod.new() + local conn = b:connect() + local reader = b:connect() + local calls = {} + + local apply_ep = bind_network_config_apply(scope, conn, calls, function () + return { + ok = false, + code = 500, + reason = { code = 'backend_failed', err = 'nft apply failed' }, + } + end) + retain_network_config_status(conn, 'available') + + local child = start_service(scope, conn, { + conn = conn, + config = cfg(), + rev = 23, + observe = false, + }) + + local view = reader:retained_view(topics.summary()) + local summary = probe.wait_versioned_until('net degrades on real network-config failure', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + local payload = msg and msg.payload + return payload and payload.state == 'degraded' and payload or nil + end, + { timeout = 0.5 }) + eq(summary.ready, false) + eq(summary.apply.state, 'failed') + contains(summary.reason, 'nft apply failed') + eq(#calls, 1) + view:close() + + apply_ep:close() + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + +function tests.test_observed_state_updates_model_and_drift() + fibers.run(function (scope) + local mailbox = require 'fibers.mailbox' + local b = busmod.new() + local conn = b:connect() + local reader = b:connect() + local calls = {} + local obs_tx, obs_rx = mailbox.new(8, { full = 'drop_oldest' }) + + local hal = success_hal(calls) + hal.open_observed_subscription = function () return obs_rx end + hal.start_observation_op = function () return op.always({ ok = true, backend = 'test', watching = true }) end + + retain_network_config_status(conn, 'available') + retain_network_state_status_payload(conn, { + schema = 'devicecode.cap.status/1', + state = 'running', + available = true, + }) + + local child = start_service(scope, conn, { + conn = conn, + config = cfg(), + rev = 19, + hal = hal, + }) + + local observed = { + schema = 'devicecode.net.observation/1', + kind = 'snapshot_done', + source = 'test', + subject = 'network', + observed = { + schema = 'devicecode.net.observed/1', + interfaces = { + lan_bridge = { id = 'lan_bridge', enabled = true }, + }, + segments = { + lan = { id = 'lan', interfaces = { 'lan_bridge' } }, + }, + }, + } + obs_tx:send({ payload = observed }) + + local view = reader:retained_view(topics.summary()) + local summary = probe.wait_versioned_until('net observed summary', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.summary()) + return msg and msg.payload and msg.payload.stats and msg.payload.stats.observations == 1 and msg.payload or nil + end, + { timeout = 0.5 }) + eq(summary.dependencies.network_state.status, 'running') + eq(summary.dependencies.network_state.available, true) + eq(summary.observed.last_subject, 'network') + eq(summary.drift.converged, true) + view:close() + + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + + +function tests.test_wan_speedtests_wait_for_multiwan_observation() + fibers.run(function (scope) + local mailbox = require 'fibers.mailbox' + local b = busmod.new() + local conn = b:connect() + local reader = b:connect() + local calls = {} + local speedtests = {} + local obs_tx, obs_rx = mailbox.new(8, { full = 'drop_oldest' }) + local hal = success_hal(calls) + hal.speedtest_op = function (_, req) + speedtests[#speedtests + 1] = req + return op.always({ ok = true, interface = req.interface, peak_mbps = 10 }) + end + hal.open_observed_subscription = function () return obs_rx end + hal.start_observation_op = function () return op.always({ ok = true, backend = 'test', watching = true }) end + + local c = cfg() + c.wan = { + load_balancing = { policy = 'balanced', speedtests = true }, + members = { + wan_a = { interface = 'wan_a', mwan_metric = 1, weight = 1 }, + }, + } + c.interfaces.wan_a = { kind = 'ethernet', role = 'wan', segment = 'wan', endpoint = { ifname = 'eth1' }, addressing = { ipv4 = { mode = 'dhcp' } } } + c.segments.wan = { kind = 'wan', firewall = { zone = 'wan' } } + + retain_network_config_status(conn, 'available') + retain_network_state_status_payload(conn, { + schema = 'devicecode.cap.status/1', + state = 'running', + available = true, + }) + local skipped_sub = reader:subscribe({ 'obs', 'v1', 'net', 'event', 'speedtest_skipped' }, { queue_len = 8, full = 'drop_oldest' }) + + local child = start_service(scope, conn, { conn = conn, config = c, rev = 21, hal = hal }) + local which, msg = fibers.perform(op.named_choice({ + event = skipped_sub:recv_op(), + timeout = sleep.sleep_op(0.5), + })) + ok(which == 'event' and msg and msg.payload, 'expected speedtest skip event') + eq(msg.payload.reason, 'waiting_for_observation') + eq(msg.payload.backhaul_status, 'missing') + eq(#speedtests, 0, 'speedtest should wait for observed multiwan status') + + obs_tx:send({ payload = { + schema = 'devicecode.net.observation/1', + kind = 'snapshot_done', + source = 'test', + subject = 'network', + observed = { + multiwan = { + interfaces_by_semantic = { + wan_a = { interface = 'wan_a', state = 'online', online = true, usable = true }, + }, + }, + live = { interfaces = { + wan_a = { ipv4 = { { address = '203.0.113.10' } } }, + } }, + }, + } }) + + ok(probe.wait_until(function () return #speedtests == 1 end, { timeout = 0.5 }), 'speedtest should start after observation') + skipped_sub:unsubscribe() + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + + +function tests.test_wan_members_trigger_speedtests_and_live_weights() + fibers.run(function (scope) + local mailbox = require 'fibers.mailbox' + local b = busmod.new() + local conn = b:connect() + local reader = b:connect() + local calls = {} + local speedtests = {} + local weights = {} + local obs_tx, obs_rx = mailbox.new(8, { full = 'drop_oldest' }) + local hal = success_hal(calls) + hal.speedtest_op = function (_, req) + speedtests[#speedtests + 1] = req + local mbps = req.interface == 'wan_a' and 80 or 20 + return op.always({ ok = true, interface = req.interface, device = req.device, peak_mbps = mbps, data_mib = 1, duration_s = 0.1 }) + end + hal.apply_live_weights_op = function (_, req) + weights[#weights + 1] = req + return op.always({ ok = true, changed = true, applied = true }) + end + hal.open_observed_subscription = function () return obs_rx end + hal.start_observation_op = function () return op.always({ ok = true, backend = 'test', watching = true }) end + + local c = cfg() + c.wan = { + load_balancing = { policy = 'balanced', speedtests = true }, + members = { + wan_a = { interface = 'wan_a', mwan_metric = 1, weight = 1 }, + wan_b = { interface = 'wan_b', mwan_metric = 1, weight = 1 }, + }, + } + c.interfaces.wan_a = { kind = 'ethernet', role = 'wan', segment = 'wan', endpoint = { ifname = 'eth1' }, addressing = { ipv4 = { mode = 'dhcp' } } } + c.interfaces.wan_b = { kind = 'cellular', role = 'wan', segment = 'wan', endpoint = { ifname = 'wwan1' }, addressing = { ipv4 = { mode = 'dhcp' } } } + c.segments.wan = { kind = 'wan', firewall = { zone = 'wan' } } + + retain_network_config_status(conn, 'available') + retain_network_state_status_payload(conn, { + schema = 'devicecode.cap.status/1', + state = 'running', + available = true, + }) + local metric_sub = reader:subscribe({ 'obs', 'v1', 'net', 'metric', 'speedtest' }, { queue_len = 8, full = 'drop_oldest' }) + + local child = start_service(scope, conn, { conn = conn, config = c, rev = 20, hal = hal }) + obs_tx:send({ payload = { + schema = 'devicecode.net.observation/1', + kind = 'snapshot_done', + source = 'test', + subject = 'network', + observed = { + multiwan = { + interfaces_by_semantic = { + wan_a = { interface = 'wan_a', state = 'online', online = true, usable = true }, + wan_b = { interface = 'wan_b', state = 'online', online = true, usable = true }, + }, + }, + live = { interfaces = { + wan_a = { ipv4 = { { address = '203.0.113.10' } } }, + wan_b = { ipv4 = { { address = '10.1.2.3' } } }, + } }, + }, + } }) + + local view = reader:retained_view(topics.domain('wan_runtime')) + local runtime = probe.wait_versioned_until('net wan runtime weights', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.domain('wan_runtime')) + local payload = msg and msg.payload + return payload and payload.live_weights and payload.live_weights.state == 'applied' and payload or nil + end, + { timeout = 1.0 }) + ok(#speedtests >= 2, 'expected every WAN member to be speed-tested') + ok(#weights >= 1, 'expected live weight apply') + local members = runtime.live_weights.members + ok(type(members) == 'table' and #members >= 2, 'members expected') + local seen = {} + for _ = 1, 2 do + local which, msg = fibers.perform(op.named_choice({ + metric = metric_sub:recv_op(), + timeout = sleep.sleep_op(0.2), + })) + ok(which == 'metric' and msg and msg.payload, 'expected speedtest metric') + local ns = msg.payload.namespace + ok(type(ns) == 'table', 'speedtest metric namespace expected') + eq(ns[1], 'net') + eq(ns[3], 'speedtest') + seen[ns[2]] = msg.payload.value + end + eq(seen.wan_a, 80) + eq(seen.wan_b, 20) + metric_sub:unsubscribe() + view:close() + child:cancel('test complete') + fibers.perform(child:join_op()) + end) +end + + +return tests diff --git a/tests/unit/net/test_wan_policy_event_led.lua b/tests/unit/net/test_wan_policy_event_led.lua new file mode 100644 index 00000000..c2e2c0b5 --- /dev/null +++ b/tests/unit/net/test_wan_policy_event_led.lua @@ -0,0 +1,260 @@ +-- tests/unit/net/test_wan_policy_event_led.lua + +local policy = require 'services.net.wan_policy' + +local tests = {} +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end +local function eq(a, b, msg) if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end end + +local function snapshot() + return { + generation = 1, + wan = { + configured_members = { + wan = { interface = 'wan', metric = 1 }, + modem_primary = { interface = 'modem_primary', metric = 1, source = { kind = 'gsm-uplink', id = 'primary' } }, + }, + load_balancing = { speedtests = { enabled = true, probe_weight = 1, weight_scale = 100 } }, + }, + backhaul = { + uplinks = { + wan = { id = 'wan', interface = 'wan', device = 'eth1', state = 'online', usable = true, path_address = { family = 'ipv4', address = '203.0.113.10' } }, + modem_primary = { id = 'modem_primary', interface = 'modem_primary', device = 'wwan1', state = 'offline', usable = false, path_address = { family = 'ipv4', address = '10.1.2.3' } }, + }, + }, + wan_runtime = { speedtests = {} }, + sources = { gsm_uplinks = { primary = { linux = { ifname = 'wwan1' } } } }, + } +end + +local function uplinks_by_id(s) + local out = {} + for _, u in ipairs(policy.collect_uplinks(s)) do out[u.uplink_id] = u end + return out +end + +local function keyed_success(s, uplink_id, mbps, completed_at) + local uplink = uplinks_by_id(s)[uplink_id] + local measurement = assert(policy.measurement(s, uplink)) + return { + state = 'ok', + generation = s.generation, + ok = true, + peak_mbps = mbps, + last_success_mbps = mbps, + completed_at = completed_at, + interface = uplink.request.interface, + measurement_key = measurement.key, + measurement = measurement, + last_success = { mbps = mbps, completed_at = completed_at, measurement_key = measurement.key, measurement = measurement }, + } +end + +function tests.test_only_observed_online_uplink_is_due() + local s = snapshot() + local uplinks = policy.collect_uplinks(s) + local by_id = {} + for _, u in ipairs(uplinks) do by_id[u.uplink_id] = u end + ok(policy.speedtest_due(s, by_id.wan, { generation = 1, now = 10 })) + local due, reason = policy.speedtest_due(s, by_id.modem_primary, { generation = 1, now = 10 }) + eq(due, false) + eq(reason, 'not_online') +end + +function tests.test_weights_include_probe_members_after_one_measurement() + local s = snapshot() + s.wan_runtime.speedtests.wan = keyed_success(s, 'wan', 80, 10) + local weights = assert(policy.compute_weights(s, 1, { now = 20 })) + eq(#weights, 2) + local by_id = {} + for _, m in ipairs(weights) do by_id[m.id] = m end + eq(by_id.wan.weight, 100) + eq(by_id.modem_primary.weight, 1) + eq(by_id.modem_primary.probe, true) +end + +function tests.test_fresh_previous_generation_success_is_used_for_weights() + local s = snapshot() + s.generation = 2 + s.wan.load_balancing.speedtests.interval_s = 100 + s.backhaul.uplinks.modem_primary.state = 'online' + s.backhaul.uplinks.modem_primary.usable = true + s.wan_runtime.speedtests.wan = keyed_success(s, 'wan', 80, 10) + s.wan_runtime.speedtests.wan.generation = 1 + s.wan_runtime.speedtests.modem_primary = keyed_success(s, 'modem_primary', 20, 20) + s.wan_runtime.speedtests.modem_primary.generation = 2 + local uplinks = policy.collect_uplinks(s) + local by_uplink = {} + for _, u in ipairs(uplinks) do by_uplink[u.uplink_id] = u end + local due, reason = policy.speedtest_due(s, by_uplink.wan, { generation = 2, now = 30 }) + eq(due, false) + eq(reason, 'fresh_same_path') + + local weights = assert(policy.compute_weights(s, 2, { now = 30 })) + local by_id = {} + for _, m in ipairs(weights) do by_id[m.id] = m end + eq(by_id.wan.weight, 80) + eq(by_id.wan.probe, false) + eq(by_id.modem_primary.weight, 20) + + local expired_weights = assert(policy.compute_weights(s, 2, { now = 119 })) + local expired_by_id = {} + for _, m in ipairs(expired_weights) do expired_by_id[m.id] = m end + eq(expired_by_id.wan.weight, 1) + eq(expired_by_id.wan.probe, true) + eq(expired_by_id.modem_primary.weight, 100) +end + + +function tests.test_failed_latest_speedtest_keeps_fresh_last_success_for_weights() + local s = snapshot() + s.generation = 2 + s.wan.load_balancing.speedtests.interval_s = 100 + s.backhaul.uplinks.modem_primary.state = 'online' + s.backhaul.uplinks.modem_primary.usable = true + s.wan_runtime.speedtests.wan = keyed_success(s, 'wan', 80, 10) + s.wan_runtime.speedtests.wan.state = 'failed' + s.wan_runtime.speedtests.wan.ok = false + s.wan_runtime.speedtests.wan.last_attempt = { state = 'failed', reason = 'counter_unavailable', completed_at = 30 } + s.wan_runtime.speedtests.modem_primary = keyed_success(s, 'modem_primary', 20, 20) + + local weights = assert(policy.compute_weights(s, 2, { now = 30 })) + local by_id = {} + for _, m in ipairs(weights) do by_id[m.id] = m end + eq(by_id.wan.weight, 80) + eq(by_id.wan.probe, false) + eq(by_id.modem_primary.weight, 20) +end + + + +function tests.test_weights_fall_back_to_mwan3_online_members_without_speedtest_success() + local s = snapshot() + local weights = assert(policy.compute_weights(s, 1, { now = 30 })) + eq(#weights, 1) + eq(weights[1].id, 'wan') + eq(weights[1].weight, 100) + eq(weights[1].probe, true) + eq(weights[1].reason, 'no_successful_speedtests') +end + +function tests.test_default_speedtest_interval_is_six_hours() + local s = snapshot() + s.generation = 2 + s.wan_runtime.speedtests.wan = keyed_success(s, 'wan', 80, 10) + s.wan_runtime.speedtests.wan.generation = 1 + local uplinks = policy.collect_uplinks(s) + local by_uplink = {} + for _, u in ipairs(uplinks) do by_uplink[u.uplink_id] = u end + + local due, reason = policy.speedtest_due(s, by_uplink.wan, { generation = 2, now = 10 + (6 * 60 * 60) - 1 }) + eq(due, false) + eq(reason, 'fresh_same_path') + + due, reason = policy.speedtest_due(s, by_uplink.wan, { generation = 2, now = 10 + (6 * 60 * 60) + 1 }) + eq(due, true) + eq(reason, 'due') +end + + +function tests.test_speedtest_request_duration_is_capped_at_one_second() + local s = snapshot() + s.wan.configured_members.wan.speedtest_duration_s = 8 + local uplinks = policy.collect_uplinks(s) + local by_uplink = {} + for _, u in ipairs(uplinks) do by_uplink[u.uplink_id] = u end + eq(by_uplink.wan.request.max_duration_s, 1) + + s.wan.configured_members.wan.speedtest_duration_s = 0.5 + uplinks = policy.collect_uplinks(s) + by_uplink = {} + for _, u in ipairs(uplinks) do by_uplink[u.uplink_id] = u end + eq(by_uplink.wan.request.max_duration_s, 0.5) +end + + +function tests.test_ip_change_invalidates_fresh_measurement() + local s = snapshot() + s.wan.load_balancing.speedtests.interval_s = 100 + s.wan_runtime.speedtests.wan = keyed_success(s, 'wan', 80, 10) + s.backhaul.uplinks.wan.path_address.address = '198.51.100.42' + local by_uplink = uplinks_by_id(s) + local due, reason = policy.speedtest_due(s, by_uplink.wan, { generation = 2, now = 30 }) + eq(due, true) + eq(reason, 'path_changed_ip') +end + + +function tests.test_modem_ip_change_invalidates_fresh_measurement() + local s = snapshot() + s.wan.load_balancing.speedtests.interval_s = 100 + s.backhaul.uplinks.modem_primary.state = 'online' + s.backhaul.uplinks.modem_primary.usable = true + s.wan_runtime.speedtests.modem_primary = keyed_success(s, 'modem_primary', 20, 10) + s.backhaul.uplinks.modem_primary.path_address.address = '10.9.8.7' + local by_uplink = uplinks_by_id(s) + local due, reason = policy.speedtest_due(s, by_uplink.modem_primary, { generation = 2, now = 30 }) + eq(due, true) + eq(reason, 'path_changed_ip') +end + +function tests.test_online_uplink_without_ip_still_runs_first_measurement() + local s = snapshot() + s.backhaul.uplinks.wan.path_address = nil + local by_uplink = uplinks_by_id(s) + local due, reason = policy.speedtest_due(s, by_uplink.wan, { generation = 1, now = 10 }) + eq(due, true) + eq(reason, 'due') + local measurement = assert(policy.measurement(s, by_uplink.wan)) + eq(measurement.address_family, 'unknown') + eq(measurement.address, 'unknown') +end + +function tests.test_fresh_weak_measurement_is_reused_when_ip_later_appears() + local s = snapshot() + s.wan.load_balancing.speedtests.interval_s = 100 + s.backhaul.uplinks.wan.path_address = nil + s.wan_runtime.speedtests.wan = keyed_success(s, 'wan', 80, 10) + s.backhaul.uplinks.wan.path_address = { family = 'ipv4', address = '203.0.113.10' } + local by_uplink = uplinks_by_id(s) + local due, reason = policy.speedtest_due(s, by_uplink.wan, { generation = 2, now = 30 }) + eq(due, false) + eq(reason, 'fresh_weak_path') +end + +function tests.test_speedtest_retry_delay_uses_configured_exponential_backoff() + local s = snapshot() + s.wan.load_balancing.speedtests.retry_after_s = 10 + s.wan.load_balancing.speedtests.retry_max_s = 60 + eq(policy.speedtest_retry_delay_s(s, { failure_count = 1 }), 10) + eq(policy.speedtest_retry_delay_s(s, { failure_count = 2 }), 20) + eq(policy.speedtest_retry_delay_s(s, { failure_count = 3 }), 40) + eq(policy.speedtest_retry_delay_s(s, { failure_count = 4 }), 60) +end + +function tests.test_speedtest_retry_delay_defaults_to_ten_seconds() + local s = snapshot() + eq(policy.speedtest_retry_delay_s(s, { failure_count = 1 }), 10) +end + + +function tests.test_retry_backoff_only_suppresses_same_path() + local s = snapshot() + local by_uplink = uplinks_by_id(s) + local measurement = assert(policy.measurement(s, by_uplink.wan)) + s.wan_runtime.speedtests.wan = { + state = 'failed', ok = false, measurement_key = measurement.key, + failure_count = 1, retry_after = 40, + } + local due, reason = policy.speedtest_due(s, by_uplink.wan, { generation = 1, now = 20 }) + eq(due, false) + eq(reason, 'retry_later') + + s.backhaul.uplinks.wan.path_address.address = '198.51.100.42' + by_uplink = uplinks_by_id(s) + due, reason = policy.speedtest_due(s, by_uplink.wan, { generation = 1, now = 20 }) + eq(due, true) + eq(reason, 'due') +end + +return tests diff --git a/tests/unit/net/test_wan_runtime.lua b/tests/unit/net/test_wan_runtime.lua new file mode 100644 index 00000000..f03e1eea --- /dev/null +++ b/tests/unit/net/test_wan_runtime.lua @@ -0,0 +1,157 @@ +-- tests/unit/net/test_wan_runtime.lua + +local fibers = require 'fibers' +local op = require 'fibers.op' +local mailbox = require 'fibers.mailbox' +local runfibers = require 'tests.support.run_fibers' + +local wan_runtime = require 'services.net.wan_runtime' +local wan_manager = require 'services.net.wan_manager' +local model_mod = require 'services.net.model' + +local tests = {} + +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end +local function eq(a, b, msg) if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end end + +function tests.test_speedtest_service_timeout_aborts_hal_bus_op_without_hidden_bus_timeout() + runfibers.run(function(scope) + local done_tx, done_rx = mailbox.new(8, { full = 'reject_newest' }) + local seen_opts + local aborted = false + local hal = { + speedtest_op = function (_, _request, opts) + seen_opts = opts + return op.never():on_abort(function () aborted = true end) + end, + } + + local handle, err = wan_runtime.start_speedtest({ + lifetime_scope = scope, + done_tx = done_tx, + hal = hal, + generation = 1, + speedtest_id = 7, + uplink_id = 'wan_a', + request = { interface = 'wan_a', max_duration_s = 0.01 }, + }) + ok(handle, err) + local ev = fibers.perform(done_rx:recv_op()) + ok(ev and ev.kind == 'net_speedtest_done', 'speedtest completion expected') + ok(aborted, 'timeout should abort the underlying HAL Op') + ok(seen_opts and seen_opts.timeout == false, 'HAL bus call should not receive hidden positive timeout') + ok(ev.result and ev.result.result and ev.result.result.timeout == true, 'timeout result expected') + eq(ev.result.result.err, 'net_speedtest_timeout') + end) +end + + +function tests.test_reconcile_applies_live_weights_when_speedtests_are_fresh() + runfibers.run(function(scope) + local done_tx, done_rx = mailbox.new(8, { full = 'reject_newest' }) + local applied + local hal = { + apply_live_weights_op = function(_, request) + applied = request + return op.always({ ok = true, changed = true, applied = true }) + end, + } + local model = model_mod.new('net-test') + model:update(function(s) + s.generation = 2 + s.wan = { + load_balancing = { speedtests = true, policy = 'balanced' }, + configured_members = { + wan = { interface = 'wan', metric = 1 }, + modem = { interface = 'modem', metric = 1 }, + }, + } + s.observed.snapshot = { + multiwan = { + interfaces_by_semantic = { + wan = { interface = 'wan', state = 'online', online = true, usable = true }, + modem = { interface = 'modem', state = 'online', online = true, usable = true }, + }, + }, + } + s.backhaul = { + uplinks = { + wan = { id = 'wan', interface = 'wan', state = 'online', usable = true, path_address = { family = 'ipv4', address = '203.0.113.10' } }, + modem = { id = 'modem', interface = 'modem', state = 'online', usable = true, path_address = { family = 'ipv4', address = '10.1.2.3' } }, + }, + } + s.wan_runtime = { + uplinks = {}, + speedtests = { + wan = { state = 'done', generation = 1, ok = true, peak_mbps = 80, last_success_mbps = 80, completed_at = 10, + measurement_key = 'v1|wan|wan||host-multiwan||ipv4|203.0.113.10|', + last_success = { mbps = 80, completed_at = 10, measurement_key = 'v1|wan|wan||host-multiwan||ipv4|203.0.113.10|' } }, + modem = { state = 'done', generation = 2, ok = true, peak_mbps = 20, last_success_mbps = 20, completed_at = 20, + measurement_key = 'v1|modem|modem||host-multiwan||ipv4|10.1.2.3|', + last_success = { mbps = 20, completed_at = 20, measurement_key = 'v1|modem|modem||host-multiwan||ipv4|10.1.2.3|' } }, + }, + live_weights = {}, + last_weight_apply = { generation = 2, members = { + { id = 'wan', interface = 'wan', metric = 1, weight = 1, probe = true }, + { id = 'modem', interface = 'modem', metric = 1, weight = 99, probe = false }, + } }, + } + return s + end) + local state = { + scope = scope, + service_id = 'net-test', + model = model, + hal = hal, + done_tx = done_tx, + active_speedtests = {}, + active_weight_apply = nil, + next_speedtest_id = 1, + next_weight_apply_id = 1, + current_generation = { generation = 2 }, + now = function() return 30 end, + } + local ok_apply, err = wan_manager.reconcile_speedtests(state, 'observed_state') + ok(ok_apply, err) + local ev = fibers.perform(done_rx:recv_op()) + ok(ev and ev.kind == 'net_live_weights_done', 'live weight apply completion expected') + ok(applied and applied.members, 'live weight request expected') + local by_id = {} + for _, m in ipairs(applied.members) do by_id[m.id] = m end + eq(by_id.wan.weight, 80) + eq(by_id.modem.weight, 20) + end) +end + +function tests.test_live_weights_service_timeout_aborts_hal_bus_op_without_hidden_bus_timeout() + runfibers.run(function(scope) + local done_tx, done_rx = mailbox.new(8, { full = 'reject_newest' }) + local seen_opts + local aborted = false + local hal = { + apply_live_weights_op = function (_, _request, opts) + seen_opts = opts + return op.never():on_abort(function () aborted = true end) + end, + } + + local handle, err = wan_runtime.start_live_weights({ + lifetime_scope = scope, + done_tx = done_tx, + hal = hal, + generation = 1, + weight_apply_id = 3, + members = { { id = 'wan_a', interface = 'wan_a', weight = 1 } }, + timeout_s = 0.01, + }) + ok(handle, err) + local ev = fibers.perform(done_rx:recv_op()) + ok(ev and ev.kind == 'net_live_weights_done', 'live weights completion expected') + ok(aborted, 'timeout should abort the underlying HAL Op') + ok(seen_opts and seen_opts.timeout == false, 'HAL bus call should not receive hidden positive timeout') + ok(ev.result and ev.result.result and ev.result.result.timeout == true, 'timeout result expected') + eq(ev.result.result.err, 'net_live_weights_timeout') + end) +end + +return tests diff --git a/tests/unit/shared/hash_xxhash32_spec.lua b/tests/unit/shared/hash_xxhash32_spec.lua new file mode 100644 index 00000000..86b4634f --- /dev/null +++ b/tests/unit/shared/hash_xxhash32_spec.lua @@ -0,0 +1,51 @@ +-- tests/unit/shared/hash_xxhash32_spec.lua + +local xxhash32 = require 'shared.hash.xxhash32' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_eq(a, b, msg) + if a ~= b then + fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) + end +end + +function tests.test_known_vectors_seed_zero() + assert_eq(xxhash32.digest_hex(''), '02cc5d05') + assert_eq(xxhash32.digest_hex('a'), '550d7456') + assert_eq(xxhash32.digest_hex('abc'), '32d153ff') + assert_eq(xxhash32.digest_hex('hello'), 'fb0077f9') + assert_eq(xxhash32.digest_hex('Hello, world!'), '31b7405d') + assert_eq(xxhash32.digest_hex('abcdefghijklmnopqrstuvwxyz'), '63a14d5f') + assert_eq(xxhash32.digest_hex('0123456789abcdef'), 'c2c45b69') + assert_eq(xxhash32.digest_hex('0123456789abcdefg'), 'cc79b217') + assert_eq(xxhash32.digest_hex('The quick brown fox jumps over the lazy dog'), 'e85ea4de') +end + +function tests.test_known_vectors_non_zero_seed() + local st = xxhash32.new(1) + xxhash32.update(st, '') + assert_eq(xxhash32.digest_hex_state(st), '0b2cb792') + + st = xxhash32.new(1) + xxhash32.update(st, 'The quick brown fox jumps over the lazy dog') + assert_eq(xxhash32.digest_hex_state(st), '234f8471') +end + +function tests.test_incremental_update_matches_one_shot_vector() + local st = xxhash32.new(0) + xxhash32.update(st, 'abc') + xxhash32.update(st, 'def') + assert_eq(xxhash32.digest_hex_state(st), '8b7cd587') + assert_eq(xxhash32.digest_hex('abcdef'), '8b7cd587') +end + +function tests.test_binary_input_known_vector() + assert_eq(xxhash32.digest_hex('abc\0def'), '7955e6e5') +end + +return tests diff --git a/tests/unit/shared/table_spec.lua b/tests/unit/shared/table_spec.lua new file mode 100644 index 00000000..603b9a60 --- /dev/null +++ b/tests/unit/shared/table_spec.lua @@ -0,0 +1,30 @@ +local tablex = require 'shared.table' + +local tests = {} + +local function assert_eq(a, b, msg) if a ~= b then error(msg or (tostring(a) .. ' ~= ' .. tostring(b)), 2) end end +local function assert_true(v, msg) if v ~= true then error(msg or 'expected true', 2) end end +local function assert_false(v, msg) if v ~= false then error(msg or 'expected false', 2) end end + +function tests.test_deep_copy_and_equal() + local src = { a = { b = 1 }, list = { 'x', 'y' } } + local cp = tablex.deep_copy(src) + assert_true(tablex.deep_equal(src, cp)) + cp.a.b = 2 + assert_false(tablex.deep_equal(src, cp)) + assert_eq(src.a.b, 1) +end + +function tests.test_array_copy_and_is_array() + local cp = tablex.array_copy({ 'a', 'b' }) + assert_eq(cp[1], 'a') + assert_true(tablex.is_array(cp)) + assert_false(tablex.is_array({ a = 1 })) +end + +function tests.test_first_non_nil() + assert_eq(tablex.first_non_nil(nil, false, 'x'), false) + assert_eq(tablex.first_non_nil(nil, nil, 'x'), 'x') +end + +return tests diff --git a/tests/unit/shared/topic_spec.lua b/tests/unit/shared/topic_spec.lua new file mode 100644 index 00000000..53c7b44e --- /dev/null +++ b/tests/unit/shared/topic_spec.lua @@ -0,0 +1,26 @@ +local topic = require 'shared.topic' + +local tests = {} +local function assert_eq(a, b, msg) if a ~= b then error(msg or (tostring(a) .. ' ~= ' .. tostring(b)), 2) end end +local function assert_true(v, msg) if v ~= true then error(msg or 'expected true', 2) end end +local function assert_false(v, msg) if v ~= false then error(msg or 'expected false', 2) end end + +function tests.test_append_flattens_topic_segments() + local out = topic.append({ 'a' }, 'b', { 'c', 'd' }) + assert_eq(table.concat(out, '/'), 'a/b/c/d') +end + +function tests.test_prefix_and_replace() + assert_true(topic.starts_with({ 'a', 'b', 'c' }, { 'a', 'b' })) + assert_false(topic.starts_with({ 'a' }, { 'a', 'b' })) + local out = assert(topic.replace_prefix({ 'a', 'b', 'c' }, { 'a' }, { 'x' })) + assert_eq(table.concat(out, '/'), 'x/b/c') +end + +function tests.test_validate_dense() + assert_true((topic.validate_dense({ 'a', 1 }))) + local ok = topic.validate_dense({ '', 'b' }) + assert_eq(ok, nil) +end + +return tests diff --git a/tests/unit/shared/validate_spec.lua b/tests/unit/shared/validate_spec.lua new file mode 100644 index 00000000..4ea5af64 --- /dev/null +++ b/tests/unit/shared/validate_spec.lua @@ -0,0 +1,21 @@ +local validate = require 'shared.validate' + +local tests = {} +local function assert_eq(a, b, msg) if a ~= b then error(msg or (tostring(a) .. ' ~= ' .. tostring(b)), 2) end end +local function assert_raises(fn) + local ok = pcall(fn) + if ok then error('expected error', 2) end +end + +function tests.test_scalar_validation() + assert_eq(validate.non_empty_string('x', 'name'), 'x') + assert_eq(validate.positive_integer(2, 'n'), 2) + assert_raises(function () validate.positive_integer(0, 'n') end) +end + +function tests.test_only_fields() + validate.only_fields({ a = 1 }, { 'a' }, 'opts') + assert_raises(function () validate.only_fields({ b = 1 }, { 'a' }, 'opts') end) +end + +return tests diff --git a/tests/unit/support/test_bus_cleanup.lua b/tests/unit/support/test_bus_cleanup.lua new file mode 100644 index 00000000..e7c83efd --- /dev/null +++ b/tests/unit/support/test_bus_cleanup.lua @@ -0,0 +1,110 @@ +-- tests/unit/support/test_bus_cleanup.lua + +local cleanup = require 'devicecode.support.bus_cleanup' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_true(v, msg) + if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end +end + +local function assert_nil(v, msg) + if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end +end + +local function assert_eq(a, b, msg) + if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end +end + +local function assert_match(s, pat, msg) + if type(s) ~= 'string' or not s:match(pat) then + fail(msg or ('expected ' .. tostring(s) .. ' to match ' .. tostring(pat))) + end +end + +function tests.test_publish_retain_and_unretain_call_immediate_bus_methods() + local calls = {} + local conn = { + publish = function (_, topic, payload, opts) + calls[#calls + 1] = { 'publish', topic, payload, opts } + return true, nil + end, + retain = function (_, topic, payload, opts) + calls[#calls + 1] = { 'retain', topic, payload, opts } + return true, nil + end, + unretain = function (_, topic, opts) + calls[#calls + 1] = { 'unretain', topic, opts } + return true, nil + end, + } + + assert_true(cleanup.publish(conn, { 'a' }, 1, { extra = true })) + assert_true(cleanup.retain(conn, { 'b' }, 2)) + assert_true(cleanup.unretain(conn, { 'c' })) + + assert_eq(calls[1][1], 'publish') + assert_eq(calls[1][2][1], 'a') + assert_eq(calls[2][1], 'retain') + assert_eq(calls[3][1], 'unretain') +end + +function tests.test_wrapper_catches_raised_bus_cleanup_error() + local conn = { + disconnect = function () + error('boom', 0) + end, + } + + local ok, err = cleanup.disconnect(conn) + assert_nil(ok) + assert_match(err, 'boom') +end + +function tests.test_feed_cleanup_can_use_handle_when_connection_is_absent() + local closed + local feed = { + unsubscribe = function () + closed = true + return true, nil + end, + } + + assert_true(cleanup.unsubscribe(nil, feed)) + assert_true(closed) +end + +function tests.test_reply_and_fail_report_duplicate_completion_as_failure() + local req = { + reply = function () return false, 'already_done' end, + fail = function () return false, 'already_done' end, + } + + local ok, err = cleanup.reply(req, { ok = true }) + assert_nil(ok) + assert_match(err, 'already_done') + + ok, err = cleanup.fail(req, 'bad') + assert_nil(ok) + assert_match(err, 'already_done') +end + +function tests.test_checked_raises_labelled_failure() + local conn = { + disconnect = function () return nil, 'not now' end, + } + + local ok, err = pcall(function () + cleanup.checked('disconnect_label', conn, 'disconnect') + end) + + assert_eq(ok, false) + assert_match(err, 'disconnect_label') + assert_match(err, 'not now') +end + +return tests diff --git a/tests/unit/support/test_capability_dependencies.lua b/tests/unit/support/test_capability_dependencies.lua new file mode 100644 index 00000000..044129b3 --- /dev/null +++ b/tests/unit/support/test_capability_dependencies.lua @@ -0,0 +1,484 @@ +-- tests/unit/support/test_capability_dependencies.lua + +local fibers = require 'fibers' +local op = require 'fibers.op' +local busmod = require 'bus' +local runfibers = require 'tests.support.run_fibers' +local deps_mod = require 'devicecode.support.capability_dependencies' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function ok(v, msg) + if not v then fail(msg) end + return v +end + +local function eq(a, b, msg) + if a ~= b then + fail((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a)) + end +end + +local function is_true(v, msg) + if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end +end + +local function is_false(v, msg) + if v ~= false then fail(msg or ('expected false, got ' .. tostring(v))) end +end + +local function is_nil(v, msg) + if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end +end + +local function status_topic(class, id) + return { 'cap', class, id or 'main', 'status' } +end + +local function retain_status(conn, class, id, payload) + return conn:retain(status_topic(class, id), payload) +end + +local function new_deps(conn, specs, opts) + local deps, err = deps_mod.open(conn, specs, opts) + return ok(deps, err) +end + +local function next_event(deps) + return fibers.perform(deps:event_source():recv_op()) +end + + +function tests.test_open_does_not_mutate_options_table() + runfibers.run(function() + local b = busmod.new() + local conn = b:connect() + local opts = { queue_len = 4, full = 'drop_oldest' } + local before = {} + for k, v in pairs(opts) do before[k] = v end + + local deps = new_deps(conn, { + { key = 'network_config', class = 'network-config', id = 'main' }, + }, opts) + + for k, v in pairs(opts) do eq(v, before[k], 'option table value changed for ' .. tostring(k)) end + for k, _ in pairs(before) do eq(opts[k], before[k], 'option table lost key ' .. tostring(k)) end + is_nil(opts._now, 'open should not add private fields to caller options') + deps:terminate('test complete') + end) +end + +function tests.test_public_contract_is_small_model_style_surface() + runfibers.run(function() + local b = busmod.new() + local conn = b:connect() + local deps = new_deps(conn, { + { key = 'network_config', class = 'network-config', id = 'main' }, + }) + + for _, name in ipairs({ + 'available', 'status', 'dependency', 'snapshot', + 'version', 'changed_op', 'event_source', + 'ref', 'classify_call_failure', 'terminate', + }) do + if type(deps[name]) ~= 'function' then fail('expected public method ' .. name) end + end + + for _, name in ipairs({ + 'close', 'is_terminated', 'why', 'keys', 'observed_status', 'reason', + 'all_available', 'record_status', 'mark_route_missing', 'clear_route_missing', + 'recv_op', 'try_recv_now', 'event_sources', + }) do + if deps[name] ~= nil then fail('unexpected public method ' .. name) end + end + + deps:terminate('test complete') + end) +end + +function tests.test_open_creates_curated_refs_and_initial_configured_state() + runfibers.run(function() + local b = busmod.new() + local conn = b:connect() + local deps = new_deps(conn, { + { key = 'network_config', class = 'network-config', id = 'main' }, + { key = 'optional_store', class = 'artifact-store', id = 'main', required = false }, + }) + + ok(deps:ref('network_config'), 'capability ref expected') + eq(deps:status('network_config'), 'configured') + is_false(deps:available('network_config')) + + local snap = deps:snapshot() + eq(snap.network_config.key, 'network_config') + eq(snap.network_config.class, 'network-config') + eq(snap.network_config.id, 'main') + eq(snap.network_config.observed_status, 'configured') + eq(snap.network_config.status, 'configured') + is_true(snap.network_config.required) + is_false(snap.network_config.available) + is_false(snap.optional_store.required) + + deps:terminate('test complete') + end) +end + +function tests.test_retained_status_replay_updates_effective_availability() + runfibers.run(function() + local b = busmod.new() + local writer = b:connect() + local conn = b:connect() + + retain_status(writer, 'network-config', 'main', { + schema = 'devicecode.cap.status/1', + state = 'available', + available = true, + }) + + local deps = new_deps(conn, { + { key = 'network_config', class = 'network-config', id = 'main' }, + }) + + local ev = next_event(deps) + eq(ev.kind, 'capability_dependency_changed') + eq(ev.key, 'network_config') + eq(ev.status, 'available') + is_true(ev.available) + is_true(deps:available('network_config')) + eq(deps:status('network_config'), 'available') + + local dep = deps:dependency('network_config') + eq(dep.observed_status, 'available') + eq(dep.status, 'available') + is_true(dep.available) + + deps:terminate('test complete') + end) +end + +function tests.test_route_missing_overrides_observed_available_until_next_available_status() + runfibers.run(function() + local b = busmod.new() + local writer = b:connect() + local conn = b:connect() + + retain_status(writer, 'control-store', 'update', { state = 'available', available = true }) + + local deps = new_deps(conn, { + { key = 'job_store', class = 'control-store', id = 'update' }, + }) + next_event(deps) + is_true(deps:available('job_store')) + eq(deps:dependency('job_store').observed_status, 'available') + + local class = deps:classify_call_failure('job_store', { err = 'no_route' }) + eq(class, 'route_missing') + eq(deps:dependency('job_store').observed_status, 'available') + eq(deps:status('job_store'), 'route_missing') + is_false(deps:available('job_store')) + eq(deps:dependency('job_store').reason, 'no_route') + + retain_status(writer, 'control-store', 'update', { state = 'available', available = true }) + local ev = next_event(deps) + eq(ev.status, 'available') + is_true(ev.available) + eq(deps:status('job_store'), 'available') + is_nil(deps:dependency('job_store').reason) + + deps:terminate('test complete') + end) +end + +function tests.test_explicit_unavailable_supersedes_route_missing() + runfibers.run(function() + local b = busmod.new() + local writer = b:connect() + local conn = b:connect() + + retain_status(writer, 'control-store', 'update', { state = 'available', available = true }) + local deps = new_deps(conn, { + { key = 'job_store', class = 'control-store', id = 'update' }, + }) + next_event(deps) + deps:classify_call_failure('job_store', { err = 'no_route' }) + eq(deps:status('job_store'), 'route_missing') + + retain_status(writer, 'control-store', 'update', { + state = 'unavailable', + available = false, + reason = 'driver_stopped', + }) + local ev = next_event(deps) + eq(ev.status, 'unavailable') + is_false(ev.available) + eq(deps:dependency('job_store').observed_status, 'unavailable') + eq(deps:status('job_store'), 'unavailable') + eq(deps:dependency('job_store').reason, 'driver_stopped') + + local snap = deps:dependency('job_store') + eq(snap.observed_reason, 'driver_stopped') + eq(snap.reason, 'driver_stopped') + is_false(snap.route_missing) + + deps:terminate('test complete') + end) +end + +function tests.test_explicit_available_false_overrides_running_state() + runfibers.run(function() + local b = busmod.new() + local writer = b:connect() + local conn = b:connect() + local deps = new_deps(conn, { + { key = 'network_state', class = 'network-state', id = 'main' }, + }) + + retain_status(writer, 'network-state', 'main', { state = 'running', available = false }) + local ev = next_event(deps) + eq(ev.status, 'running') + is_false(ev.available) + is_false(deps:available('network_state')) + eq(deps:status('network_state'), 'running') + + deps:terminate('test complete') + end) +end + +function tests.test_changed_op_reports_material_status_changes_only() + runfibers.run(function() + local b = busmod.new() + local writer = b:connect() + local conn = b:connect() + local deps = new_deps(conn, { + { key = 'http', class = 'http', id = 'main' }, + }) + + local seen = deps:version() + retain_status(writer, 'http', 'main', { state = 'available', available = true }) + local ev = next_event(deps) + is_true(ev.changed) + + local version, snap, err = fibers.perform(deps:changed_op(seen)) + is_nil(err) + ok(version and version > seen, 'version should advance') + is_true(snap.http.available) + seen = deps:version() + + retain_status(writer, 'http', 'main', { state = 'available', available = true }) + ev = next_event(deps) + is_false(ev.changed) + eq(deps:version(), seen) + + deps:terminate('test complete') + end) +end + +function tests.test_snapshot_is_copied() + runfibers.run(function() + local b = busmod.new() + local conn = b:connect() + local deps = new_deps(conn, { + { key = 'network_config', class = 'network-config', id = 'main' }, + }) + + local class = deps:classify_call_failure('network_config', { err = 'no_route' }) + eq(class, 'route_missing') + + local snap = deps:snapshot() + snap.network_config.status = 'mutated' + snap.network_config.last_error.err = 'mutated' + + local snap2 = deps:snapshot() + eq(snap2.network_config.status, 'configured') + eq(snap2.network_config.last_error.err, 'no_route') + + deps:terminate('test complete') + end) +end + +function tests.test_no_route_classifier_handles_only_canonical_call_shapes() + is_true(deps_mod.is_no_route(nil, 'no_route')) + is_true(deps_mod.is_no_route({ err = 'network HAL call failed', detail = 'no_route' })) + is_true(deps_mod.is_no_route({ result = { err = 'no_route' } })) + is_false(deps_mod.is_no_route({ reason = { err = 'no_route' } })) + is_false(deps_mod.is_no_route({ reason = { err = 'backend_failed' } })) + is_false(deps_mod.is_no_route({ primary = { err = 'no_route' } })) + is_false(deps_mod.is_no_route({ report = { primary = { err = 'no_route' } } })) + is_false(deps_mod.is_no_route({ children = { { primary = { err = 'no_route' } } } })) +end + +function tests.test_classify_call_failure_marks_route_missing() + runfibers.run(function() + local b = busmod.new() + local writer = b:connect() + local conn = b:connect() + retain_status(writer, 'control-store', 'update', { state = 'available', available = true }) + local deps = new_deps(conn, { + { key = 'job_store', class = 'control-store', id = 'update' }, + }) + next_event(deps) + + local class, reason, dep = deps:classify_call_failure('job_store', { result = { err = 'no_route' } }) + eq(class, 'route_missing') + ok(reason, 'reason expected') + is_false(dep.available) + eq(deps:status('job_store'), 'route_missing') + is_false(deps:available('job_store')) + + class = deps:classify_call_failure('job_store', { reason = { err = 'backend_failed' } }) + eq(class, 'failure') + + deps:terminate('test complete') + end) +end + +function tests.test_terminate_closes_change_op_and_subscriptions() + runfibers.run(function() + local b = busmod.new() + local conn = b:connect() + local deps = new_deps(conn, { + { key = 'network_config', class = 'network-config', id = 'main' }, + }) + local seen = deps:version() + deps:terminate('closed_for_test') + + local version, snap, err = fibers.perform(deps:changed_op(seen)) + is_nil(version) + is_nil(snap) + eq(err, 'closed_for_test') + end) +end + +function tests.test_closed_status_feed_disables_event_source() + runfibers.run(function() + local fake_sub = {} + function fake_sub:recv_op() + return op.always(nil, 'closed_for_test') + end + function fake_sub:why() + return 'closed_for_test' + end + + local fake_ref = {} + function fake_ref:get_status_sub() + return fake_sub + end + + local deps = new_deps(nil, { + { key = 'closed_dep', ref = fake_ref }, + }) + is_true(deps:event_source().enabled()) + + local ev = next_event(deps) + eq(ev.kind, 'capability_dependency_closed') + eq(ev.key, 'closed_dep') + eq(ev.reason, 'closed_for_test') + eq(ev.status, 'unavailable') + is_false(ev.available) + is_false(deps:event_source().enabled()) + + deps:terminate('test complete') + end) +end + +function tests.test_required_status_watch_failure_fails_open() + runfibers.run(function() + local fake_ref = {} + function fake_ref:get_status_sub() + error('boom') + end + + local deps, err = deps_mod.open(nil, { + { key = 'required_dep', ref = fake_ref, required = true }, + }) + is_nil(deps) + ok(tostring(err):find('dependency_status_watch_failed:required_dep', 1, true), err) + ok(tostring(err):find('boom', 1, true), err) + end) +end + +function tests.test_get_status_sub_returned_error_is_preserved() + runfibers.run(function() + local fake_ref = {} + function fake_ref:get_status_sub() + return nil, 'detail_from_ref' + end + + local deps, err = deps_mod.open(nil, { + { key = 'required_dep', ref = fake_ref, required = true }, + }) + is_nil(deps) + ok(tostring(err):find('dependency_status_watch_failed:required_dep', 1, true), err) + ok(tostring(err):find('detail_from_ref', 1, true), err) + end) +end + +function tests.test_optional_status_watch_failure_is_reported_in_snapshot() + runfibers.run(function() + local fake_ref = {} + function fake_ref:get_status_sub() + return nil + end + + local deps = new_deps(nil, { + { key = 'optional_dep', ref = fake_ref, required = false }, + }) + local snap = deps:snapshot() + eq(snap.optional_dep.status, 'watch_failed') + is_false(snap.optional_dep.available) + eq(snap.optional_dep.observed_status, 'watch_failed') + ok(snap.optional_dep.observed_reason, 'observed_reason expected') + deps:terminate('test complete') + end) +end + +function tests.test_required_watch_failure_can_be_made_unavailable_explicitly() + runfibers.run(function() + local fake_ref = {} + function fake_ref:get_status_sub() + return nil + end + + local deps = new_deps(nil, { + { key = 'required_dep', ref = fake_ref, required = true, watch_failure = 'unavailable' }, + }) + local snap = deps:snapshot() + eq(snap.required_dep.status, 'watch_failed') + is_false(snap.required_dep.available) + eq(snap.required_dep.observed_status, 'watch_failed') + deps:terminate('test complete') + end) +end + + +function tests.test_ensure_adds_dynamic_dependency_and_receives_status() + runfibers.run(function() + local b = busmod.new() + local conn = b:connect() + local deps = new_deps(conn, { + { key = 'first', class = 'first', id = 'main' }, + }) + + local ok_add, err, dep = deps:ensure({ key = 'dynamic', class = 'dynamic', id = 'main' }) + is_true(ok_add, err) + eq(dep.key, 'dynamic') + eq(deps:status('dynamic'), 'configured') + is_false(deps:available('dynamic')) + + retain_status(conn, 'dynamic', 'main', { state = 'available' }) + local ev = next_event(deps) + eq(ev.key, 'dynamic') + is_true(ev.available) + is_true(deps:available('dynamic')) + + local ok_again = deps:ensure({ key = 'dynamic', class = 'dynamic', id = 'main' }) + is_true(ok_again) + deps:terminate('test complete') + end) +end + +return tests diff --git a/tests/unit/support/test_capdeps.lua b/tests/unit/support/test_capdeps.lua new file mode 100644 index 00000000..b0a41451 --- /dev/null +++ b/tests/unit/support/test_capdeps.lua @@ -0,0 +1,75 @@ +local capdeps = require 'services.support.capdeps' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end + +local function ok(v, msg) + if not v then error(msg or 'assertion failed', 2) end + return v +end + +function M.test_capability_dependency_returns_only_declared_methods() + local calls = {} + local sdk = { + new_ref = function (conn, id) + eq(conn, 'bus-conn') + eq(id, 'main') + return { + status_op = function (_, opts) calls[#calls + 1] = { 'status', opts }; return 'status-result' end, + exchange_op = function (_, args, opts) calls[#calls + 1] = { 'exchange', args, opts }; return 'exchange-result' end, + listen_op = function () error('not narrowed', 0) end, + } + end, + } + local decl = ok(capdeps.capability({ sdk = sdk, default_cap_id = 'main', methods = { 'status_op', 'exchange_op' } })) + local resolver = ok(capdeps.new('bus-conn', { http_client = decl })) + local port = ok(resolver:get('http_client')) + eq(type(port.status_op), 'function') + eq(type(port.exchange_op), 'function') + eq(port.listen_op, nil) + eq(port.conn, nil) + eq(port:status_op({ timeout = 1 }), 'status-result') + eq(port:exchange_op({ uri = 'http://example.test/' }, { timeout = 2 }), 'exchange-result') + eq(#calls, 2) + eq(calls[1][1], 'status') + eq(calls[2][1], 'exchange') +end + +function M.test_dependency_factory_uses_requested_cap_id_and_caches_ports() + local ids = {} + local sdk = { + new_ref = function (_, id) + ids[#ids + 1] = id + return { status_op = function () return id end } + end, + } + local decl = ok(capdeps.capability({ sdk = sdk, default_cap_id = 'main', methods = { 'status_op' } })) + local resolver = ok(capdeps.new('conn', { http_client = decl })) + local factory = resolver:factory('http_client') + local a = ok(factory('switch-http')) + local b = ok(factory('switch-http')) + local c = ok(factory()) + eq(a, b) + eq(a:status_op(), 'switch-http') + eq(c:status_op(), 'main') + eq(#ids, 2) +end + +function M.test_capability_dependency_rejects_undeclared_and_missing_methods() + local decl = ok(capdeps.capability({ + sdk = { new_ref = function () return {} end }, + methods = { 'status_op' }, + })) + local resolver = ok(capdeps.new('conn', { http_client = decl })) + local port, err = resolver:get('missing') + eq(port, nil) + ok(tostring(err):find('undeclared dependency', 1, true)) + port, err = resolver:get('http_client') + eq(port, nil) + ok(tostring(err):find('missing method status_op', 1, true)) +end + +return M diff --git a/tests/unit/support/test_config_watch.lua b/tests/unit/support/test_config_watch.lua new file mode 100644 index 00000000..a23b0ce1 --- /dev/null +++ b/tests/unit/support/test_config_watch.lua @@ -0,0 +1,111 @@ +local fibers = require 'fibers' +local bus = require 'bus' + +local config_watch = require 'devicecode.support.config_watch' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end + +local function ok(v, msg) + if not v then error(msg or 'assertion failed', 2) end + return v +end + +function M.test_extracts_data_from_retained_config_record() + local rec = { rev = 12, data = { enabled = true } } + eq(config_watch._test.data_of(rec), rec.data) + eq(config_watch._test.rev_of(rec, 0), 12) +end + +function M.test_plain_payload_is_supported_for_bootstrap_and_unit_tests() + local raw = { schema = 'x' } + eq(config_watch._test.data_of(raw), raw) + eq(config_watch._test.rev_of(raw, 7), 7) +end + +function M.test_standard_cfg_topic_shape() + local topic = config_watch.topic('ui') + eq(topic[1], 'cfg') + eq(topic[2], 'ui') +end + +function M.test_open_replays_retained_cfg_record_published_before_watch_creation() + fibers.run(function () + local b = bus.new() + local conn = b:connect({ origin_base = { kind = 'test' } }) + ok(conn:retain({ 'cfg', 'update' }, { + rev = 42, + data = { schema = 'devicecode.config/update/1', enabled = true }, + })) + + local watch, err = config_watch.open(conn, 'update', { queue_len = 2 }) + ok(watch, err) + + local ev, recv_err = fibers.perform(watch:recv_op()) + ok(ev, recv_err) + eq(ev.kind, 'config_changed') + eq(ev.service, 'update') + eq(ev.generation, 42) + eq(ev.rev, 42) + eq(ev.raw.enabled, true) + eq(ev.record.rev, 42) + eq(ev.msg.topic[1], 'cfg') + eq(ev.msg.topic[2], 'update') + + ok(watch:close()) + end) +end + +function M.test_try_recv_now_can_consume_bootstrap_replay_without_waiting() + fibers.run(function () + local b = bus.new() + local conn = b:connect({ origin_base = { kind = 'test' } }) + ok(conn:retain({ 'cfg', 'net' }, { + rev = 3, + data = { schema = 'devicecode.config/net/1' }, + })) + + local watch, err = config_watch.open(conn, 'net', { queue_len = 1 }) + ok(watch, err) + + local ev = watch:try_recv_now() + ok(ev, 'retained cfg/net replay should be immediately ready after open') + eq(ev.kind, 'config_changed') + eq(ev.service, 'net') + eq(ev.generation, 3) + eq(ev.raw.schema, 'devicecode.config/net/1') + + local none, why = watch:try_recv_now() + eq(none, nil) + eq(why, nil) + + ok(watch:close()) + end) +end + +function M.test_watch_receives_live_cfg_after_retained_bootstrap() + fibers.run(function () + local b = bus.new() + local conn = b:connect({ origin_base = { kind = 'test' } }) + ok(conn:retain({ 'cfg', 'ui' }, { rev = 1, data = { enabled = false } })) + + local watch, err = config_watch.open(conn, 'ui', { queue_len = 3 }) + ok(watch, err) + local boot = ok(fibers.perform(watch:recv_op())) + eq(boot.generation, 1) + eq(boot.raw.enabled, false) + + ok(conn:retain({ 'cfg', 'ui' }, { rev = 2, data = { enabled = true } })) + local live = ok(fibers.perform(watch:recv_op())) + eq(live.kind, 'config_changed') + eq(live.generation, 2) + eq(live.raw.enabled, true) + + ok(watch:close()) + end) +end + +return M diff --git a/tests/unit/support/test_config_watch_architecture.lua b/tests/unit/support/test_config_watch_architecture.lua new file mode 100644 index 00000000..b19c0bba --- /dev/null +++ b/tests/unit/support/test_config_watch_architecture.lua @@ -0,0 +1,127 @@ +-- tests/unit/support/test_config_watch_architecture.lua +-- +-- Guardrails for the retained cfg/ bootstrap path used by modern +-- Devicecode services. Service code should consume configuration through +-- devicecode.support.config_watch, which relies on lua-bus subscription replay +-- for retained cfg bootstrap. + +local tests = {} + +local modern_services = { + fabric = '../src/services/fabric/service.lua', + device = '../src/services/device/service.lua', + update = '../src/services/update/service.lua', + http = '../src/services/http/service.lua', + ui = '../src/services/ui/service.lua', + net = '../src/services/net/service.lua', + wired = '../src/services/wired/service.lua', +} + +local modern_roots = { + '../src/services/fabric', + '../src/services/device', + '../src/services/update', + '../src/services/http', + '../src/services/ui', + '../src/services/net', + '../src/services/wired', +} + +local function fail(msg) error(msg or 'assertion failed', 2) end + +local function read_file(path) + local f = assert(io.open(path, 'r')) + local s = f:read('*a') + f:close() + return s +end + +local function scan_files(root) + local p = assert(io.popen(('find %s -type f -name "*.lua" | sort'):format(root))) + local out = {} + for line in p:lines() do out[#out + 1] = line end + p:close() + return out +end + +local function split_lines(s) + local lines = {} + for line in (s .. '\n'):gmatch('([^\n]*)\n') do + lines[#lines + 1] = line + end + return lines +end + +local function line_window(lines, i, before, after) + local out = {} + local first = math.max(1, i - before) + local last = math.min(#lines, i + after) + for n = first, last do out[#out + 1] = lines[n] end + return table.concat(out, '\n') +end + +local function looks_like_service_cfg_reference(window) + local needles = { + "{ 'cfg'", + '{ "cfg"', + "'cfg'", + '"cfg"', + 'topics.config()', + 'topics.cfg()', + "config_topic", + "cfg_topic", + } + for _, needle in ipairs(needles) do + if window:find(needle, 1, true) then return true end + end + return false +end + +function tests.test_modern_service_shells_use_shared_config_watch_helper() + for service, path in pairs(modern_services) do + local s = read_file(path) + if not s:find("devicecode.support.config_watch", 1, true) then + fail(service .. ' service shell does not require devicecode.support.config_watch') + end + if not s:find('config_watch.open', 1, true) then + fail(service .. ' service shell does not open configuration through config_watch.open') + end + end +end + +function tests.test_modern_service_shells_do_not_subscribe_to_cfg_directly() + for service, path in pairs(modern_services) do + local s = read_file(path) + if s:find('conn:subscribe', 1, true) or s:find('bus_cleanup.subscribe', 1, true) then + fail(service .. ' service shell subscribes directly instead of using config_watch') + end + end +end + +function tests.test_modern_services_do_not_add_direct_cfg_subscription_regressions() + for _, root in ipairs(modern_roots) do + for _, path in ipairs(scan_files(root)) do + local lines = split_lines(read_file(path)) + for i, line in ipairs(lines) do + if line:find(':subscribe%s*%(') or line:find('bus_cleanup%.subscribe%s*%(') then + local window = line_window(lines, i, 4, 6) + if looks_like_service_cfg_reference(window) then + fail(path .. ':' .. tostring(i) .. ' appears to subscribe to cfg directly; use config_watch.open') + end + end + end + end + end +end + +function tests.test_config_watch_helper_is_the_only_shared_helper_that_subscribes_to_cfg_service_directly() + local helper = read_file('../src/devicecode/support/config_watch.lua') + if not helper:find('bus_cleanup.subscribe', 1, true) then + fail('config_watch helper should own the local bus subscription') + end + if not helper:find("{ 'cfg', service }", 1, true) then + fail('config_watch helper should own cfg/ topic construction') + end +end + +return tests diff --git a/tests/unit/support/test_dependency_failure.lua b/tests/unit/support/test_dependency_failure.lua new file mode 100644 index 00000000..69f73f65 --- /dev/null +++ b/tests/unit/support/test_dependency_failure.lua @@ -0,0 +1,38 @@ +local dep_failure = require 'devicecode.support.dependency_failure' + +local T = {} + +function T.canonical_no_route_failure_is_strict_and_copied() + local detail = { err = 'no_route', nested = { a = 1 } } + local failure = dep_failure.no_route('transport:uart0', detail, { source = 'uart' }) + assert(failure.kind == 'dependency_failure') + assert(failure.err == 'no_route') + assert(failure.dependency_key == 'transport:uart0') + assert(failure.source == 'uart') + assert(dep_failure.is(failure)) + assert(dep_failure.is_no_route(failure)) + detail.nested.a = 2 + assert(failure.detail.nested.a == 1) +end + +function T.no_route_classifier_does_not_search_wrappers_or_nested_reason_tables() + assert(dep_failure.is_no_route('no_route')) + assert(dep_failure.is_no_route({ err = 'no_route' })) + assert(dep_failure.is_no_route({ result = { err = 'no_route' } })) + assert(not dep_failure.is_no_route({ primary = { err = 'no_route' } })) + assert(not dep_failure.is_no_route({ report = { primary = { err = 'no_route' } } })) + assert(not dep_failure.is_no_route({ children = { { primary = { err = 'no_route' } } } })) + assert(not dep_failure.is_no_route({ reason = { err = 'no_route' } })) +end + +function T.from_no_route_requires_key_and_direct_no_route_shape() + assert(dep_failure.from_no_route(nil, 'no_route') == nil) + assert(dep_failure.from_no_route('transport:uart0', { reason = { err = 'no_route' } }) == nil) + local failure = dep_failure.from_no_route('transport:uart0', { err = 'no_route' }, { link_id = 'uart0' }) + assert(failure.kind == 'dependency_failure') + assert(failure.err == 'no_route') + assert(failure.dependency_key == 'transport:uart0') + assert(failure.link_id == 'uart0') +end + +return T diff --git a/tests/unit/support/test_dependency_slot.lua b/tests/unit/support/test_dependency_slot.lua new file mode 100644 index 00000000..c70035f6 --- /dev/null +++ b/tests/unit/support/test_dependency_slot.lua @@ -0,0 +1,59 @@ +local fibers = require 'fibers' +local busmod = require 'bus' +local runfibers = require 'tests.support.run_fibers' +local slot = require 'devicecode.support.dependency_slot' + +local T = {} + +local function assert_eq(a, b) + if a ~= b then error('expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end + +local function status_topic(class, id) + return { 'cap', class, id or 'main', 'status' } +end + +function T.replace_opens_projects_and_terminates_previous_dependencies() + runfibers.run(function () + local b = busmod.new() + local conn = b:connect() + local writer = b:connect() + local state = {} + + assert(slot.replace(state, 'deps', conn, { + { key = 'one', class = 'one', id = 'main' }, + })) + assert(state.deps ~= nil) + writer:retain(status_topic('one', 'main'), { state = 'available', available = true }) + local ev = fibers.perform(state.deps:event_source():recv_op()) + assert_eq(ev.key, 'one') + assert_eq(slot.snapshot(state, 'deps').one.available, true) + + assert(slot.replace(state, 'deps', conn, { + { key = 'two', class = 'two', id = 'main' }, + })) + assert(state.deps ~= nil) + assert(slot.snapshot(state, 'deps').one == nil) + writer:retain(status_topic('two', 'main'), { state = 'available', available = true }) + ev = fibers.perform(state.deps:event_source():recv_op()) + assert_eq(ev.key, 'two') + + slot.terminate(state, 'deps', 'test_complete') + assert(state.deps == nil) + end) +end + +function T.empty_specs_clear_slot_without_opening_manager() + runfibers.run(function () + local state = { deps = { terminate = function (self, reason) self.reason = reason end } } + local ok, err, deps = slot.replace(state, 'deps', nil, {}, { replace_reason = 'empty' }) + assert_eq(ok, true) + assert_eq(err, nil) + assert_eq(deps, nil) + assert(state.deps == nil) + local src = slot.event_source(state, 'deps', { name = 'unused' }) + assert_eq(src, nil) + end) +end + +return T diff --git a/tests/unit/support/test_priority_event.lua b/tests/unit/support/test_priority_event.lua new file mode 100644 index 00000000..222471de --- /dev/null +++ b/tests/unit/support/test_priority_event.lua @@ -0,0 +1,137 @@ +-- tests/unit/support/test_priority_event.lua + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local op = require 'fibers.op' +local queue = require 'devicecode.support.queue' +local priority_event = require 'devicecode.support.priority_event' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_eq(a, b, msg) + if a ~= b then + fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) + end +end + +local function assert_true(v, msg) + if v ~= true then + fail(msg or ('expected true, got ' .. tostring(v))) + end +end + +local function assert_nil(v, msg) + if v ~= nil then + fail(msg or ('expected nil, got ' .. tostring(v))) + end +end + +local function recv_event(rx, kind) + return function () + local item, err = queue.try_recv_now(rx) + if item ~= nil then + return { kind = kind, value = item } + end + if err ~= 'not_ready' then + return { kind = kind .. '_closed' } + end + return nil + end +end + +local function recv_op(rx, kind) + return function () + return rx:recv_op():wrap(function (item) + if item == nil then + return { kind = kind .. '_closed' } + end + return { kind = kind, value = item } + end) + end +end + +function tests.test_sources_op_selects_ready_sources_by_explicit_order() + fibers.run(function () + local low_tx, low_rx = mailbox.new(4, { full = 'reject_newest' }) + local high_tx, high_rx = mailbox.new(4, { full = 'reject_newest' }) + assert_true(low_tx:send('low-one')) + assert_true(high_tx:send('high-one')) + + local pending = {} + local function next_event_op() + return priority_event.sources_op { + label = 'test.priority.ready_order', + pending = pending, + sources = { + { name = 'high', try_now = recv_event(high_rx, 'high'), recv_op = recv_op(high_rx, 'high') }, + { name = 'low', try_now = recv_event(low_rx, 'low'), recv_op = recv_op(low_rx, 'low') }, + }, + } + end + + local ev = fibers.perform(next_event_op()) + assert_eq(ev.kind, 'high') + assert_eq(ev.value, 'high-one') + + ev = fibers.perform(next_event_op()) + assert_eq(ev.kind, 'low') + assert_eq(ev.value, 'low-one') + end) +end + +function tests.test_next_op_treats_blocking_winner_as_wake_and_rechecks_priority() + fibers.run(function () + local pending = {} + local high_tx, high_rx = mailbox.new(1, { full = 'reject_newest' }) + + local selected = fibers.perform(priority_event.next_op { + label = 'test.priority.recheck_after_wake', + select_now = function () + local high, err = queue.try_recv_now(high_rx) + if high ~= nil then + return { kind = 'high', value = high } + end + assert_eq(err, 'not_ready') + + local low = priority_event.take_pending(pending, 'low') + if low ~= nil then + return low + end + return nil + end, + wait_op = function () + return op.always('low', { kind = 'low', value = 'blocked-low' }) + end, + store_wake = function (name, ev) + pending[name] = ev + assert_true(high_tx:send('became-ready-high')) + end, + }) + + assert_eq(selected.kind, 'high') + assert_eq(selected.value, 'became-ready-high') + + local low = priority_event.take_pending(pending, 'low') + assert_eq(low.kind, 'low') + assert_eq(low.value, 'blocked-low') + end) +end + +function tests.test_next_op_allows_no_event_when_explicitly_requested() + fibers.run(function () + local ev = fibers.perform(priority_event.next_op { + label = 'test.priority.no_event', + allow_no_event = true, + select_now = function () return nil end, + wait_op = function () return op.always('woke') end, + }) + + assert_nil(ev) + end) +end + +return tests diff --git a/tests/unit/support/test_queue.lua b/tests/unit/support/test_queue.lua new file mode 100644 index 00000000..460881ab --- /dev/null +++ b/tests/unit/support/test_queue.lua @@ -0,0 +1,220 @@ +-- tests/devicecode/support/test_queue.lua + +local fibers = require 'fibers' +local scope = require 'fibers.scope' +local mailbox = require 'fibers.mailbox' +local queue = require 'devicecode.support.queue' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_true(v, msg) + if v ~= true then + fail(msg or ('expected true, got ' .. tostring(v))) + end +end + +local function assert_false(v, msg) + if v ~= false then + fail(msg or ('expected false, got ' .. tostring(v))) + end +end + +local function assert_nil(v, msg) + if v ~= nil then + fail(msg or ('expected nil, got ' .. tostring(v))) + end +end + +local function assert_not_nil(v, msg) + if v == nil then + fail(msg or 'expected non-nil value') + end +end + +local function assert_eq(a, b, msg) + if a ~= b then + fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) + end +end + +local function assert_match(s, pat, msg) + if type(s) ~= 'string' or not s:match(pat) then + fail(msg or ('expected "' .. tostring(s) .. '" to match ' .. tostring(pat))) + end +end + +------------------------------------------------------------------------------- +-- try_send_now accepts immediate buffered admission +------------------------------------------------------------------------------- + +function tests.test_try_send_now_accepts_buffered_admission() + fibers.run(function () + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = queue.try_send_now(tx, 'hello') + + assert_true(ok) + assert_nil(err) + + local item = fibers.perform(rx:recv_op()) + assert_eq(item, 'hello') + end) +end + +------------------------------------------------------------------------------- +-- try_send_now does not wait when rendezvous send would block +------------------------------------------------------------------------------- + +function tests.test_try_send_now_returns_would_block_for_rendezvous_without_receiver() + fibers.run(function () + local tx = mailbox.new(0, { full = 'block' }) + + local ok, err = queue.try_send_now(tx, 'hello') + + assert_nil(ok) + assert_eq(err, 'would_block') + end) +end + +------------------------------------------------------------------------------- +-- try_send_now preserves bounded rejection semantics +------------------------------------------------------------------------------- + +function tests.test_try_send_now_preserves_reject_newest_full_result() + fibers.run(function () + local tx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = queue.try_send_now(tx, 'one') + assert_true(ok) + assert_nil(err) + + ok, err = queue.try_send_now(tx, 'two') + assert_false(ok) + assert_eq(err, 'full') + end) +end + +------------------------------------------------------------------------------- +-- try_send_now reports closed send side +------------------------------------------------------------------------------- + +function tests.test_try_send_now_reports_closed_mailbox() + fibers.run(function () + local tx = mailbox.new(1, { full = 'reject_newest' }) + + tx:close('closed_for_test') + + local ok, err = queue.try_send_now(tx, 'hello') + + assert_nil(ok) + assert_eq(err, 'closed') + end) +end + +------------------------------------------------------------------------------- +-- try_recv_now receives immediately available values +------------------------------------------------------------------------------- + +function tests.test_try_recv_now_receives_available_value() + fibers.run(function () + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = queue.try_send_now(tx, 'hello') + assert_true(ok) + assert_nil(err) + + local item, rerr = queue.try_recv_now(rx) + + assert_eq(item, 'hello') + assert_nil(rerr) + end) +end + +------------------------------------------------------------------------------- +-- try_recv_now does not wait when no value is available +------------------------------------------------------------------------------- + +function tests.test_try_recv_now_returns_not_ready_when_empty_and_open() + fibers.run(function () + local _, rx = mailbox.new(1, { full = 'reject_newest' }) + + local item, err = queue.try_recv_now(rx) + + assert_nil(item) + assert_eq(err, 'not_ready') + end) +end + + +------------------------------------------------------------------------------- +-- try_recv_now reports closed receive side distinctly from not-ready +------------------------------------------------------------------------------- + +function tests.test_try_recv_now_reports_closed_mailbox() + fibers.run(function () + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + tx:close('closed_for_test') + local expected = tostring(rx:why() or 'closed') + + local item, err = queue.try_recv_now(rx) + + assert_nil(item) + assert_eq(err, expected) + if err == 'not_ready' then fail('closed mailbox reported not_ready') end + end) +end + +------------------------------------------------------------------------------- +-- assert_admit_required raises on non-immediate admission failure +------------------------------------------------------------------------------- + +function tests.test_assert_admit_required_raises_on_would_block() + fibers.run(function () + local tx = mailbox.new(0, { full = 'block' }) + + local ok, err = pcall(function () + queue.assert_admit_required(tx, 'hello', 'completion_report') + end) + + assert_false(ok) + assert_match(err, 'completion_report') + assert_match(err, 'would_block') + end) +end + +------------------------------------------------------------------------------- +-- helpers still use scope-aware perform +------------------------------------------------------------------------------- + +function tests.test_try_send_now_still_observes_scope_cancellation() + fibers.run(function (root_scope) + local child = assert(root_scope:child()) + + local ok_spawn, spawn_err = child:spawn(function (s) + s:cancel('cancelled_for_test') + + local ok, err = pcall(function () + local tx = mailbox.new(1, { full = 'reject_newest' }) + return queue.try_send_now(tx, 'hello') + end) + + assert_false(ok) + assert_true(scope.is_cancelled(err), 'expected scope cancellation sentinel') + assert_eq(scope.cancel_reason(err), 'cancelled_for_test') + end) + + assert_true(ok_spawn, spawn_err) + + local st, rep, primary = fibers.perform(child:join_op()) + + assert_eq(st, 'cancelled') + assert_eq(primary, 'cancelled_for_test') + assert_eq(#rep.extra_errors, 0) + end) +end + +return tests diff --git a/tests/unit/support/test_request_owner.lua b/tests/unit/support/test_request_owner.lua new file mode 100644 index 00000000..108e34dd --- /dev/null +++ b/tests/unit/support/test_request_owner.lua @@ -0,0 +1,135 @@ +-- tests/unit/support/test_request_owner.lua + +local request_owner = require 'devicecode.support.request_owner' +local fibers = require 'fibers' +local op = require 'fibers.op' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_true(v, msg) + if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end +end + +local function assert_false(v, msg) + if v ~= false then fail(msg or ('expected false, got ' .. tostring(v))) end +end + +local function assert_eq(a, b, msg) + if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end +end + +local Request = {} +Request.__index = Request + +function Request:reply(value) + self.replies = (self.replies or 0) + 1 + self.value = value + return true +end + +function Request:fail(reason) + self.fails = (self.fails or 0) + 1 + self.reason = reason + return true +end + +local function new_request() + return setmetatable({}, Request) +end + +function tests.test_reply_once_prevents_later_finalise() + local req = new_request() + local owner = request_owner.new(req) + + assert_true(owner:reply_once('ok')) + assert_false(owner:finalise_unresolved('closed')) + + assert_eq(req.replies, 1) + assert_eq(req.value, 'ok') + assert_eq(req.fails, nil) +end + +function tests.test_fail_once_prevents_later_reply() + local req = new_request() + local owner = request_owner.new(req) + + assert_true(owner:fail_once('bad')) + assert_false(owner:reply_once('late')) + + assert_eq(req.fails, 1) + assert_eq(req.reason, 'bad') + assert_eq(req.replies, nil) +end + +function tests.test_finalise_unresolved_fails_unresolved_request_once() + local req = new_request() + local owner = request_owner.new(req) + + assert_true(owner:finalise_unresolved('terminated')) + assert_false(owner:finalise_unresolved('late')) + + assert_eq(req.fails, 1) + assert_eq(req.reason, 'terminated') + assert_eq(req.replies, nil) +end + +function tests.test_abandon_unresolved_resolves_without_reply_or_fail() + local req = new_request() + local owner = request_owner.new(req) + + assert_true(owner:abandon_unresolved('client_closed')) + assert_false(owner:reply_once('late')) + assert_false(owner:fail_once('late')) + + assert_eq(req.fails, nil) + assert_eq(req.replies, nil) +end + + +function tests.test_caller_cancel_op_abandons_on_bus_request_abandoned() + fibers.run(function () + local req = new_request() + function req:done_op() + return op.always('abandoned', nil, 'timeout') + end + + local owner = request_owner.new(req) + local reason = fibers.perform(owner:caller_cancel_op()) + + assert_eq(reason, 'timeout') + assert_true(owner:done()) + assert_false(owner:reply_once('late')) + assert_eq(req.replies, nil) + assert_eq(req.fails, nil) + end) +end + +function tests.test_caller_cancel_op_ignores_non_abandoned_done_status() + fibers.run(function () + local req = new_request() + function req:done_op() + return op.always('replied', 'ok', nil) + end + + local owner = request_owner.new(req) + local reason = fibers.perform(owner:caller_cancel_op()) + + assert_false(reason) + assert_false(owner:done()) + assert_true(owner:reply_once('late')) + assert_eq(req.replies, 1) + end) +end + +function tests.test_caller_cancel_op_requires_request_done_op() + local owner = request_owner.new(new_request()) + local cancel_op, err = owner:caller_cancel_op() + assert_eq(cancel_op, nil) + assert_eq(err, 'request has no done_op') +end + +return tests diff --git a/tests/unit/support/test_resource.lua b/tests/unit/support/test_resource.lua new file mode 100644 index 00000000..cb3aff34 --- /dev/null +++ b/tests/unit/support/test_resource.lua @@ -0,0 +1,299 @@ +-- tests/unit/support/test_resource.lua +-- +-- Unit tests for devicecode.support.resource. + +local resource = require 'devicecode.support.resource' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_true(v, msg) + if v ~= true then + fail(msg or ('expected true, got ' .. tostring(v))) + end +end + +local function assert_nil(v, msg) + if v ~= nil then + fail(msg or ('expected nil, got ' .. tostring(v))) + end +end + +local function assert_false(v, msg) + if v ~= false then + fail(msg or ('expected false, got ' .. tostring(v))) + end +end + +local function assert_eq(a, b, msg) + if a ~= b then + fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) + end +end + +local function assert_match(s, pat, msg) + if type(s) ~= 'string' or not s:match(pat) then + fail(msg or ('expected "' .. tostring(s) .. '" to match ' .. tostring(pat))) + end +end + +function tests.test_terminate_nil_succeeds() + local ok, err = resource.terminate(nil, 'shutdown') + + assert_true(ok) + assert_nil(err) +end + +function tests.test_terminate_rejects_function_resource() + local ok, err = resource.terminate(function () + error('function cleanup must not be called') + end, 'shutdown') + + assert_nil(ok) + assert_match(err, 'no terminate') +end + +function tests.test_terminate_passes_self_and_reason_to_table_method() + local obj = {} + + function obj:terminate(reason) + self.seen_self = self + self.seen_reason = reason + return true + end + + local ok, err = resource.terminate(obj, 'closing') + + assert_true(ok) + assert_nil(err) + assert_eq(obj.seen_self, obj) + assert_eq(obj.seen_reason, 'closing') +end + +function tests.test_terminate_rejects_close_only_resource() + local obj = { + close = function () + error('close must not be called') + end, + } + + local ok, err = resource.terminate(obj, 'closing') + + assert_nil(ok) + assert_match(err, 'no terminate') +end + +function tests.test_terminate_rejects_close_op_only_resource() + local obj = { + close_op = function () + error('close_op must not be called') + end, + } + + local ok, err = resource.terminate(obj, 'closing') + + assert_nil(ok) + assert_match(err, 'no terminate') +end + +function tests.test_terminate_reports_false_return_failure() + local obj = { + terminate = function () + return false, 'terminate rejected' + end, + } + + local ok, err = resource.terminate(obj, 'closing') + + assert_nil(ok) + assert_eq(err, 'terminate rejected') +end + +function tests.test_terminate_reports_nil_error_failure() + local obj = { + terminate = function () + return nil, 'terminate unavailable' + end, + } + + local ok, err = resource.terminate(obj, 'closing') + + assert_nil(ok) + assert_eq(err, 'terminate unavailable') +end + +function tests.test_terminate_reports_failure_fallback_when_error_missing() + local obj = { + terminate = function () + return false + end, + } + + local ok, err = resource.terminate(obj, 'closing') + + assert_nil(ok) + assert_match(err, 'termination failed') +end + +function tests.test_terminate_catches_raised_cleanup_error() + local obj = { + terminate = function () + error('boom', 0) + end, + } + + local ok, err = resource.terminate(obj, 'closing') + + assert_nil(ok) + assert_match(err, 'boom') +end + +function tests.test_terminate_accepts_truthy_non_boolean_success() + local obj = { + terminate = function () + return 'terminated' + end, + } + + local ok, err = resource.terminate(obj, 'closing') + + assert_true(ok) + assert_nil(err) +end + +function tests.test_terminate_checked_succeeds_when_cleanup_succeeds() + local obj = { + terminate = function () return true end, + } + + local ok, ret = pcall(function () + return resource.terminate_checked(obj, 'closing', 'source cleanup') + end) + + assert_true(ok) + assert_true(ret) +end + +function tests.test_terminate_checked_raises_labelled_failure() + local obj = { + terminate = function () + return nil, 'device refused terminate' + end, + } + + local ok, err = pcall(function () + resource.terminate_checked(obj, 'closing', 'source cleanup') + end) + + assert_false(ok) + assert_match(err, 'source cleanup') + assert_match(err, 'device refused terminate') +end + +function tests.test_owned_resource_terminates_when_not_handed_off() + local terminated = 0 + local seen_reason + local obj = { + terminate = function (_, reason) + terminated = terminated + 1 + seen_reason = reason + return true + end, + } + + local lease = resource.owned(obj) + assert_true(lease:is_owned()) + assert_eq(lease:value(), obj) + + local ok, err = lease:terminate('scope closed') + assert_true(ok) + assert_nil(err) + assert_eq(terminated, 1) + assert_eq(seen_reason, 'scope closed') + assert_true(lease:terminate('again')) + assert_eq(terminated, 1, 'owned lease terminate must be idempotent') +end + +function tests.test_owned_resource_terminate_checked_raises_labelled_failure() + local lease = resource.owned({ + terminate = function () return nil, 'nope' end, + }) + + local ok, err = pcall(function () + lease:terminate_checked('closed', 'lease cleanup') + end) + + assert_false(ok) + assert_match(err, 'lease cleanup') + assert_match(err, 'nope') +end + +function tests.test_handoff_installs_receiver_before_releasing_owner_cleanup() + local owner_terminated = 0 + local receiver_installed = false + local obj = { + terminate = function () + owner_terminated = owner_terminated + 1 + return true + end, + } + + local lease = resource.owned(obj) + local handed, err = lease:handoff(function (v) + assert_eq(v, obj) + receiver_installed = true + return true + end) + + assert_eq(handed, obj) + assert_nil(err) + assert_true(receiver_installed) + assert_false(lease:is_owned()) + assert_true(lease:terminate('child finalised')) + assert_eq(owner_terminated, 0, 'handoff must disable child cleanup') +end + +function tests.test_handoff_receiver_failure_keeps_child_cleanup_ownership() + local terminated = 0 + local obj = { + terminate = function () + terminated = terminated + 1 + return true + end, + } + + local lease = resource.owned(obj) + local handed, err = lease:handoff(function () + return nil, 'receiver refused' + end) + + assert_nil(handed) + assert_match(err, 'receiver refused') + assert_true(lease:is_owned(), 'failed handoff must keep original cleanup owner') + assert_true(lease:terminate('cleanup')) + assert_eq(terminated, 1) +end + +function tests.test_detach_releases_cleanup_ownership() + local terminated = 0 + local obj = { + terminate = function () + terminated = terminated + 1 + return true + end, + } + + local lease = resource.owned(obj) + local detached, err = lease:detach() + + assert_eq(detached, obj) + assert_nil(err) + assert_false(lease:is_owned()) + assert_true(lease:terminate('later')) + assert_eq(terminated, 0) +end + +return tests diff --git a/tests/unit/support/test_retained_publish.lua b/tests/unit/support/test_retained_publish.lua new file mode 100644 index 00000000..79a17d4c --- /dev/null +++ b/tests/unit/support/test_retained_publish.lua @@ -0,0 +1,78 @@ +local retained = require 'devicecode.support.retained_publish' + +local tests = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +local function topic_key(t) + local parts = {} + for i = 1, #(t or {}) do parts[#parts + 1] = tostring(t[i]) end + return table.concat(parts, '/') +end + +local function fake_conn() + return { + retains = {}, + unretains = {}, + retain = function(self, topic, payload) + self.retains[#self.retains + 1] = { topic = topic_key(topic), payload = payload } + return true + end, + unretain = function(self, topic) + self.unretains[#self.unretains + 1] = topic_key(topic) + return true + end, + } +end + +function tests.test_retain_if_changed_suppresses_identical_payloads() + local conn = fake_conn() + local cache = {} + local ret, err, changed = retained.retain_if_changed(conn, cache, 'a', { 'state', 'x' }, { value = 1 }) + ok(ret, err) + eq(changed, true) + eq(#conn.retains, 1) + ret, err, changed = retained.retain_if_changed(conn, cache, 'a', { 'state', 'x' }, { value = 1 }) + ok(ret, err) + eq(changed, false) + eq(#conn.retains, 1) + ret, err, changed = retained.retain_if_changed(conn, cache, 'a', { 'state', 'x' }, { value = 2 }) + ok(ret, err) + eq(changed, true) + eq(#conn.retains, 2) +end + +function tests.test_unretain_if_present_is_idempotent() + local conn = fake_conn() + local cache = { a = { value = 1 } } + local ret, err, changed = retained.unretain_if_present(conn, cache, 'a', { 'state', 'x' }) + ok(ret, err) + eq(changed, true) + eq(#conn.unretains, 1) + ret, err, changed = retained.unretain_if_present(conn, cache, 'a', { 'state', 'x' }) + ok(ret, err) + eq(changed, false) + eq(#conn.unretains, 1) +end + +function tests.test_publish_map_changed_retains_changed_and_unretains_removed_entries() + local conn = fake_conn() + local cache = {} + local topic = function(id) return { 'state', 'item', id } end + local payload = function(rec) return rec end + local ret, err, changed = retained.publish_map_changed(conn, cache, { a = { n = 1 }, b = { n = 2 } }, topic, payload) + ok(ret, err) + eq(changed, 2) + eq(#conn.retains, 2) + ret, err, changed = retained.publish_map_changed(conn, cache, { a = { n = 1 }, c = { n = 3 } }, topic, payload) + ok(ret, err) + eq(changed, 2) + eq(#conn.retains, 3) + eq(#conn.unretains, 1) + eq(conn.unretains[1], 'state/item/b') +end + +return tests diff --git a/tests/unit/support/test_scoped_work.lua b/tests/unit/support/test_scoped_work.lua new file mode 100644 index 00000000..85301448 --- /dev/null +++ b/tests/unit/support/test_scoped_work.lua @@ -0,0 +1,885 @@ +-- tests/unit/support/test_scoped_work.lua +-- +-- Standalone semantic tests for devicecode.support.scoped_work. +-- +-- Run with package.path pointing at src/, for example: +-- +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' +local cond = require 'fibers.cond' +local scoped_work = require 'devicecode.support.scoped_work' +local queue = require 'devicecode.support.queue' + +local tests = {} + +local function fail(msg) + error(msg or 'assertion failed', 2) +end + +local function assert_true(v, msg) + if v ~= true then + fail(msg or ('expected true, got ' .. tostring(v))) + end +end + +local function assert_false(v, msg) + if v ~= false then + fail(msg or ('expected false, got ' .. tostring(v))) + end +end + +local function assert_nil(v, msg) + if v ~= nil then + fail(msg or ('expected nil, got ' .. tostring(v))) + end +end + +local function assert_not_nil(v, msg) + if v == nil then + fail(msg or 'expected non-nil value') + end +end + +local function assert_eq(a, b, msg) + if a ~= b then + fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) + end +end + +local function assert_match(s, pat, msg) + if type(s) ~= 'string' or not s:match(pat) then + fail(msg or ('expected "' .. tostring(s) .. '" to match ' .. tostring(pat))) + end +end + +local function child_scope(parent) + local child, err = parent:child() + assert_not_nil(child, err) + return child +end + +local function capture_next_child(parent) + local old_child = parent.child + local captured + + function parent:child(...) + local child, err + if old_child then + child, err = old_child(self, ...) + else + child, err = getmetatable(self).__index.child(self, ...) + end + + captured = child + return child, err + end + + return function () + parent.child = old_child + return captured + end +end + +local function start_required(spec) + local handle, err = scoped_work.start(spec) + assert_not_nil(handle, err) + return handle +end + +------------------------------------------------------------------------------- +-- 1. body-ended happens before ordinary reaping +------------------------------------------------------------------------------- + +function tests.test_body_ended_happens_before_ordinary_reaping() + fibers.run(function (scope) + local late_spawn_ok = false + local late_child_ran = false + + local handle = start_required { + lifetime_scope = scope, + + identity = { + kind = 'body_done_test', + id = 'work-1', + }, + + run = function (work_scope) + -- Give an unsafe reaper a chance to run. If it joined immediately, + -- it would close child admission before the spawn below. + fibers.perform(sleep.sleep_op(0.001)) + + local ok, err = work_scope:spawn(function () + late_child_ran = true + end) + + late_spawn_ok = ok == true + + if not ok then + error(err or 'late spawn rejected', 0) + end + + return { + ok = true, + } + end, + } + + local ev = fibers.perform(handle:outcome_op()) + + assert_eq(ev.kind, 'body_done_test') + assert_eq(ev.status, 'ok') + assert_true(late_spawn_ok, 'late child spawn should be admitted before reaping') + assert_true(late_child_ran, 'late child should be joined before outcome is stored') + end) +end + +------------------------------------------------------------------------------- +-- 2. reporter does not call join_op +------------------------------------------------------------------------------- + +function tests.test_reporter_does_not_call_join_op() + fibers.run(function (scope) + local report_scope = child_scope(scope) + local report_tx, report_rx = mailbox.new(4, { full = 'reject_newest' }) + local captured_child = capture_next_child(scope) + + local handle = start_required { + lifetime_scope = scope, + reaper_scope = scope, + report_scope = report_scope, + + identity = { + kind = 'reporter_join_test', + id = 'work-2', + }, + + run = function () + fibers.perform(sleep.sleep_op(0.001)) + return { + ok = true, + } + end, + + report = function (ev) + return queue.try_admit_required(report_tx, ev, 'report_failed') + end, + } + + local child = captured_child() + assert_not_nil(child, 'expected captured child scope') + local old_join_op = child.join_op + local join_count = 0 + + function child:join_op(...) + join_count = join_count + 1 + return old_join_op(self, ...) + end + + local ev = fibers.perform(report_rx:recv_op()) + + assert_not_nil(ev, 'expected reported completion') + assert_eq(ev.kind, 'reporter_join_test') + assert_eq(ev.status, 'ok') + assert_eq(join_count, 1, 'only the authorised reaper should call join_op') + end) +end + +------------------------------------------------------------------------------- +-- 3. non-parent / non-lifetime reaping requires explicit delegation +------------------------------------------------------------------------------- + +function tests.test_non_lifetime_reaping_requires_delegation() + fibers.run(function (scope) + local reaper_scope = child_scope(scope) + + local handle, err = scoped_work.start { + lifetime_scope = scope, + reaper_scope = reaper_scope, + + identity = { + kind = 'delegation_test', + id = 'work-3', + }, + + run = function () + return { + ok = true, + } + end, + } + + assert_nil(handle) + assert_match(err, 'reaping requires explicit reaper_delegation') + end) +end + +------------------------------------------------------------------------------- +-- 4. stored completion survives reporter cancellation +------------------------------------------------------------------------------- + +function tests.test_stored_completion_survives_reporter_cancellation() + fibers.run(function (scope) + local report_scope = child_scope(scope) + local report_count = 0 + + local handle = start_required { + lifetime_scope = scope, + reaper_scope = scope, + report_scope = report_scope, + + identity = { + kind = 'reporter_cancel_test', + id = 'work-4', + }, + + run = function () + fibers.perform(sleep.sleep_op(0.001)) + return { + ok = true, + value = 42, + } + end, + + report = function (_) + report_count = report_count + 1 + return true + end, + } + + -- The reporter scope is cancelled before the outcome exists. This must + -- not prevent the authorised reaper from storing the outcome. + report_scope:cancel('observer stopped') + + local ev = fibers.perform(handle:outcome_op()) + + assert_eq(ev.kind, 'reporter_cancel_test') + assert_eq(ev.status, 'ok') + assert_eq(ev.result.value, 42) + assert_eq(report_count, 0, 'cancelled reporter should not report') + end) +end + +------------------------------------------------------------------------------- +-- 5. start failure leaves no live child +------------------------------------------------------------------------------- + +function tests.test_start_failure_leaves_no_live_child() + fibers.run(function (scope) + local lifetime_scope = child_scope(scope) + local report_scope = child_scope(scope) + + report_scope:cancel('reporter unavailable') + + local handle, err = scoped_work.start { + lifetime_scope = lifetime_scope, + report_scope = report_scope, + + identity = { + kind = 'start_failure_test', + id = 'work-5', + }, + + run = function () + return { + ok = true, + } + end, + + report = function () + return true + end, + } + + assert_nil(handle) + assert_match(err, 'report_scope is not running') + + local st, rep = fibers.perform(lifetime_scope:join_op()) + + assert_eq(st, 'ok') + assert_eq(#rep.children, 0, 'failed start should not create an attached work child') + end) +end + + +------------------------------------------------------------------------------- +-- 6. worker is not admitted until reporter infrastructure is ready +------------------------------------------------------------------------------- + +function tests.test_worker_not_started_when_reporter_spawn_fails() + fibers.run(function (scope) + local lifetime_scope = child_scope(scope) + local worker_started = false + local reporter_spawn_attempted = false + + local fake_report_scope = { + status = function () + return 'running', nil + end, + + admission = function () + return 'open', nil + end, + + spawn = function () + reporter_spawn_attempted = true + return false, 'synthetic reporter spawn failure' + end, + } + + local handle, err = scoped_work.start { + lifetime_scope = lifetime_scope, + report_scope = fake_report_scope, + + identity = { + kind = 'admission_order_test', + id = 'work-6', + }, + + run = function () + worker_started = true + return { + ok = true, + } + end, + + report = function () + return true + end, + } + + assert_nil(handle) + assert_match(err, 'synthetic reporter spawn failure') + assert_true(reporter_spawn_attempted, 'reporter spawn should have been attempted') + assert_false(worker_started, 'worker must not start until reporting infrastructure is admitted') + + local st, rep = fibers.perform(lifetime_scope:join_op()) + assert_eq(st, 'ok') + assert_eq(#rep.children, 0, 'failed start should not leave an attached work child') + end) +end + + +------------------------------------------------------------------------------- +-- 7. success path does not start join inline from scoped_work.start +------------------------------------------------------------------------------- + +function tests.test_success_path_does_not_run_join_inline() + fibers.run(function (scope) + local release = cond.new() + local captured_child = capture_next_child(scope) + + local handle = start_required { + lifetime_scope = scope, + + identity = { + kind = 'no_inline_join_test', + id = 'work-7', + }, + + run = function () + fibers.perform(release:wait_op()) + return { ok = true } + end, + } + + local child = captured_child() + assert_not_nil(child, 'expected captured child scope') + local old_join_op = child.join_op + local join_count = 0 + + function child:join_op(...) + join_count = join_count + 1 + return old_join_op(self, ...) + end + + fibers.perform(sleep.sleep_op(0.001)) + assert_eq(join_count, 0, 'successful start must not join the child inline') + assert_nil(handle:outcome(), 'outcome should not be stored before body ends') + + release:signal() + + local ev = fibers.perform(handle:outcome_op()) + assert_eq(ev.status, 'ok') + assert_eq(join_count, 1, 'authorised reaper should join after body-ended') + end) +end + +------------------------------------------------------------------------------- +-- 8. normal start failure is coordinator-safe and leaves structural cleanup to parent join +------------------------------------------------------------------------------- + +function tests.test_normal_start_failure_does_not_join_inline_and_parent_reports_child() + fibers.run(function (scope) + local lifetime_scope = child_scope(scope) + local reaper_spawn_attempted = false + local worker_started = false + + local fake_reaper_scope = { + status = function () + return 'running', nil + end, + + admission = function () + return 'open', nil + end, + + spawn = function () + reaper_spawn_attempted = true + return false, 'synthetic reaper spawn failure' + end, + } + + local handle, err = scoped_work.start { + lifetime_scope = lifetime_scope, + reaper_scope = fake_reaper_scope, + reaper_delegation = 'unit-test delegated reaper', + + identity = { + kind = 'reaper_start_failure_test', + id = 'work-8', + }, + + run = function () + worker_started = true + return { ok = true } + end, + } + + assert_nil(handle) + assert_match(err, 'synthetic reaper spawn failure') + assert_true(reaper_spawn_attempted) + assert_false(worker_started, 'worker must not be admitted when reaper infrastructure fails') + + local st, rep = fibers.perform(lifetime_scope:join_op()) + assert_eq(st, 'ok') + assert_eq(#rep.children, 1, 'coordinator-safe failed start should leave parent join to account for child') + assert_eq(rep.children[1].status, 'cancelled') + end) +end + + +------------------------------------------------------------------------------- +-- 8b. setup-checked start failure performs eager cleanup and leaves no child +------------------------------------------------------------------------------- + +function tests.test_setup_checked_start_failure_performs_eager_cleanup() + fibers.run(function (scope) + local lifetime_scope = child_scope(scope) + local fake_reaper_scope = { + status = function () return 'running', nil end, + admission = function () return 'open', nil end, + spawn = function () return false, 'synthetic reaper spawn failure' end, + } + + local handle, err = scoped_work.start_setup_checked { + lifetime_scope = lifetime_scope, + reaper_scope = fake_reaper_scope, + reaper_delegation = 'unit-test delegated reaper', + + identity = { + kind = 'setup_checked_failure_test', + id = 'work-8b', + }, + + run = function () + return { ok = true } + end, + } + + assert_nil(handle) + assert_match(err, 'synthetic reaper spawn failure') + + local st, rep = fibers.perform(lifetime_scope:join_op()) + assert_eq(st, 'ok') + assert_eq(#rep.children, 0, 'setup-checked failed start should eagerly reap the empty child') + end) +end + + +------------------------------------------------------------------------------- +-- 9. production handle does not expose raw child-scope join authority +------------------------------------------------------------------------------- + +function tests.test_production_handle_does_not_expose_join_authority() + fibers.run(function (scope) + local handle = start_required { + lifetime_scope = scope, + + identity = { + kind = 'handle_surface_test', + id = 'work-9', + }, + + run = function () + return { ok = true } + end, + } + + assert_eq(type(handle.cancel), 'function') + assert_eq(type(handle.outcome_op), 'function') + assert_eq(type(handle.outcome), 'function') + assert_eq(type(handle.identity), 'function') + assert_nil(handle.scope, 'production handle must not expose child scope') + assert_nil(handle.join_op, 'production handle must not expose join authority') + assert_nil(handle._scope, 'production handle must not carry raw child scope') + + local ev = fibers.perform(handle:outcome_op()) + assert_eq(ev.status, 'ok') + end) +end + + +------------------------------------------------------------------------------- +-- 10. report queue overflow fails observing scope, not child work scope +------------------------------------------------------------------------------- + +function tests.test_report_queue_overflow_fails_observer_not_child_work() + fibers.run(function (scope) + local report_scope = child_scope(scope) + local report_tx, _ = mailbox.new(0, { full = 'reject_newest' }) + + local handle = start_required { + lifetime_scope = scope, + reaper_scope = scope, + report_scope = report_scope, + + identity = { + kind = 'report_overflow_test', + id = 'work-10', + }, + + run = function () + return { ok = true } + end, + + report = function (ev) + return queue.try_admit_required(report_tx, ev, 'synthetic_report_overflow') + end, + } + + local ev = fibers.perform(handle:outcome_op()) + assert_eq(ev.status, 'ok', 'child work outcome should be stored as ok') + + local st, _, primary = fibers.perform(report_scope:join_op()) + assert_eq(st, 'failed') + assert_match(primary, 'synthetic_report_overflow') + end) +end + + +------------------------------------------------------------------------------- +-- 11. completion envelopes are copied at report and observe boundaries +------------------------------------------------------------------------------- + +function tests.test_completion_envelopes_are_copied_on_report_and_observe() + fibers.run(function (scope) + local report_tx, report_rx = mailbox.new(4, { full = 'reject_newest' }) + + local handle = start_required { + lifetime_scope = scope, + + identity = { + kind = 'completion_copy_test', + id = 'work-11', + }, + + run = function () + return { + value = 1, + } + end, + + report = function (ev) + ev.result.value = 99 + ev.kind = 'mutated_report' + return queue.try_admit_required(report_tx, ev, 'copy_report_failed') + end, + } + + local reported = fibers.perform(report_rx:recv_op()) + assert_eq(reported.kind, 'mutated_report') + assert_eq(reported.result.value, 99) + + local observed = fibers.perform(handle:outcome_op()) + assert_eq(observed.kind, 'completion_copy_test') + assert_eq(observed.result.value, 1) + + observed.result.value = 123 + observed.kind = 'mutated_observer' + + local observed_again = handle:outcome() + assert_eq(observed_again.kind, 'completion_copy_test') + assert_eq(observed_again.result.value, 1) + end) +end + + +------------------------------------------------------------------------------- +-- 12. successful result is snapshotted before child finalisers run +------------------------------------------------------------------------------- + +function tests.test_worker_result_is_snapshotted_before_finalisers_mutate_it() + fibers.run(function (scope) + local returned = { value = 1 } + + local handle = start_required { + lifetime_scope = scope, + + identity = { + kind = 'result_snapshot_before_finaliser_test', + id = 'work-12', + }, + + run = function (work_scope) + work_scope:finally(function () + returned.value = 99 + returned.extra = 'mutated by finaliser' + end) + + return returned + end, + } + + local ev = fibers.perform(handle:outcome_op()) + assert_eq(ev.status, 'ok') + assert_eq(ev.result.value, 1) + assert_nil(ev.result.extra) + end) +end + + +------------------------------------------------------------------------------- +-- 13. setup hook provides internal setup data without exposing it on handle +------------------------------------------------------------------------------- + +function tests.test_setup_hook_runs_before_worker_and_returns_internal_data() + fibers.run(function (scope) + local setup_seen_by_worker = false + local setup_child_status + + local handle, err, setup = scoped_work.start { + lifetime_scope = scope, + + identity = { + kind = 'setup_hook_test', + id = 'work-13', + }, + + setup = function (work_scope) + local owned_child, child_err = work_scope:child() + if not owned_child then + return nil, child_err + end + + owned_child:finally(function (_, status) + setup_child_status = status + end) + + return { + owned_child = owned_child, + marker = 'setup-complete', + } + end, + + run = function (_, setup_data) + assert_eq(setup_data.marker, 'setup-complete') + setup_seen_by_worker = true + return { ok = true } + end, + } + + assert_not_nil(handle, err) + assert_not_nil(setup) + assert_not_nil(setup.owned_child) + assert_nil(handle.setup, 'production handle must not expose setup data') + assert_nil(handle.scope, 'production handle must not expose work scope') + + local ev = fibers.perform(handle:outcome_op()) + assert_eq(ev.status, 'ok') + assert_true(setup_seen_by_worker) + assert_eq(setup_child_status, 'ok') + end) +end + +------------------------------------------------------------------------------- +-- 14. setup failure cancels the just-created child but does not start worker +------------------------------------------------------------------------------- + +function tests.test_setup_failure_prevents_worker_start_and_is_structurally_reported() + fibers.run(function (scope) + local lifetime_scope = child_scope(scope) + local worker_started = false + + local handle, err = scoped_work.start { + lifetime_scope = lifetime_scope, + + identity = { + kind = 'setup_failure_test', + id = 'work-14', + }, + + setup = function () + return nil, 'synthetic setup failure' + end, + + run = function () + worker_started = true + return { ok = true } + end, + } + + assert_nil(handle) + assert_match(err, 'synthetic setup failure') + assert_false(worker_started) + + local st, rep = fibers.perform(lifetime_scope:join_op()) + assert_eq(st, 'ok') + assert_eq(#rep.children, 1) + assert_eq(rep.children[1].status, 'cancelled') + assert_match(tostring(rep.children[1].primary), 'synthetic setup failure') + end) +end + + +------------------------------------------------------------------------------- +-- 15. setup-owned resources are finalised on post-setup start failure +------------------------------------------------------------------------------- + +function tests.test_post_setup_start_failure_runs_cancel_owned_now() + fibers.run(function (scope) + local lifetime_scope = child_scope(scope) + local finalised + local fake_report_scope = { + status = function () return 'running', nil end, + admission = function () return 'open', nil end, + spawn = function () return false, 'synthetic reporter spawn failure' end, + } + + local handle, err = scoped_work.start { + lifetime_scope = lifetime_scope, + report_scope = fake_report_scope, + + identity = { + kind = 'setup_owned_cancel_test', + id = 'work-15', + }, + + setup = function () + return { + cancel_owned_now = function (reason) + finalised = reason + return true + end, + } + end, + + run = function () + return { ok = true } + end, + + report = function () + return true + end, + } + + assert_nil(handle) + assert_match(err, 'synthetic reporter spawn failure') + assert_eq(finalised, 'synthetic reporter spawn failure') + end) +end + + +------------------------------------------------------------------------------- +-- 16. cancel_op cancels admitted child work and immediate owned resources +------------------------------------------------------------------------------- + +function tests.test_cancel_op_cancels_child_and_setup_owned_resources() + fibers.run(function (scope) + local cancel_tx, cancel_rx = mailbox.new(1, { full = 'reject_newest' }) + local cancel_reason + local worker_started = false + + local handle = start_required { + lifetime_scope = scope, + + identity = { + kind = 'cancel_op_test', + id = 'work-16', + }, + + setup = function () + return { + cancel_owned_now = function (reason) + cancel_reason = reason + return true + end, + } + end, + + cancel_op = cancel_rx:recv_op(), + + run = function () + worker_started = true + fibers.perform(sleep.sleep_op(10.0)) + return { ok = true } + end, + } + + assert_true(cancel_tx:send('caller_aborted')) + local ev = fibers.perform(handle:outcome_op()) + + -- The cancellation watcher may win before the worker body first runs; + -- the important contract is that admitted scoped work is cancelled and + -- setup-owned resources receive their immediate cancellation path. + assert_eq(cancel_reason, 'caller_aborted') + assert_eq(ev.kind, 'cancel_op_test') + assert_eq(ev.status, 'cancelled') + assert_eq(ev.primary, 'caller_aborted') + end) +end + +------------------------------------------------------------------------------- +-- 17. cancel_op losing to body_done must not rewrite successful completion +------------------------------------------------------------------------------- + +function tests.test_cancel_op_loses_to_body_done() + fibers.run(function (scope) + local cancel_tx, cancel_rx = mailbox.new(1, { full = 'reject_newest' }) + local cancel_reason + + local handle = start_required { + lifetime_scope = scope, + + identity = { + kind = 'cancel_op_loses_test', + id = 'work-17', + }, + + setup = function () + return { + cancel_owned_now = function (reason) + cancel_reason = reason + return true + end, + } + end, + + cancel_op = cancel_rx:recv_op(), + + run = function () + return { ok = true, value = 17 } + end, + } + + local ev = fibers.perform(handle:outcome_op()) + assert_eq(ev.status, 'ok') + assert_eq(ev.result.value, 17) + + assert_true(cancel_tx:send('late_cancel')) + fibers.perform(sleep.sleep_op(0.001)) + assert_eq(cancel_reason, nil) + end) +end + +return tests diff --git a/tests/unit/support/test_service_events.lua b/tests/unit/support/test_service_events.lua new file mode 100644 index 00000000..b39ccc4d --- /dev/null +++ b/tests/unit/support/test_service_events.lua @@ -0,0 +1,66 @@ +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' + +local service_events = require 'devicecode.support.service_events' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end + +function tests.test_port_stamps_identity_and_preserves_event_fields() + fibers.run(function () + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local port = service_events.port(tx, { + service_id = 'svc', + source = 'component', + source_id = 'c1', + generation = 7, + }) + + assert_true(port:emit_required({ kind = 'changed', generation = 8, value = 42 })) + local ev = fibers.perform(rx:recv_op()) + assert_eq(ev.kind, 'changed') + assert_eq(ev.service_id, 'svc') + assert_eq(ev.source, 'component') + assert_eq(ev.source_id, 'c1') + assert_eq(ev.generation, 8) + assert_eq(ev.value, 42) + end) +end + +function tests.test_route_events_are_marked_for_mixed_request_queues() + fibers.run(function () + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local port = service_events.port(tx, { service_id = 'svc' }, { mark_route_events = true }) + assert_true(port:emit_required('service_active_snapshot', { snapshot = nil })) + local ev = fibers.perform(rx:recv_op()) + assert_true(service_events.is_route_event(ev)) + assert_eq(ev.kind, 'service_active_snapshot') + end) +end + + +function tests.test_wrap_stamps_identity_over_existing_event_port() + local captured + local target = { + emit_required = function (_, ev, label) + captured = { ev = ev, label = label } + return true, nil + end, + } + local port = service_events.wrap(target, { + service_id = 'svc', + source = 'child', + source_id = 'c2', + }, { label = 'child_report_failed' }) + assert_true(port:emit_required({ kind = 'done' })) + assert_eq(captured.ev.kind, 'done') + assert_eq(captured.ev.service_id, 'svc') + assert_eq(captured.ev.source, 'child') + assert_eq(captured.ev.source_id, 'c2') + assert_eq(captured.label, 'child_report_failed') +end + +return tests diff --git a/tests/unit/system_spec.lua b/tests/unit/system_spec.lua new file mode 100644 index 00000000..42db4fbd --- /dev/null +++ b/tests/unit/system_spec.lua @@ -0,0 +1,62 @@ +-- tests/unit/system_spec.lua + +local fibers = require 'fibers' +local op = require 'fibers.op' + +local system = require 'services.system' + +local T = {} + +local function assert_true(v, msg) + if not v then error(msg or 'assertion failed', 2) end +end + +local function assert_eq(a, b, msg) + if a ~= b then + error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) + end +end + +function T.publish_platform_identity_publishes_state_and_legacy_metrics() + fibers.run(function() + local metrics = {} + local retained = {} + local svc = { + wall = function() return '2026-07-06 12:00:00' end, + _retain = function(_, topic, payload) + retained[table.concat(topic, '/')] = payload + end, + obs_metric = function(_, key, payload) + metrics[key] = payload + end, + obs_log = function() end, + } + local cap = { + call_control_op = function(_, method, opts) + assert_eq(method, 'get') + assert_eq(opts.field, 'identity') + return op.always({ + ok = true, + reason = { + hw_revision = 'bigbox-v1-cm-2', + fw_version = 'bigbox-v1-cm-2-v0.11.0', + serial = 'BB7YJWANBEA6GE', + board_revision = '0xC04180', + }, + }) + end, + } + + assert_true(system._test.publish_platform_identity_metrics(svc, cap, 0.1)) + assert_eq(metrics.hw_id.value, 'bigbox-v1-cm-2') + assert_eq(metrics.fw_id.value, 'bigbox-v1-cm-2-v0.11.0') + assert_eq(metrics.serial.value, 'BB7YJWANBEA6GE') + assert_eq(metrics.board_revision.value, '0xC04180') + assert_eq(metrics.hw_id.namespace[1], 'system') + assert_eq(metrics.hw_id.namespace[2], 'hw_id') + assert_eq(retained['state/system/identity'].schema, 'devicecode.system.identity/1') + assert_eq(retained['state/system/identity'].hw_revision, 'bigbox-v1-cm-2') + end) +end + +return T diff --git a/tests/unit/ui/test_architecture.lua b/tests/unit/ui/test_architecture.lua new file mode 100644 index 00000000..0dec5805 --- /dev/null +++ b/tests/unit/ui/test_architecture.lua @@ -0,0 +1,287 @@ +-- tests/unit/ui/test_architecture.lua + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end + +local function read_file(path) + local f = assert(io.open(path, 'r')) + local s = f:read('*a') + f:close() + return s +end + +local function scan_files(root) + local p = io.popen(('find %s -type f -name "*.lua"'):format(root)) + local out = {} + for line in p:lines() do out[#out + 1] = line end + p:close() + return out +end + +function tests.test_ui_service_code_does_not_use_infrastructure_waits() + for _, path in ipairs(scan_files('../src/services/ui')) do + local s = read_file(path) + if s:find('perform_raw', 1, true) then fail(path .. ' uses perform_raw') end + if s:find('join_op', 1, true) then fail(path .. ' uses join_op') end + if s:find('close_op', 1, true) then fail(path .. ' uses close_op') end + end +end + +function tests.test_ui_no_longer_owns_http_transport_backend() + local p = io.popen('find ../src/services/ui -type f -name "*.lua"') + for path in p:lines() do + local s = read_file(path) + if s:find("services.ui.transport", 1, true) then fail(path .. ' still depends on UI transport') end + if s:find("services.http.transport", 1, true) then fail(path .. ' reaches into HTTP transport internals') end + if s:find("require 'cqueues", 1, true) or s:find('require "cqueues', 1, true) then fail(path .. ' requires cqueues') end + if s:find("require 'http%.", 1) or s:find('require "http%.', 1) then fail(path .. ' requires raw lua-http') end + end + p:close() +end + +function tests.test_ui_public_entry_has_no_websocket_export_for_first_cut() + local s = read_file('../src/services/ui.lua') + if s:find('services.ui.transport', 1, true) then fail('ui.lua exports UI transport') end + if s:find('services.ui.ws', 1, true) then fail('ui.lua exports WebSocket modules in first cut') end + if not s:find("services.ui.http.sse", 1, true) then fail('ui.lua does not export SSE helper') end +end + +function tests.test_http_listener_consumes_http_sdk_not_transport() + local s = read_file('../src/services/ui/http/listener.lua') + if not s:find("services.http'.sdk", 1, true) and not s:find('services.http".sdk', 1, true) then + fail('http listener does not consume services.http.sdk') + end + if s:find('run_pump', 1, true) then fail('http listener still owns a transport pump') end + if s:find('cqueues', 1, true) or s:find('lua_http', 1, true) then fail('http listener mentions backend internals') end + if not s:find('obtain_listener_op', 1, true) then fail('http listener lacks cap/http listen boundary') end + if not s:find('http_request_rejected', 1, true) then fail('http listener lacks explicit overload rejection path') end +end + + +function tests.test_ui_listener_lifecycle_uses_service_event_ports() + local service = read_file('../src/services/ui/service.lua') + if not service:find('devicecode.support.service_events', 1, true) then fail('ui service does not use service event helper') end + if not service:find('events_port = service_events.port', 1, true) then fail('ui listener is not wired through an event port') end + if not service:find('listener_id', 1, true) or not service:find('listener_generation', 1, true) then + fail('ui listener events do not carry listener identity/generation') + end + + local listener = read_file('../src/services/ui/http/listener.lua') + if not listener:find('opts.events_port', 1, true) then fail('ui http listener does not prefer events_port') end +end + +function tests.test_ui_service_starts_listener_from_cfg_ui_not_main_options() + local s = read_file('../src/services/ui/service.lua') + if s:find('params%.http[^_%w]') then fail('ui service still reads HTTP config from start params') end + if s:find('params.verify_login', 1, true) then fail('ui service still reads verifier from start params') end + if not s:find("{ 'cfg', 'ui' }", 1, true) then fail('ui service does not watch cfg/ui') end + if not s:find('cap_id = h.cap_id', 1, true) then fail('ui service does not pass configured HTTP capability id') end + if s:find('start_transport_pump', 1, true) then fail('ui service still starts transport pump') end +end + +function tests.test_ui_and_http_do_not_consume_main_legacy_injection_fields() + for _, root in ipairs({ '../src/services/ui', '../src/services/http' }) do + for _, path in ipairs(scan_files(root)) do + local s = read_file(path) + for _, needle in ipairs({ 'run_http', 'verify_login', 'from_legacy_params', 'params.http_listen', 'opts.http_listen' }) do + if s:find(needle, 1, true) then fail(path .. ' consumes legacy main field ' .. needle) end + end + end + end +end + +function tests.test_read_model_store_has_no_queue_or_mailbox_fanout() + local s = read_file('../src/services/ui/read_model_store.lua') + if s:find("devicecode.support.queue", 1, true) then fail('read_model_store depends on queue') end + if s:find("fibers.mailbox", 1, true) then fail('read_model_store depends on mailbox') end + if s:find("fibers.perform", 1, true) then fail('read_model_store performs Ops') end + if s:find("perform_raw", 1, true) then fail('read_model_store uses perform_raw') end +end + +function tests.test_read_model_watches_owns_watch_fanout_boundary() + local s = read_file('../src/services/ui/read_model_watches.lua') + if not s:find("devicecode.support.queue", 1, true) then fail('read_model_watches does not own queue fanout') end + if not s:find("fibers.mailbox", 1, true) then fail('read_model_watches does not own bounded watch queues') end + if not s:find('watch_open', 1, true) then fail('read_model_watches does not expose watch_open') end +end + +function tests.test_ui_service_wires_same_store_into_read_model_component() + local s = read_file('../src/services/ui/service.lua') + if not s:find('read_model_opts.model = model', 1, true) then + fail('ui service does not pass service model into read_model.start') + end + if not s:find('read_model_opts.watch_owner = watch_owner', 1, true) then + fail('ui service does not pass watch owner into read_model.start') + end +end + +function tests.test_ui_service_records_component_outcomes_before_policy() + local s = read_file('../src/services/ui/service.lua') + if not s:find('record_component_done', 1, true) then fail('ui service does not record component outcomes') end + if not s:find('classify_service_component_done', 1, true) then fail('ui service component completion does not use supervision classifier') end +end + +function tests.test_upload_accepts_only_read_chunk_op_body_boundary() + local upload = read_file('../src/services/ui/update/upload.lua') + if upload:find('read_some_op', 1, true) or upload:find('read_op', 1, true) then + fail('upload accepts non read_chunk_op body compatibility path') + end + if not upload:find('request body has no read_chunk_op', 1, true) then fail('upload does not require read_chunk_op') end +end + + +function tests.test_ui_command_forwarding_is_op_first() + local request = read_file('../src/services/ui/http/request.lua') + if not request:find('user_operation.run_op', 1, true) then fail('command forwarding does not use user_operation.run_op') end + if not request:find('conn:call_op', 1, true) then fail('command forwarding does not use conn:call_op') end + if request:find('conn:call(', 1, true) then fail('command forwarding still hides blocking conn:call') end + + local user_operation = read_file('../src/services/ui/user_operation.lua') + if not user_operation:find('spec.run_op', 1, true) then fail('user_operation does not accept op-first specs') end +end + +function tests.test_ui_upload_and_user_operation_have_timeout_races() + local upload = read_file('../src/services/ui/update/upload.lua') + if not upload:find('upload_timeout', 1, true) then fail('upload timeout option missing') end + if not upload:find('boolean_choice', 1, true) then fail('upload timeout is not an explicit race') end + + local user_operation = read_file('../src/services/ui/user_operation.lua') + if not user_operation:find('boolean_choice', 1, true) then fail('user_operation timeout is not an explicit race') end + if user_operation:find("require 'fibers.mailbox'", 1, true) then fail('user_operation still uses mailbox timeout worker') end +end + +function tests.test_http_response_writes_are_op_only() + local response = read_file('../src/services/ui/http/response.lua') + if response:find('function Response:reply%(') then fail('response exposes non-op reply') end + if response:find('function Response:reply_json%(') then fail('response exposes non-op reply_json') end + if response:find('function Response:reply_error%(') then fail('response exposes non-op reply_error') end + if not response:find('function Response:abandon_now%(') then fail('response lacks immediate abandon_now') end + if not response:find('function Response:terminate%(') then fail('response lacks finaliser-safe terminate') end + if not response:find('write_headers_op', 1, true) then fail('response does not use write_headers_op') end + if not response:find('write_chunk_op', 1, true) then fail('response does not use write_chunk_op') end + + for _, path in ipairs(scan_files('../src/services/ui')) do + local src = read_file(path) + if src:find('owner:reply%(') then fail(path .. ' uses non-op response reply') end + if src:find('owner:reply_json%(') then fail(path .. ' uses non-op response reply_json') end + if src:find('owner:reply_error%(') then fail(path .. ' uses non-op response reply_error') end + end +end + +function tests.test_sse_is_request_owned_streaming_http_not_transport_code() + local s = read_file('../src/services/ui/http/sse.lua') + if not s:find('watch_open', 1, true) then fail('SSE does not open a read-model watch') end + if not s:find('text/event%-stream') then fail('SSE does not emit event-stream headers') end + if s:find('cqueues', 1, true) or s:find('lua_http', 1, true) then fail('SSE mentions backend internals') end +end + +function tests.test_artifact_ingest_boundary_has_no_raw_blocking_fallbacks() + local ingest = read_file('../src/services/ui/update/artifact_ingest.lua') + for _, raw in ipairs({ 'open_ingest(', ':append(', ':write(', ':commit(', ':abort(', ':close(' }) do + if ingest:find(raw, 1, true) then fail('artifact_ingest contains raw blocking fallback: ' .. raw) end + end + if not ingest:find('open_ingest_op', 1, true) then fail('artifact_ingest lacks open_ingest_op boundary') end + if not ingest:find('append_chunk_op', 1, true) then fail('artifact_ingest lacks append_chunk_op boundary') end + if not ingest:find('commit_op', 1, true) then fail('artifact_ingest lacks commit_op boundary') end + if not ingest:find('abort_now', 1, true) then fail('artifact_ingest lacks abort_now boundary') end +end + +function tests.test_read_model_store_declares_first_token_replay_index() + local store = read_file('../src/services/ui/read_model_store.lua') + if not store:find('_index_first', 1, true) then fail('read_model_store lacks first-token index') end + if not store:find('_candidate_keys', 1, true) then fail('read_model_store lacks indexed candidate selection') end +end + +function tests.test_ui_service_observes_session_pulse_not_event_sink() + local service = read_file('../src/services/ui/service.lua') + local store = read_file('../src/services/ui/sessions.lua') + if service:find('set_event_sink', 1, true) or store:find('set_event_sink', 1, true) then fail('ui still exposes session event sink') end + if store:find('event_sink', 1, true) or store:find('on_event', 1, true) then fail('session store still accepts event-sink compatibility options') end + if not service:find('sessions:changed_op', 1, true) and not service:find('state.sessions:changed_op', 1, true) then + fail('ui service does not observe session changed_op') + end +end + +function tests.test_read_model_component_uses_scope_aware_never_wait() + local service = read_file('../src/services/ui/service.lua') + if service:find('component_scope:not_ok_op()', 1, true) then fail('read-model component waits on its own not_ok_op') end + if not service:find('fibers.perform(fibers.never())', 1, true) then fail('read-model component does not use never() as long-lived wait') end +end + +function tests.test_response_headers_state_is_committed_after_transport_write() + local response = read_file('../src/services/ui/http/response.lua') + local guard_pos = response:find('function Response:write_headers_op', 1, true) + local token_pos = response:find('acquire_start_token', guard_pos, true) + local state_pos = response:find("self._state = opts.end_stream and 'ended' or 'headers_sent'", guard_pos, true) + if not token_pos then fail('write_headers_op does not use start token') end + if not state_pos then fail('write_headers_op does not commit headers_sent after write') end + if state_pos < token_pos then fail('headers state appears to be committed before start token/write path') end +end + +function tests.test_response_headers_op_does_not_mutate_in_guard_path() + local response = read_file('../src/services/ui/http/response.lua') + local start_pos = response:find('function Response:write_headers_op', 1, true) + local end_pos = response:find('function Response:write_chunk_op', start_pos, true) + local body = response:sub(start_pos, end_pos - 1) + if body:find('fibers.guard(function', 1, true) then fail('write_headers_op still uses a mutating guard path') end + if not body:find('acquire_start_token_op', 1, true) then fail('write_headers_op does not use the choice-safe start token op') end +end + +function tests.test_ui_service_does_not_retain_ui_projection_state() + local service = read_file('../src/services/ui/service.lua') + for _, needle in ipairs({ 'topics.summary()', 'topics.read_model_status()', 'topics.session_count()', "'state', 'ui'" }) do + if service:find(needle, 1, true) then fail('ui service retains or unretains UI projection state: ' .. needle) end + end + if not service:find('self%-ingesting projection loop') then fail('ui service does not document projection feedback boundary') end +end + +function tests.test_read_model_declares_self_ingestion_exclusions() + local topics = read_file('../src/services/ui/topics.lua') + local read_model = read_file('../src/services/ui/read_model.lua') + if not topics:find('default_excluded_retained_patterns', 1, true) then fail('ui topics do not declare retained-input exclusions') end + if not topics:find("t('state', 'ui', '#')", 1, true) then fail('state/ui/# is not excluded') end + if not topics:find("t('obs', 'v1', 'ui', '#')", 1, true) then fail('obs/v1/ui/# is not excluded') end + if not read_model:find('should_ingest_event', 1, true) then fail('read model has no retained-feed exclusion check') end +end + + +function tests.test_reusable_ui_clients_have_no_hidden_bus_timeouts() + local client = read_file('../src/services/ui/update/client.lua') + if not client:find('out.timeout = false', 1, true) then + fail('ui update client should default bus calls to timeout=false') + end + if client:find('default_timeout or 10.0', 1, true) or client:find('opts.timeout or 10.0', 1, true) then + fail('ui update client still has a hidden default timeout') + end + + local ingest = read_file('../src/services/ui/update/artifact_ingest.lua') + if not ingest:find('out.timeout = false', 1, true) then + fail('artifact ingest bus client should default bus calls to timeout=false') + end + if ingest:find('opts.timeout or 10.0', 1, true) then + fail('artifact ingest bus client still has a hidden timeout') + end + + local req = read_file('../src/services/ui/http/request.lua') + if not req:find('conn:call_op(route.topic, payload, { timeout = false })', 1, true) then + fail('UI command bridge should leave timeout policy to user_operation.run_op') + end + + local upload = read_file('../src/services/ui/update/upload.lua') + if upload:find('remaining_timeout(opts, deadline)', 1, true) then + fail('upload should not convert its outer deadline into hidden upstream bus timeouts') + end +end + + +function tests.test_ui_listener_has_no_separate_pending_generation_field() + local f = assert(io.open('../src/services/ui/service.lua', 'r')) + local src = f:read('*a'); f:close() + if src:find('pending_listener_generation', 1, true) then + error('ui listener should carry pending listener config only; pending_listener_generation is redundant', 2) + end + assert(src:find('pending_listener_cfg', 1, true), 'ui should still keep pending listener config') +end + +return tests diff --git a/tests/unit/ui/test_config.lua b/tests/unit/ui/test_config.lua new file mode 100644 index 00000000..4c096100 --- /dev/null +++ b/tests/unit/ui/test_config.lua @@ -0,0 +1,144 @@ +local config = require 'services.ui.config' + +local M = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +local function update_policy(overrides) + local upload = { + enabled = true, + max_bytes = 1024 * 1024, + require_auth = false, + component = 'mcu', + create_job = true, + start_job = true, + } + local commit = { require_auth = false } + for k, v in pairs((overrides and overrides.upload) or {}) do upload[k] = v end + for k, v in pairs((overrides and overrides.commit) or {}) do commit[k] = v end + return { upload = upload, commit = commit } +end + +function M.test_normalise_supplies_http_static_sse_defaults_and_accepts_explicit_update_policy() + local cfg = ok(config.normalise({ schema = config.SCHEMA, updates = update_policy() })) + eq(cfg.enabled, true) + eq(cfg.http.enabled, true) + eq(cfg.http.cap_id, 'main') + eq(cfg.http.port, 80) + eq(cfg.static.root, 'services/ui/www') + eq(cfg.sse.enabled, true) + eq(cfg.sse.queue_len, 256) + eq(cfg.sse.replay, false) + eq(cfg.updates.upload.enabled, true) + eq(cfg.updates.upload.max_bytes, 1024 * 1024) + eq(cfg.updates.upload.require_auth, false) + eq(cfg.updates.upload.component, 'mcu') + eq(cfg.updates.upload.create_job, true) + eq(cfg.updates.upload.start_job, true) + eq(cfg.updates.commit.require_auth, false) + eq(config.DEFAULTS.updates, nil) + eq(cfg.observability.status_interval_s, 30) + eq(cfg.observability.coalesce_status_s, 0.05) +end + +function M.test_sse_replay_can_be_enabled_explicitly() + local cfg = ok(config.normalise({ + schema = config.SCHEMA, + updates = update_policy(), + sse = { replay = true }, + })) + eq(cfg.sse.replay, true) +end + + +function M.test_observability_status_interval_is_configurable() + local cfg = ok(config.normalise({ + schema = config.SCHEMA, + updates = update_policy(), + observability = { status_interval_s = 15, coalesce_status_s = 0.1 }, + })) + eq(cfg.observability.status_interval_s, 15) + eq(cfg.observability.coalesce_status_s, 0.1) + + cfg = ok(config.normalise({ + schema = config.SCHEMA, + updates = update_policy(), + observability = { status_interval_s = false }, + })) + eq(cfg.observability.status_interval_s, false) + + local bad, err = config.normalise({ + schema = config.SCHEMA, + updates = update_policy(), + observability = { every_status = true }, + }) + eq(bad, nil) + ok(tostring(err):find('observability has unknown field', 1, true)) +end + +function M.test_update_upload_and_commit_can_require_authentication() + local cfg = ok(config.normalise({ + schema = config.SCHEMA, + updates = update_policy({ upload = { require_auth = true }, commit = { require_auth = true } }), + })) + eq(cfg.updates.upload.require_auth, true) + eq(cfg.updates.commit.require_auth, true) +end + +function M.test_update_upload_policy_requires_component_create_and_start_fields() + local cfg, err = config.normalise({ + schema = config.SCHEMA, + updates = { upload = { enabled = true, max_bytes = 1024, require_auth = false }, commit = { require_auth = false } }, + }) + eq(cfg, nil) + ok(tostring(err):find('updates.upload.component is required', 1, true)) +end + +function M.test_update_upload_start_requires_create() + local cfg, err = config.normalise({ + schema = config.SCHEMA, + updates = update_policy({ upload = { create_job = false, start_job = true } }), + }) + eq(cfg, nil) + ok(tostring(err):find('start_job requires', 1, true)) +end + +function M.test_can_disable_http_listener_without_disabling_service() + local cfg = ok(config.normalise({ schema = config.SCHEMA, http = { enabled = false }, updates = update_policy() })) + eq(cfg.enabled, true) + eq(cfg.http.enabled, false) +end + +function M.test_config_is_explicit_not_derived_from_main_options() + eq(config.from_legacy_params, nil) + local cfg = ok(config.normalise({ + schema = config.SCHEMA, + http = { host = '127.0.0.1', port = 9000, cap_id = 'main' }, + static = { root = '/srv/ui' }, + updates = update_policy(), + })) + eq(cfg.http.host, '127.0.0.1') + eq(cfg.http.port, 9000) + eq(cfg.static.root, '/srv/ui') +end + +function M.test_rejects_unknown_fields() + local cfg, err = config.normalise({ schema = config.SCHEMA, transport = {}, updates = update_policy() }) + eq(cfg, nil) + ok(tostring(err):find('unknown field', 1, true)) +end + +function M.test_rejects_legacy_uploads_field() + local cfg, err = config.normalise({ + schema = config.SCHEMA, + uploads = { require_auth = false }, + updates = update_policy(), + }) + eq(cfg, nil) + ok(tostring(err):find('unknown field', 1, true)) +end + +return M diff --git a/tests/unit/ui/test_http_listener.lua b/tests/unit/ui/test_http_listener.lua new file mode 100644 index 00000000..79d32671 --- /dev/null +++ b/tests/unit/ui/test_http_listener.lua @@ -0,0 +1,252 @@ +-- tests/unit/ui/test_http_listener.lua + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local pulse = require 'fibers.pulse' +local run_fibers = require 'tests.support.run_fibers' +local listener_mod = require 'services.ui.http.listener' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end + +local FakeContext = {} +FakeContext.__index = FakeContext + +local function make_ctx(spec) + spec = spec or {} + return setmetatable({ + _id = spec.id, + _method = spec.method or 'GET', + _path = spec.path or '/', + terminated = false, + terminate_reason = nil, + }, FakeContext) +end + +function FakeContext:id() return self._id end +function FakeContext:method() return self._method end +function FakeContext:path() return self._path end +function FakeContext:terminate(reason) + self.terminated = true + self.terminate_reason = reason + return true, nil +end +local FakeListener = {} +FakeListener.__index = FakeListener + +local function fake_listener() + local tx, rx = mailbox.new(16, { full = 'reject_newest' }) + local accept_started = pulse.new() + return setmetatable({ + _tx = tx, + _rx = rx, + _accept_started = accept_started, + _accept_seen = accept_started:version(), + _terminated = false, + _reason = nil, + }, FakeListener) +end + +function FakeListener:accept_now(ctx) + return self._tx:send(ctx) +end + +function FakeListener:accept_op() + self._accept_started:signal() + return self._rx:recv_op():wrap(function (ctx) + if ctx == nil then return nil, tostring(self._rx:why() or 'closed') end + return ctx, nil + end) +end + +function FakeListener:terminate(reason) + if self._terminated then return true, nil end + self._terminated = true + self._reason = reason or 'closed' + self._tx:close(self._reason) + return true, nil +end +function FakeListener:terminated() return self._terminated end +function FakeListener:accept_started_op() return self._accept_started:changed_op(self._accept_seen) end + + +local function event_port(tx) + return { + emit_required = function (_, ev, label) + local ok, err = tx:send(ev) + if ok ~= true then return nil, err or label end + return true, nil + end, + } +end + +local function recv_kind(rx, kind) + while true do + local ev = fibers.perform(rx:recv_op()) + assert_not_nil(ev, 'event stream closed before '..kind) + if ev.kind == kind then return ev end + end +end + +function tests.test_accept_transfers_context_to_request_scope_and_reports_started_done() + run_fibers.run(function (scope) + local done_tx, done_rx = mailbox.new(16, { full = 'reject_newest' }) + local listener = fake_listener() + local seen_in_request = false + + local ok, err = scope:spawn(function (s) + listener_mod.run(s, { + listener = listener, + events_port = event_port(done_tx), + run_request = function (_request_scope, ctx) + seen_in_request = true + assert_eq(ctx:method(), 'GET') + assert_eq(ctx:path(), '/status') + return { status = 'ok' } + end, + }) + end) + assert_true(ok, err) + + local raw = make_ctx({ id = 'req-1', method = 'GET', path = '/status' }) + assert_true(listener:accept_now(raw)) + + local started = recv_kind(done_rx, 'http_request_started') + assert_eq(started.request_id, 'req-1') + assert_eq(started.active_requests, 1) + + local done = recv_kind(done_rx, 'http_request_done') + assert_eq(done.request_id, 'req-1') + assert_eq(done.status, 'ok') + assert_eq(done.active_requests, 0) + assert_true(seen_in_request) + assert_true(raw.terminated) + end) +end + +function tests.test_listener_can_obtain_handle_from_http_capability_sdk() + run_fibers.run(function (scope) + local done_tx, done_rx = mailbox.new(16, { full = 'reject_newest' }) + local listener = fake_listener() + local call_topic + local call_payload + local conn = { + call_op = function(_, topic, payload) + call_topic = topic + call_payload = payload + return fibers.always({ listener = listener }, nil) + end, + } + + local ok, err = scope:spawn(function (s) + listener_mod.run(s, { + conn = conn, + listen = { host = '127.0.0.1', port = 8080 }, + events_port = event_port(done_tx), + run_request = function () return { status = 'ok' } end, + }) + end) + assert_true(ok, err) + + local raw = make_ctx({ id = 'req-cap', method = 'GET', path = '/' }) + assert_true(listener:accept_now(raw)) + local started = recv_kind(done_rx, 'http_request_started') + assert_eq(started.request_id, 'req-cap') + assert_not_nil(call_topic) + assert_eq(call_topic[1], 'cap') + assert_eq(call_topic[2], 'http') + assert_eq(call_topic[3], 'main') + assert_eq(call_topic[4], 'rpc') + assert_eq(call_topic[5], 'listen') + assert_eq(call_payload.host, '127.0.0.1') + end) +end + +function tests.test_overloaded_listener_rejects_without_request_scope() + run_fibers.run(function (scope) + local done_tx, done_rx = mailbox.new(8, { full = 'reject_newest' }) + local listener = fake_listener() + local ran_request = false + + local ok, err = scope:spawn(function (s) + listener_mod.run(s, { + listener = listener, + events_port = event_port(done_tx), + max_active_requests = 0, + overload_reason = 'too_many_requests', + run_request = function () + ran_request = true + return { status = 'unexpected' } + end, + }) + end) + assert_true(ok, err) + + local raw = make_ctx({ id = 'req-2', method = 'GET', path = '/' }) + assert_true(listener:accept_now(raw)) + + local rejected = recv_kind(done_rx, 'http_request_rejected') + assert_eq(rejected.request_id, 'req-2') + assert_eq(rejected.reason, 'too_many_requests') + assert_eq(rejected.active_requests, 0) + assert_eq(rejected.max_active_requests, 0) + assert_true(rejected.cleanup_ok) + assert_true(raw.terminated) + assert_eq(raw.terminate_reason, 'too_many_requests') + assert_eq(ran_request, false) + end) +end + +function tests.test_request_scope_finaliser_owns_context_on_failure() + run_fibers.run(function (scope) + local done_tx, done_rx = mailbox.new(8, { full = 'reject_newest' }) + local listener = fake_listener() + + local ok, err = scope:spawn(function (s) + listener_mod.run(s, { + listener = listener, + events_port = event_port(done_tx), + run_request = function () error('boom', 0) end, + }) + end) + assert_true(ok, err) + + local raw = make_ctx({ id = 'req-3', method = 'GET', path = '/boom' }) + assert_true(listener:accept_now(raw)) + + local done = recv_kind(done_rx, 'http_request_done') + assert_eq(done.request_id, 'req-3') + assert_eq(done.status, 'failed') + assert_eq(done.primary, 'boom') + assert_true(raw.terminated) + assert_eq(raw.terminate_reason, 'boom') + end) +end + +function tests.test_listener_finaliser_terminates_http_listener_handle() + run_fibers.run(function (scope) + local listener = fake_listener() + local owner, cerr = scope:child() + assert_not_nil(owner, cerr) + local ok, err = owner:spawn(function (s) + listener_mod.run(s, { + listener = listener, + run_request = function () return { status = 'ok' } end, + }) + end) + assert_true(ok, err) + + local _, wait_err = fibers.perform(listener:accept_started_op()) + assert_eq(wait_err, nil) + + owner:cancel('test_cancel') + fibers.perform(owner:join_op()) + assert_true(listener:terminated()) + end) +end + +return tests diff --git a/tests/unit/ui/test_http_request.lua b/tests/unit/ui/test_http_request.lua new file mode 100644 index 00000000..3d78322d --- /dev/null +++ b/tests/unit/ui/test_http_request.lua @@ -0,0 +1,387 @@ +-- tests/unit/ui/test_http_request.lua + +local run_fibers = require 'tests.support.run_fibers' +local fibers = require 'fibers' +local channel = require 'fibers.channel' +local busmod = require 'bus' +local request = require 'services.ui.http.request' +local read_model = require 'services.ui.read_model' +local sse = require 'services.ui.http.sse' + +local ok_cjson, cjson = pcall(require, 'cjson.safe') +if not ok_cjson then cjson = require 'cjson' end + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) + if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end +end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end +local function assert_true(v, msg) + if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end +end + +local function fake_ctx(method, path) + return { + method = method, + path = path, + replies = {}, + _current = nil, + write_headers_op = function(self, status, headers, opts) + return fibers.always(function () + self._current = { status = status, body = '', headers = headers, end_stream = opts and opts.end_stream } + if opts and opts.end_stream then + self.replies[#self.replies + 1] = self._current + self._current = nil + end + return true, nil + end):wrap(function (th) + return th() + end) + end, + write_chunk_op = function(self, chunk, opts) + return fibers.always(function () + assert(self._current, 'no response headers') + self._current.body = self._current.body .. tostring(chunk or '') + if opts and opts.end_stream then + self.replies[#self.replies + 1] = self._current + self._current = nil + end + return true, nil + end):wrap(function (th) + return th() + end) + end, + terminate = function(self, reason) + self.abandoned = reason + return true + end, + } +end + +function tests.test_http_read_request_uses_model_and_replies_once() + run_fibers.run(function (scope) + local model = read_model.new() + model:set({ 'svc', 'ui', 'status' }, { state = 'running' }) + local ctx = fake_ctx('GET', '/api/state/svc/ui/status') + local result = request.run(scope, ctx, { model = model }) + assert_eq(result.status, 'ok') + assert_eq(#ctx.replies, 1) + assert_eq(ctx.replies[1].status, 200) + assert_not_nil(ctx.replies[1].body) + end) +end + + +function tests.test_http_response_writer_may_yield_inside_request_scope_without_blocking_peers() + run_fibers.run(function (scope) + local model = read_model.new() + model:set({ 'svc', 'ui', 'status' }, { state = 'running' }) + + local started_ch = channel.new() + local resume_ch = channel.new() + local events = {} + local ctx = fake_ctx('GET', '/api/state/svc/ui/status') + ctx.write_headers_op = function(self, status, headers, opts) + events[#events + 1] = 'reply-start' + started_ch:put(true) + return resume_ch:get_op():wrap(function (token) + events[#events + 1] = 'reply-finish:' .. tostring(token) + self._current = { status = status, body = '', headers = headers, end_stream = opts and opts.end_stream } + return true, nil + end) + end + + fibers.spawn(function () + started_ch:get() + events[#events + 1] = 'peer-fiber-ran' + resume_ch:put('resumed') + end) + + local result = request.run(scope, ctx, { model = model }) + assert_eq(result.status, 'ok') + assert_eq(#ctx.replies, 1) + assert_eq(ctx.replies[1].status, 200) + assert_eq(events[1], 'reply-start') + assert_eq(events[2], 'peer-fiber-ran') + assert_eq(events[3], 'reply-finish:resumed') + end) +end + +function tests.test_sse_route_opens_without_replaying_large_bootstrap_state() + run_fibers.run(function (scope) + local model = read_model.new() + for i = 1, 40 do + model:set({ 'state', 'bulk', tostring(i) }, { n = i }) + end + local watch_owner = read_model.new_watches(model) + local headers_ch = channel.new(1) + local chunk_ch = channel.new(1) + local ctx = fake_ctx('GET', '/events') + + ctx.write_headers_op = function(self, status, headers, opts) + return fibers.always(function () + self._current = { status = status, body = '', headers = headers, end_stream = opts and opts.end_stream } + headers_ch:put({ status = status, headers = headers }) + return true, nil + end):wrap(function (th) + return th() + end) + end + ctx.write_chunk_op = function(self, chunk, opts) + return fibers.always(function () + assert(self._current, 'no response headers') + self._current.body = self._current.body .. tostring(chunk or '') + chunk_ch:put(tostring(chunk or '')) + if opts and opts.end_stream then + self.replies[#self.replies + 1] = self._current + self._current = nil + end + return true, nil + end):wrap(function (th) + return th() + end) + end + + local ok_spawn, spawn_err = scope:spawn(function () + request.run(scope, ctx, { + watch_owner = watch_owner, + encode_json = function (v) return assert(cjson.encode(v)) end, + }) + end) + assert_true(ok_spawn, spawn_err) + + local headers = headers_ch:get() + assert_not_nil(headers) + assert_eq(headers.status, 200) + assert_eq(headers.headers['content-type'], 'text/event-stream') + assert_eq(watch_owner:watch_count(), #sse.default_patterns()) + + watch_owner:set({ 'state', 'system', 'stats' }, { + cpu = { utilisation = 12.5 }, + }) + local metric_chunk = chunk_ch:get() + assert_not_nil(metric_chunk) + assert_true(metric_chunk:find('event: set', 1, true) ~= nil, metric_chunk) + assert_true(metric_chunk:find('state/system/stats', 1, true) ~= nil, metric_chunk) + assert_true(metric_chunk:find('12.5', 1, true) ~= nil, metric_chunk) + + watch_owner:set({ 'raw', 'secret' }, { leaked = true }) + watch_owner:set({ 'state', 'device', 'component', 'switch-main' }, { + available = true, + observed = { wired = { raw = { leaked = true } } }, + raw = { leaked = true }, + }) + local chunk = chunk_ch:get() + assert_not_nil(chunk) + assert_true(chunk:find('event: set', 1, true) ~= nil, chunk) + assert_true(chunk:find('state/device/component/switch%-main') ~= nil, chunk) + assert_true(chunk:find('raw/secret', 1, true) == nil, chunk) + assert_true(chunk:find('leaked', 1, true) == nil, chunk) + assert_true(chunk:find('"observed"', 1, true) == nil, chunk) + end) +end + +function tests.test_sse_route_opens_only_local_ui_prefix_watches() + run_fibers.run(function (scope) + local headers_ch = channel.new(1) + local opened = {} + local ctx = fake_ctx('GET', '/events') + + ctx.write_headers_op = function(self, status, headers, opts) + return fibers.always(function () + self._current = { status = status, body = '', headers = headers, end_stream = opts and opts.end_stream } + headers_ch:put({ status = status, headers = headers }) + return true, nil + end):wrap(function (th) + return th() + end) + end + + local watch_owner = { + watch_open = function(_, pattern, opts) + local watch = { + pattern = pattern, + opts = opts, + terminated = nil, + recv_op = function () + return fibers.never() + end, + terminate = function(self, reason) + self.terminated = reason or true + return true + end, + } + opened[#opened + 1] = { + pattern = pattern, + opts = opts, + } + return watch, nil + end, + } + + local ok_spawn, spawn_err = scope:spawn(function () + request.run(scope, ctx, { + watch_owner = watch_owner, + encode_json = function (v) return assert(cjson.encode(v)) end, + }) + end) + assert_true(ok_spawn, spawn_err) + + local headers = headers_ch:get() + assert_not_nil(headers) + assert_eq(headers.status, 200) + assert_eq(headers.headers['content-type'], 'text/event-stream') + + local defaults = sse.default_patterns() + assert_eq(#opened, #defaults) + local seen = {} + for _, rec in ipairs(opened) do + local key = table.concat(rec.pattern, '/') + if key == '#' then fail('SSE opened root # watch') end + seen[key] = true + assert_eq(rec.opts.replay, false) + assert_eq(rec.opts.queue_len, 32) + assert_eq(rec.opts.full, 'drop_oldest') + end + assert_true(seen['state/device/#'] == true, 'state/device watch missing') + assert_true(seen['state/net/#'] == true, 'state/net watch missing') + assert_true(seen['obs/v1/system/metric/#'] == nil, 'system metric watch should not be opened') + assert_true(seen['state/system/#'] == true, 'state/system watch missing') + end) +end + + +function tests.test_http_command_route_parses_real_json_body_and_calls_bus() + run_fibers.run(function (scope) + local bus = busmod.new() + local admin = bus:connect({ origin_base = { service = 'ui-command-test' } }) + local received + local ep = assert(admin:bind({ 'cap', 'test', 'main', 'rpc', 'echo' }, { queue_len = 1 })) + scope:finally(function () ep:close(); admin:disconnect() end) + + fibers.spawn(function () + local req = ep:recv() + received = req.payload + req:reply({ echoed = req.payload }) + end) + + local ctx = fake_ctx('POST', '/api/call/cap/test/main/rpc/echo') + ctx.headers = { ['content-type'] = 'application/json', ['x-session-id'] = 'sid-1' } + ctx.read_body_as_string_op = function () + return fibers.always('{"job_id":"job-1","n":7}', nil) + end + + local sessions = { + get = function (_, sid) + if sid == 'sid-1' then return { id = sid, principal = { kind = 'user', id = 'tester' } } end + return nil + end, + } + + local result = request.run(scope, ctx, { + bus = bus, + sessions = sessions, + encode_json = function (v) return assert(cjson.encode(v)) end, + }) + + assert_eq(result.status, 'ok') + assert_not_nil(received) + assert_eq(received.job_id, 'job-1') + assert_eq(received.n, 7) + assert_eq(#ctx.replies, 1) + assert_eq(ctx.replies[1].status, 200) + end) +end + + +function tests.test_update_commit_route_calls_update_manager_without_session() + run_fibers.run(function (scope) + local bus = busmod.new() + local admin = bus:connect({ origin_base = { service = 'ui-update-commit-test' } }) + local caller = bus:connect({ origin_base = { service = 'ui' } }) + local received + local ep = assert(admin:bind({ 'cap', 'update-manager', 'main', 'rpc', 'commit-job' }, { queue_len = 1 })) + scope:finally(function () ep:close(); caller:disconnect(); admin:disconnect() end) + + fibers.spawn(function () + local req = ep:recv() + received = req.payload + req:reply({ ok = true }) + end) + + local ctx = fake_ctx('POST', '/api/update/commit') + ctx.headers = { ['content-type'] = 'application/json' } + ctx.read_body_as_string_op = function () + return fibers.always('{"job_id":"job-1"}', nil) + end + + local result = request.run(scope, ctx, { + bus = bus, + update = { + conn = caller, + commit_require_auth = false, + connect = function () error('public update commit should borrow supplied service conn') end, + }, + encode_json = function (v) return assert(cjson.encode(v)) end, + }) + + assert_eq(result.status, 'ok') + assert_not_nil(received) + assert_eq(received.job_id, 'job-1') + assert_eq(#ctx.replies, 1) + assert_eq(ctx.replies[1].status, 200) + local decoded = assert(cjson.decode(ctx.replies[1].body), ctx.replies[1].body) + assert_eq(decoded.value.ok, true) + end) +end + +function tests.test_update_commit_route_obeys_auth_policy_when_enabled() + run_fibers.run(function (scope) + local bus = busmod.new() + local ctx = fake_ctx('POST', '/api/update/commit') + ctx.headers = { ['content-type'] = 'application/json' } + ctx.read_body_as_string_op = function () return fibers.always('{"job_id":"job-1"}', nil) end + + local result = request.run(scope, ctx, { + bus = bus, + update = { bus = bus, commit_require_auth = true }, + encode_json = function (v) return assert(cjson.encode(v)) end, + }) + + assert_eq(result.status, 'unauthenticated') + assert_eq(#ctx.replies, 1) + assert_eq(ctx.replies[1].status, 401) + end) +end + +function tests.test_http_command_route_rejects_non_json_body() + run_fibers.run(function (scope) + local bus = busmod.new() + local ctx = fake_ctx('POST', '/api/call/cap/test/main/rpc/echo') + ctx.headers = { ['content-type'] = 'text/plain', ['x-session-id'] = 'sid-1' } + ctx.read_body_as_string_op = function () return fibers.always('not json', nil) end + local sessions = { get = function () return { id = 'sid-1', principal = 'tester' } end } + local result = request.run(scope, ctx, { + bus = bus, + sessions = sessions, + encode_json = function (v) return assert(cjson.encode(v)) end, + }) + assert_eq(result.status, 'bad_request') + assert_eq(#ctx.replies, 1) + assert_eq(ctx.replies[1].status, 415) + end) +end + +function tests.test_unknown_route_replies_not_found_once() + run_fibers.run(function (scope) + local ctx = fake_ctx('GET', '/api/nope') + local result = request.run(scope, ctx, { model = read_model.new() }) + assert_eq(result.status, 'not_found') + assert_eq(#ctx.replies, 1) + assert_eq(ctx.replies[1].status, 404) + end) +end + +return tests diff --git a/tests/unit/ui/test_http_response_stream.lua b/tests/unit/ui/test_http_response_stream.lua new file mode 100644 index 00000000..6fd26207 --- /dev/null +++ b/tests/unit/ui/test_http_response_stream.lua @@ -0,0 +1,164 @@ +-- tests/unit/ui/test_http_response_stream.lua + +local fibers = require 'fibers' +local response = require 'services.ui.http.response' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got '..tostring(v))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end + +local function ctx() + return { + calls = {}, + write_headers_op = function(self, status, headers, opts) + return fibers.always(function() + self.calls[#self.calls + 1] = { kind = 'headers', status = status, headers = headers, end_stream = opts and opts.end_stream } + return true, nil + end):wrap(function(th) return th() end) + end, + write_chunk_op = function(self, chunk, opts) + return fibers.always(function() + self.calls[#self.calls + 1] = { kind = 'chunk', chunk = chunk, end_stream = opts and opts.end_stream } + return true, nil + end):wrap(function(th) return th() end) + end, + terminate = function(self, reason) + self.abandoned = reason + return true, nil + end, + } +end + +function tests.test_stream_headers_chunks_and_end_state() + fibers.run(function() + local raw = ctx() + local owner = response.new(raw) + local ok, err = fibers.perform(owner:write_headers_op(200, { ['content-type'] = 'text/plain' })) + assert_true(ok, err) + ok, err = fibers.perform(owner:write_chunk_op('abc')) + assert_true(ok, err) + ok, err = fibers.perform(owner:end_stream_op()) + assert_true(ok, err) + assert_eq(owner:state(), 'ended') + assert_eq(raw.calls[1].kind, 'headers') + assert_eq(raw.calls[2].chunk, 'abc') + assert_eq(raw.calls[3].end_stream, true) + end) +end + +function tests.test_invalid_response_state_transitions_fail() + fibers.run(function() + local owner = response.new(ctx()) + local ok, err = fibers.perform(owner:write_chunk_op('late')) + assert_nil(ok) + assert_eq(err, 'response stream is not open') + ok, err = fibers.perform(owner:write_headers_op(200, {})) + assert_true(ok, err) + ok, err = fibers.perform(owner:reply_op(200, 'body', {})) + assert_nil(ok) + assert_eq(err, 'response already resolved') + end) +end + +function tests.test_reply_op_uses_streaming_transport() + fibers.run(function() + local raw = ctx() + local owner = response.new(raw) + local ok, err = fibers.perform(owner:reply_op(201, 'hello', { ['content-type'] = 'text/plain' })) + assert_true(ok, err) + assert_eq(owner:state(), 'replied') + assert_eq(#raw.calls, 2) + assert_eq(raw.calls[1].kind, 'headers') + assert_eq(raw.calls[1].status, 201) + assert_eq(raw.calls[2].kind, 'chunk') + assert_eq(raw.calls[2].chunk, 'hello') + assert_eq(raw.calls[2].end_stream, true) + end) +end + +function tests.test_terminate_abandons_without_writing() + local raw = ctx() + local owner = response.new(raw) + local ok, err = owner:terminate('closed') + assert_true(ok, err) + assert_eq(raw.abandoned, 'closed') + assert_eq(#raw.calls, 0) + assert_eq(owner:state(), 'abandoned') +end + +function tests.test_transport_write_failure_abandons_response() + fibers.run(function() + local raw = ctx() + raw.write_chunk_op = function() + return fibers.always(nil, 'chunk_failed') + end + local owner = response.new(raw) + local ok, err = fibers.perform(owner:write_headers_op(200, {})) + assert_true(ok, err) + ok, err = fibers.perform(owner:write_chunk_op('abc')) + assert_nil(ok) + assert_eq(err, 'chunk_failed') + assert_eq(owner:state(), 'abandoned') + end) +end + +function tests.test_cancellation_while_streaming_chunk_abandons_response_now() + fibers.run(function() + local raw = ctx() + local abandoned + raw.terminate = function(_, reason) + abandoned = reason + return true, nil + end + raw.write_chunk_op = function() + return require('fibers.sleep').sleep_op(1):wrap(function () return true, nil end) + end + + local owner = response.new(raw) + local ok, err = fibers.perform(owner:write_headers_op(200, {})) + assert_true(ok, err) + + local completed = fibers.perform(fibers.boolean_choice( + owner:write_chunk_op('slow'), + require('fibers.sleep').sleep_op(0.01) + )) + assert_eq(completed, false) + assert_eq(owner:state(), 'abandoned') + assert_eq(abandoned, 'response_chunk_aborted') + end) +end + + +function tests.test_headers_op_losing_choice_releases_start_token_and_abandons() + fibers.run(function() + local sleep = require 'fibers.sleep' + local raw = ctx() + local abandoned + raw.terminate = function(_, reason) + abandoned = reason + return true, nil + end + raw.write_headers_op = function() + return sleep.sleep_op(1):wrap(function () return true, nil end) + end + + local owner = response.new(raw) + local completed = fibers.perform(fibers.boolean_choice( + owner:write_headers_op(200, {}), + sleep.sleep_op(0.01) + )) + assert_eq(completed, false) + assert_eq(owner:state(), 'abandoned') + assert_eq(abandoned, 'response_headers_aborted') + + local ok, err = fibers.perform(owner:write_headers_op(200, {})) + assert_nil(ok) + assert_eq(err, 'response already started') + end) +end + +return tests diff --git a/tests/unit/ui/test_local_ui.lua b/tests/unit/ui/test_local_ui.lua new file mode 100644 index 00000000..24561761 --- /dev/null +++ b/tests/unit/ui/test_local_ui.lua @@ -0,0 +1,161 @@ +-- tests/unit/ui/test_local_ui.lua + +local routes = require 'services.ui.http.routes' +local sse = require 'services.ui.http.sse' +local read_model = require 'services.ui.read_model' +local local_model = require 'services.ui.local_model' + +local ok_cjson, cjson = pcall(require, 'cjson.safe') +if not ok_cjson then cjson = require 'cjson' end + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function ok(v,msg) if not v then fail(msg or 'expected truthy') end end + +local function ctx(method, path) + return { method = method, path = path } +end + +local function sse_data(frame) + local lines = {} + for line in tostring(frame):gmatch('([^\n]*)\n') do + local data = line:match('^data:%s?(.*)$') + if data ~= nil then lines[#lines + 1] = data end + end + return table.concat(lines, '\n') +end + +function tests.test_routes_decode_local_ui_and_apn_routes() + eq(routes.decode(ctx('GET', '/api/local-ui/bootstrap')).kind, 'local_ui_bootstrap') + eq(routes.decode(ctx('GET', '/api/gsm/apns/custom')).kind, 'gsm_apns_get') + eq(routes.decode(ctx('PUT', '/api/gsm/apns/custom')).kind, 'gsm_apns_put') + eq(routes.decode(ctx('GET', '/api/diagnostics')).kind, 'diagnostics_stub') + local logs_route = routes.decode(ctx('GET', '/api/logs?boot=true')) + eq(logs_route.kind, 'logs_query') + eq(logs_route.boot, true) + local header_logs_route = routes.decode({ + method = 'GET', + path = function () return '/api/logs' end, + headers = { [':path'] = '/api/logs?boot=true' }, + }) + eq(header_logs_route.kind, 'logs_query') + eq(header_logs_route.boot, true) + local route = routes.decode(ctx('GET', '/events')) + eq(route.kind, 'sse') + eq(route.pattern, nil) + eq(route.patterns, nil) +end + +function tests.test_sse_defaults_use_local_ui_prefixes_without_root_hash() + local defaults = sse.default_patterns() + local seen = {} + ok(#defaults > 0) + for _, pattern in ipairs(defaults) do + local key = table.concat(pattern, '/') + if key == '#' then fail('local-ui SSE must not subscribe to root #') end + seen[key] = true + end + ok(seen['state/device/#'], 'state/device stream missing') + ok(seen['state/net/#'], 'state/net stream missing') + ok(seen['state/system/#'], 'state/system stream missing') + eq(seen['obs/v1/system/metric/#'], nil, 'system metrics should not be a default UI stream') + eq(seen['obs/v1/gsm/event/#'], nil, 'generic GSM events should not be watched by /events') + eq(seen['obs/v1/+/event/log'], nil, 'logs should use the monitor log follow endpoint') +end + +function tests.test_sse_default_encoder_emits_parseable_json_for_multiline_strings() + local restore = '*mangle\n' + .. '-F mwan3_policy_balanced\n' + .. '-A mwan3_policy_balanced -m comment --comment "wan 88 88" -j MARK\n' + .. 'COMMIT\n' + local frame = sse.frame_event({ + kind = 'read_model_event', + op = 'set', + topic = { 'state', 'net', 'wan_runtime' }, + payload = { + live_weights = { + result = { + restore = restore, + }, + }, + }, + }) + local decoded, err = cjson.decode(sse_data(frame)) + ok(decoded, err or frame) + eq(decoded.topic[1], 'state') + eq(decoded.topic[2], 'net') + eq(decoded.topic[3], 'wan_runtime') + eq(decoded.payload.live_weights.result.restore, restore) +end + +function tests.test_local_model_allow_list_excludes_cfg_and_raw() + local model = read_model.new() + model:set({ 'state', 'net', 'summary' }, { ok = true }) + model:set({ 'state', 'gsm', 'apns', 'custom' }, { records = {} }) + model:set({ 'state', 'system', 'stats' }, { + cpu = { utilisation = 12.5 }, + }) + model:set({ 'state', 'device', 'component', 'switch-main' }, { + available = true, + observed = { wired = { raw = { secret = true } } }, + raw = { secret = true }, + }) + model:set({ 'state', 'device', 'components' }, { + counts = { total = 1 }, + components = { ['switch-main'] = { raw = { secret = true } } }, + }) + model:set({ 'cfg', 'gsm' }, { secret = true }) + model:set({ 'raw', 'member', 'mcu' }, { secret = true }) + local boot = local_model.bootstrap(model:snapshot()) + ok(boot.items['state/net/summary']) + ok(boot.items['state/gsm/apns/custom']) + ok(boot.items['state/system/stats']) + eq(boot.items['obs/v1/system/metric/cpu_util'], nil, 'system metrics should not be in local-ui bootstrap') + ok(boot.items['state/device/component/switch-main']) + eq(boot.items['state/device/component/switch-main'].payload.raw, nil) + eq(boot.items['state/device/component/switch-main'].payload.observed, nil) + ok(boot.items['state/device/components']) + eq(boot.items['state/device/components'].payload.components, nil) + if boot.items['cfg/gsm'] then fail('cfg/gsm leaked into local-ui bootstrap') end + if boot.items['raw/member/mcu'] then fail('raw member leaked into local-ui bootstrap') end +end + +function tests.test_local_model_keeps_logs_out_of_default_stream_and_bootstrap() + local model = read_model.new() + model:set({ 'obs', 'v1', 'net', 'event', 'log' }, { + level = 'warn', + message = 'not retained', + }) + + local log_event = local_model.project_event({ + op = 'set', + topic = { 'obs', 'v1', 'net', 'event', 'log' }, + payload = { + level = 'warn', + message = 'not retained', + }, + }) + ok(log_event, 'explicit canonical service log projection should remain supported') + eq(log_event.payload.message, 'not retained') + + local ui_log = local_model.project_event({ + op = 'set', + topic = { 'obs', 'v1', 'ui', 'event', 'log' }, + payload = { level = 'info', message = 'hidden' }, + }) + eq(ui_log, nil, 'ui service logs should remain denied') + + local gsm_log = local_model.project_event({ + op = 'set', + topic = { 'obs', 'v1', 'gsm', 'event', 'log' }, + payload = { level = 'info', message = 'hidden' }, + }) + ok(gsm_log, 'explicit GSM service log projection should remain supported') + + local boot = local_model.bootstrap(model:snapshot()) + if boot.items['obs/v1/net/event/log'] then fail('log event leaked into local-ui bootstrap') end +end + +return tests diff --git a/tests/unit/ui/test_read_model.lua b/tests/unit/ui/test_read_model.lua new file mode 100644 index 00000000..a35ae68a --- /dev/null +++ b/tests/unit/ui/test_read_model.lua @@ -0,0 +1,231 @@ +-- tests/unit/ui/test_read_model.lua + +local fibers = require 'fibers' +local busmod = require 'bus' +local read_model = require 'services.ui.read_model' +local store_mod = require 'services.ui.read_model_store' +local watches_mod = require 'services.ui.read_model_watches' +local run_fibers = require 'tests.support.run_fibers' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) + if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end +end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got '..tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end + + +function tests.test_read_model_default_exclusions_prevent_self_ingesting_ui_state() + local should = read_model._test.should_ingest_event + assert_eq(should({ op = 'retain', topic = { 'svc', 'ui', 'status' }, payload = {} }, {}), false) + assert_eq(should({ op = 'retain', topic = { 'svc', 'ui', 'meta' }, payload = {} }, {}), false) + assert_eq(should({ op = 'retain', topic = { 'svc', 'ui', 'announce' }, payload = {} }, {}), false) + assert_eq(should({ op = 'retain', topic = { 'state', 'device', 'summary' }, payload = {} }, {}), true) + assert_eq(should({ op = 'retain', topic = { 'state', 'ui', 'summary' }, payload = {} }, {}), false) + assert_eq(should({ op = 'retain', topic = { 'state', 'ui', 'read-model' }, payload = {} }, {}), false) + assert_eq(should({ op = 'unretain', topic = { 'state', 'ui', 'sessions' } }, {}), false) + assert_eq(should({ op = 'retain', topic = { 'obs', 'v1', 'ui', 'metric', 'requests' }, payload = {} }, {}), false) + assert_eq(should({ op = 'replay_done' }, {}), true) +end + + +function tests.test_default_retained_patterns_exclude_raw_unless_opted_in() + local topics = require 'services.ui.topics' + local function has_raw(patterns) + for _, p in ipairs(patterns or {}) do + if p[1] == 'raw' and p[2] == '#' then return true end + end + return false + end + assert_eq(has_raw(topics.default_retained_patterns()), false) + assert_eq(has_raw(topics.default_retained_patterns({ include_raw = true })), true) +end + +function tests.test_store_snapshot_changed_op_and_material_change() + run_fibers.run(function () + local model = read_model.new() + local seen = model:version() + + model:set({ 'svc', 'ui', 'status' }, { state = 'running' }) + local version, snap, err = fibers.perform(model:changed_op(seen)) + assert_nil(err) + assert_eq(version, 1) + assert_not_nil(snap.items) + + local a, b = fibers.perform(model:changed_op(version):or_else(function () + return nil, 'not_ready' + end)) + assert_nil(a) + assert_eq(b, 'not_ready') + + model:set({ 'svc', 'ui', 'status' }, { state = 'running' }) + local c, d = fibers.perform(model:changed_op(version):or_else(function () + return nil, 'not_ready' + end)) + assert_nil(c) + assert_eq(d, 'not_ready') + end) +end + +function tests.test_store_close_changed_op_returns_reason_shape() + run_fibers.run(function () + local model = store_mod.new() + local seen = model:version() + model:terminate('done') + local version, snap, err = fibers.perform(model:changed_op(seen)) + assert_nil(version) + assert_nil(snap) + assert_eq(err, 'done') + end) +end + +function tests.test_watch_replay_overflow_fails_open() + run_fibers.run(function () + local model = store_mod.new() + local watch_owner = watches_mod.new(model) + model:set({ 'state', 'a' }, 1) + model:set({ 'state', 'b' }, 2) + + local watch, err = watch_owner:watch_open({ 'state', '#' }, { queue_len = 1, full = 'reject_newest' }) + assert_nil(watch) + assert_eq(err, 'watch_replay_overflow') + end) +end + + +function tests.test_watch_replay_default_bound_fails_large_replay() + run_fibers.run(function () + local model = store_mod.new() + local watch_owner = watches_mod.new(model) + for i = 1, watches_mod.DEFAULT_MAX_REPLAY + 1 do + model:set({ 'state', i }, i) + end + + local watch, err = watch_owner:watch_open({ 'state', '#' }, { queue_len = 128, full = 'reject_newest' }) + assert_nil(watch) + assert_eq(err, 'watch_replay_overflow') + end) +end + +function tests.test_watch_receives_live_events_and_closes_on_overflow() + run_fibers.run(function () + local model = store_mod.new() + local watch_owner = watches_mod.new(model) + local watch = assert(watch_owner:watch_open({ 'state', '#' }, { queue_len = 1, replay = false })) + watch_owner:set({ 'state', 'a' }, 1) + local ev = fibers.perform(watch:recv_op()) + assert_eq(ev.op, 'set') + assert_eq(ev.topic[1], 'state') + + watch_owner:set({ 'state', 'b' }, 2) + watch_owner:set({ 'state', 'c' }, 3) + assert_eq(watch:why(), 'watch_overflow') + end) +end + +function tests.test_read_model_start_uses_supplied_store_and_watch_owner() + run_fibers.run(function (scope) + local store = store_mod.new() + local watch_owner = watches_mod.new(store) + local returned_store, returned_watches = read_model.start(scope, nil, { + model = store, + watch_owner = watch_owner, + }) + assert_eq(returned_store, store) + assert_eq(returned_watches, watch_owner) + + local watch = assert(watch_owner:watch_open({ 'svc', '#' }, { replay = false })) + watch_owner:ingest({ op = 'retain', topic = { 'svc', 'ui' }, payload = { status = 'running' } }) + local ev = fibers.perform(watch:recv_op()) + assert_eq(ev.op, 'set') + assert_eq(store:get({ 'svc', 'ui' }).payload.status, 'running') + end) +end + +function tests.test_read_model_forwards_non_retained_logs_without_storing_them() + run_fibers.run(function (scope) + local bus = busmod.new() + local feed_conn = bus:connect({ origin_base = { service = 'ui-read-model-test' } }) + local publisher = bus:connect({ origin_base = { service = 'net' } }) + scope:finally(function () + feed_conn:disconnect() + publisher:disconnect() + end) + + local store, watch_owner = read_model.start(scope, feed_conn, { + patterns = {}, + event_patterns = { + { 'obs', 'v1', '+', 'event', 'log' }, + }, + event_queue_len = 8, + }) + local watch = assert(watch_owner:watch_open({ 'obs', 'v1', '+', 'event', 'log' }, { + queue_len = 8, + replay = false, + })) + + publisher:publish({ 'obs', 'v1', 'net', 'event', 'log' }, { + level = 'warn', + message = 'wan offline', + }) + + local ev = fibers.perform(watch:recv_op()) + assert_eq(ev.op, 'set') + assert_eq(ev.topic[1], 'obs') + assert_eq(ev.topic[3], 'net') + assert_eq(ev.payload.level, 'warn') + assert_eq(ev.payload.message, 'wan offline') + assert_nil(store:get({ 'obs', 'v1', 'net', 'event', 'log' })) + + publisher:publish({ 'obs', 'v1', 'ui', 'event', 'log' }, { + level = 'info', + message = 'hidden', + }) + local unexpected = fibers.perform(watch:recv_op():or_else(function () + return nil, 'not_ready' + end)) + assert_nil(unexpected) + end) +end + +function tests.test_query_limited_stops_before_materialising_large_replay() + local model = store_mod.new() + for i = 1, 5 do model:set({ 'state', i }, i) end + local items, err = model:query_limited({ 'state', '#' }, 3) + assert_nil(items) + assert_eq(err, 'query_limit_exceeded') + items, err = model:query_limited({ 'state', '#' }, 5) + assert_not_nil(items, err) + assert_eq(#items, 5) +end + +function tests.test_read_model_query_uses_first_token_index_for_replay_candidates() + local model = store_mod.new() + for i = 1, 10 do model:set({ 'state', i }, i) end + model:set({ 'svc', 'ui' }, 'ui') + model:set({ 'svc', 'fabric' }, 'fabric') + + assert_eq(store_mod._test.candidate_count(model, { 'svc', '#' }), 2) + assert_eq(store_mod._test.candidate_count(model, { 'state', '#' }), 10) + assert_eq(store_mod._test.candidate_count(model, { '#'}), 12) + + local items, err = model:query_limited({ 'svc', '#' }, 2) + assert_not_nil(items, err) + assert_eq(#items, 2) +end + +function tests.test_read_model_index_updates_on_delete() + local model = store_mod.new() + model:set({ 'svc', 'ui' }, 'ui') + model:set({ 'svc', 'fabric' }, 'fabric') + assert_eq(store_mod._test.candidate_count(model, { 'svc', '#' }), 2) + model:delete({ 'svc', 'ui' }) + assert_eq(store_mod._test.candidate_count(model, { 'svc', '#' }), 1) + local items = model:query({ 'svc', '#' }) + assert_eq(#items, 1) + assert_eq(items[1].topic[2], 'fabric') +end + +return tests diff --git a/tests/unit/ui/test_service.lua b/tests/unit/ui/test_service.lua new file mode 100644 index 00000000..202fb07a --- /dev/null +++ b/tests/unit/ui/test_service.lua @@ -0,0 +1,277 @@ +-- tests/unit/ui/test_service.lua + +local service = require 'services.ui.service' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or 'expected true') end end + +local function base_state() + return { + components = { + read_model = { status = 'running' }, + http_listener = { status = 'running' }, + }, + service_status = 'running', + read_model_status = 'running', + listener_status = 'running', + } +end + +function tests.test_records_component_completion_before_policy() + local state = base_state() + local decision = service._test.reduce_event(state, { + kind = 'ui_component_done', + component = 'read_model', + status = 'ok', + result = { status = 'stopped' }, + }) + assert_true(decision.publish) + assert_eq(state.components.read_model.status, 'ok') + assert_eq(state.read_model_status, 'ok') +end + +function tests.test_read_model_failure_marks_service_failed() + local state = base_state() + local decision = service._test.reduce_event(state, { + kind = 'ui_component_done', + component = 'read_model', + status = 'failed', + primary = 'boom', + }) + assert_true(decision.publish) + assert_eq(decision.fail, 'read_model failed: boom') + assert_eq(state.service_status, 'failed') + assert_eq(state.last_error, 'read_model failed: boom') + assert_eq(state.components.read_model.status, 'failed') +end + +function tests.test_http_listener_failure_marks_service_failed() + local state = base_state() + local decision = service._test.reduce_event(state, { + kind = 'ui_component_done', + component = 'http_listener', + status = 'failed', + primary = 'listen failed', + }) + assert_eq(decision.fail, 'http_listener failed: listen failed') + assert_eq(state.service_status, 'failed') + assert_eq(state.listener_status, 'failed') +end + +function tests.test_stale_component_completion_is_ignored() + local state = base_state() + service._test.reduce_event(state, { + kind = 'ui_component_done', + component = 'read_model', + status = 'ok', + result = { status = 'stopped' }, + }) + local decision = service._test.reduce_event(state, { + kind = 'ui_component_done', + component = 'read_model', + status = 'failed', + primary = 'late', + }) + assert_eq(next(decision), nil) + assert_eq(state.components.read_model.status, 'ok') +end + + + +function tests.test_publish_summary_is_local_projection_not_retained_state() + local retain_calls = 0 + local state = base_state() + state.conn = { + retain = function () retain_calls = retain_calls + 1; return nil, 'retain_failed' end, + } + state.sessions = { count = function () return 0 end } + state.active_requests = 0 + state.rejected_requests = 0 + + local payload = service._test.publish_summary(state) + assert_eq(payload.service_status, 'running') + assert_eq(payload.read_model_status, 'running') + assert_eq(payload.listener_status, 'running') + assert_eq(retain_calls, 0) +end + +function tests.test_session_events_are_first_class_service_events() + local state = base_state() + local decision = service._test.reduce_event(state, { + kind = 'session_created', + session_id = 's1', + count = 1, + }) + assert_true(decision.publish) + assert_eq(state.last_session_event.kind, 'session_created') + assert_eq(state.last_session_event.session_id, 's1') + + decision = service._test.reduce_event(state, { + kind = 'session_pruned', + session_ids = { 's1' }, + count = 0, + }) + assert_true(decision.publish) + assert_eq(state.last_session_event.kind, 'session_pruned') +end + + + +function tests.test_stale_http_listener_request_events_are_ignored() + local state = base_state() + state.listener_generation = 3 + state.active_requests = 0 + state.rejected_requests = 0 + + local decision = service._test.reduce_event(state, { + kind = 'http_request_started', + generation = 2, + listener_id = 'http_listener:2', + request_id = 'old', + active_requests = 1, + }) + assert_eq(next(decision), nil) + assert_eq(state.active_requests, 0) + + decision = service._test.reduce_event(state, { + kind = 'http_request_started', + generation = 3, + listener_id = 'http_listener:3', + request_id = 'new', + active_requests = 1, + }) + assert_true(decision.publish) + assert_eq(state.active_requests, 1) +end + +function tests.test_read_model_changed_updates_seen_without_lifecycle_publish() + local state = base_state() + state.model_seen = 4 + + local decision = service._test.reduce_event(state, { + kind = 'read_model_changed', + version = 5, + snapshot = { services = {} }, + }) + + assert_eq(state.model_seen, 5) + assert_true(decision.coalesce_publish) + assert_eq(decision.reason, 'read_model_changed') +end + + +function tests.test_http_dependency_changes_are_coalesced_before_lifecycle_publish() + local state = base_state() + local decision = service._test.reduce_event(state, { + kind = 'http_dependency_changed', + key = 'http', + available = false, + reason = 'http_unavailable', + }) + assert_true(decision.coalesce_publish) + assert_eq(decision.publish, nil) + service._test.schedule_lifecycle_publish(state, decision.reason) + assert_true(state.lifecycle_publish_pending) + assert_true(type(state.lifecycle_publish_due_at) == 'number') +end + +function tests.test_session_changed_updates_seen_without_lifecycle_publish() + local state = base_state() + state.sessions_seen = 2 + state.last_session_event = nil + + local ev = { + kind = 'session_changed', + version = 3, + last_event = { + kind = 'session_count_changed', + count = 1, + }, + } + + local decision = service._test.reduce_event(state, ev) + + assert_eq(state.sessions_seen, 3) + assert_eq(state.last_session_event, ev.last_event) + assert_eq(next(decision), nil) +end + +function tests.test_cleanup_error_recording_is_explicit_and_non_throwing() + local state = base_state() + local rec = service._test.record_cleanup_error(state, 'ui_cleanup_failed', 'cleanup_failed') + assert_eq(rec.kind, 'ui_cleanup_failed') + assert_eq(rec.err, 'cleanup_failed') + assert_eq(state.last_cleanup_error, rec) +end + + + +function tests.test_lifecycle_status_is_gated_by_semantic_change() + local calls = {} + local lifecycle = { + running = function (_, payload) calls[#calls + 1] = payload; return payload end, + } + local state = base_state() + state.lifecycle = lifecycle + state.config_status = 'ok' + state.config_generation = 1 + state.config = { enabled = true, http = { enabled = true }, observability = { status_interval_s = 30 } } + state.lifecycle_obs = {} + + service._test.update_lifecycle(state) + assert_eq(#calls, 1) + service._test.update_lifecycle(state) + assert_eq(#calls, 1, 'unchanged lifecycle status should not republish') + + state.config_generation = 2 + service._test.update_lifecycle(state) + assert_eq(#calls, 2, 'semantic lifecycle change should publish') +end + +function tests.test_lifecycle_status_key_ignores_volatile_dependency_timestamps() + local a = service._test.lifecycle_status_key('running', { + ready = true, + dependencies = { + http = { status = 'ready', available = true, updated_at = 1.0 }, + }, + }) + local b = service._test.lifecycle_status_key('running', { + ready = true, + dependencies = { + http = { status = 'ready', available = true, updated_at = 2.0 }, + }, + }) + assert_eq(a, b) +end + +function tests.test_lifecycle_not_ready_until_config_and_listener_are_ready() + local calls = {} + local lifecycle = { + running = function (_, payload) calls[#calls + 1] = payload; return payload end, + } + local state = base_state() + state.lifecycle = lifecycle + state.config_status = 'waiting' + + service._test.update_lifecycle(state) + assert_eq(calls[#calls].ready, false) + assert_eq(calls[#calls].reason, 'waiting_for_config') + + state.config_status = 'ok' + state.config = { enabled = true, http = { enabled = true } } + state.listener_status = 'waiting_for_http' + state.last_error = 'http_unavailable' + service._test.update_lifecycle(state) + assert_eq(calls[#calls].ready, false) + assert_eq(calls[#calls].reason, 'http_unavailable') + + state.listener_status = 'running' + state.last_error = nil + service._test.update_lifecycle(state) + assert_eq(calls[#calls].ready, true) +end + +return tests diff --git a/tests/unit/ui/test_sessions.lua b/tests/unit/ui/test_sessions.lua new file mode 100644 index 00000000..02fbeed6 --- /dev/null +++ b/tests/unit/ui/test_sessions.lua @@ -0,0 +1,124 @@ +-- tests/unit/ui/test_sessions.lua + +local sessions = require 'services.ui.sessions' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or 'expected true') end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got '..tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end + +function tests.test_create_touch_delete_and_prune_are_immediate() + local now = 100 + local store = sessions.new({ now = function() return now end, default_ttl = 10 }) + + local s = store:create('alice', { id = 's1', data = { role = 'admin' } }) + assert_eq(s.id, 's1') + assert_eq(s.principal, 'alice') + assert_eq(store:count(), 1) + + now = 105 + local touched, err = store:touch('s1', { ttl = 20 }) + assert_not_nil(touched, err) + assert_eq(touched.expires_at, 125) + + assert_true(store:delete('s1')) + assert_nil(store:get('s1')) + + store:create('bob', { id = 's2', ttl = 1 }) + store:create('carol', { id = 's3', ttl = 100 }) + now = 107 + local removed = store:prune() + assert_eq(#removed, 1) + assert_eq(removed[1], 's2') + assert_eq(store:count(), 1) +end + +function tests.test_session_mutations_record_first_class_last_event() + local now = 100 + local store = sessions.new({ + now = function() return now end, + default_ttl = 10, + }) + + store:create('alice', { id = 's1' }) + assert_eq(store:last_event().kind, 'session_created') + assert_eq(store:last_event().session_id, 's1') + assert_eq(store:last_event().count, 1) + + store:touch('s1') + assert_eq(store:last_event().kind, 'session_touched') + + store:delete('s1') + assert_eq(store:last_event().kind, 'session_deleted') + assert_eq(store:last_event().count, 0) + + store:create('bob', { id = 's2', ttl = 1 }) + assert_eq(store:last_event().kind, 'session_created') + + now = 102 + store:prune() + assert_eq(store:last_event().kind, 'session_pruned') + assert_eq(store:last_event().session_ids[1], 's2') + assert_eq(store:last_event().count, 0) +end + +function tests.test_expired_session_get_is_pure_and_prune_records_event() + local now = 100 + local store = sessions.new({ + now = function() return now end, + }) + store:create('alice', { id = 's1', ttl = 1 }) + local created = store:last_event() + assert_eq(created.kind, 'session_created') + now = 102 + assert_nil(store:get('s1')) + assert_eq(store:last_event().kind, 'session_created') + assert_eq(store:count(), 0) + local removed = store:prune() + assert_eq(removed[1], 's1') + assert_eq(store:last_event().kind, 'session_pruned') + assert_eq(store:last_event().session_ids[1], 's1') + assert_eq(store:last_event().count, 0) +end + + +function tests.test_session_mutations_signal_versioned_changed_op() + local fibers = require 'fibers' + fibers.run(function() + local now = 100 + local store = sessions.new({ now = function() return now end, default_ttl = 10 }) + local seen = store:version() + + store:create('alice', { id = 's1' }) + local version, snap, err = fibers.perform(store:changed_op(seen)) + assert_nil(err) + assert_eq(version, 1) + assert_eq(snap.count, 1) + assert_eq(snap.last_event.kind, 'session_created') + assert_eq(snap.last_event.session_id, 's1') + + local v2, _, e2 = fibers.perform(store:changed_op(version):or_else(function() + return nil, nil, 'not_ready' + end)) + assert_nil(v2) + assert_eq(e2, 'not_ready') + end) +end + +function tests.test_session_store_records_session_events_in_model_state() + local now = 100 + local store = sessions.new({ + now = function() return now end, + }) + + local sess = store:create('alice', { id = 's1' }) + assert_eq(sess.id, 's1') + assert_eq(store:count(), 1) + assert_eq(store:version(), 1) + assert_eq(store:last_event().kind, 'session_created') +end + +return tests diff --git a/tests/unit/ui/test_supervision.lua b/tests/unit/ui/test_supervision.lua new file mode 100644 index 00000000..62f20bda --- /dev/null +++ b/tests/unit/ui/test_supervision.lua @@ -0,0 +1,42 @@ +-- tests/unit/ui/test_supervision.lua + +local supervision = require 'services.ui.supervision' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end + +function tests.test_service_core_component_ok_continues() + local d = supervision.classify_service_component_done({}, { + component = 'http_listener', + status = 'ok', + result = { status = 'stopped' }, + }) + assert_eq(d.class, 'normal_close') + assert_eq(d.action, 'continue') +end + +function tests.test_service_core_component_failure_fails_service() + local d = supervision.classify_service_component_done({}, { + component = 'read_model', + status = 'failed', + primary = 'boom', + }) + assert_eq(d.class, 'failed') + assert_eq(d.action, 'fail_service') + assert_eq(d.reason, 'read_model failed: boom') +end + +function tests.test_service_core_component_cancellation_fails_service() + local d = supervision.classify_service_component_done({}, { + component = 'http_listener', + status = 'cancelled', + primary = 'lost', + }) + assert_eq(d.class, 'cancelled_unexpected') + assert_eq(d.action, 'fail_service') + assert_eq(d.reason, 'http_listener cancelled unexpectedly: lost') +end + +return tests diff --git a/tests/unit/ui/test_update_upload.lua b/tests/unit/ui/test_update_upload.lua new file mode 100644 index 00000000..a2a89e6e --- /dev/null +++ b/tests/unit/ui/test_update_upload.lua @@ -0,0 +1,276 @@ +-- tests/unit/ui/test_update_upload.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local upload = require 'services.ui.update.upload' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got '..tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end + +local function body_from_chunks(chunks) + return { + _chunks = chunks, + read_chunk_op = function (self) + local chunk = table.remove(self._chunks, 1) + return fibers.always(chunk, nil) + end, + } +end + +local function ingest_client(handle) + return { + open_ingest_op = function () + return fibers.always(handle, nil) + end, + } +end + +function tests.test_upload_abort_finaliser_runs_when_append_fails() + fibers.run(function () + local handle = { + aborted = nil, + append_chunk_op = function () return fibers.always(nil, 'append_failed') end, + commit_op = function () error('commit must not be called') end, + abort_now = function (self, reason) self.aborted = reason; return true end, + } + + local st, _, primary = fibers.perform(upload.run_op({ + body_stream = body_from_chunks({ 'abc' }), + }, { + ingest = ingest_client(handle), + })) + + assert_eq(st, 'failed') + assert_eq(primary, 'append_failed') + assert_eq(handle.aborted, 'append_failed') + end) +end + +function tests.test_committed_artifact_is_not_aborted_when_update_job_create_fails() + fibers.run(function () + local handle = { + aborted = nil, + chunks = {}, + append_chunk_op = function (self, chunk) self.chunks[#self.chunks + 1] = chunk; return fibers.always(true, nil) end, + commit_op = function () return fibers.always('artifact-1', nil) end, + abort_now = function (self, reason) self.aborted = reason; return true end, + } + local conn = { + call_op = function () return fibers.always(nil, 'create_failed') end, + } + + local st, _, primary = fibers.perform(upload.run_op({ + body_stream = body_from_chunks({ 'abc' }), + }, { + ingest = ingest_client(handle), + create_job = true, + conn = conn, + })) + + assert_eq(st, 'failed') + assert_eq(primary, 'create_failed') + assert_nil(handle.aborted) + assert_eq(handle.chunks[1], 'abc') + end) +end + + +function tests.test_upload_fails_when_create_job_reply_has_no_job_id() + fibers.run(function () + local handle = { + append_chunk_op = function () return fibers.always(true, nil) end, + commit_op = function () return fibers.always('artifact-missing-job-id', nil) end, + abort_now = function () error('abort must not be called after commit') end, + } + local conn = { + call_op = function () return fibers.always({}, nil) end, + } + + local st, _, primary = fibers.perform(upload.run_op({ + body_stream = body_from_chunks({ 'abc' }), + }, { + ingest = ingest_client(handle), + create_job = true, + conn = conn, + })) + + assert_eq(st, 'failed') + assert_eq(primary, 'create_job_reply_missing_job_id') + end) +end + +function tests.test_upload_disconnects_owned_update_connection_after_success() + fibers.run(function () + local handle = { + append_chunk_op = function () return fibers.always(true, nil) end, + commit_op = function () return fibers.always('artifact-2', nil) end, + abort_now = function () error('abort must not be called after commit') end, + } + local disconnected = false + local conn = { + call_op = function (_, topic) + if topic[5] == 'create-job' then return fibers.always({ job_id = 'job-1' }, nil) end + assert_eq(topic[5], 'start-job') + return fibers.always({ started = true }, nil) + end, + disconnect = function () disconnected = true; return true end, + } + + local st, _, result = fibers.perform(upload.run_op({ + body_stream = body_from_chunks({ 'abc' }), + }, { + ingest = ingest_client(handle), + create_job = true, + start_job = true, + connect = function () return conn, nil end, + })) + + assert_eq(st, 'ok') + assert_eq(result.artifact_id, 'artifact-2') + assert_eq(result.job_id, 'job-1') + assert_nil(result.job) + assert_not_nil(result.started) + assert_eq(disconnected, true) + end) +end + + +function tests.test_upload_uses_borrowed_service_connection_for_public_route() + fibers.run(function () + local handle = { + append_chunk_op = function () return fibers.always(true, nil) end, + commit_op = function () return fibers.always('artifact-borrowed', nil) end, + abort_now = function () error('abort must not be called after commit') end, + } + local disconnected = false + local conn = { + call_op = function (_, topic) + if topic[5] == 'create-job' then return fibers.always({ job_id = 'job-borrowed' }, nil) end + assert_eq(topic[5], 'start-job') + return fibers.always({ started = true }, nil) + end, + disconnect = function () disconnected = true; return true end, + } + + local st, _, result = fibers.perform(upload.run_op({ + body_stream = body_from_chunks({ 'abc' }), + }, { + ingest = ingest_client(handle), + conn = conn, + create_job = true, + start_job = true, + connect = function () error('public upload should borrow supplied service conn') end, + })) + + assert_eq(st, 'ok') + assert_eq(result.artifact_id, 'artifact-borrowed') + assert_eq(result.job_id, 'job-borrowed') + assert_nil(result.job) + assert_not_nil(result.started) + assert_eq(disconnected, false) + end) +end + + +function tests.test_upload_timeout_cancels_scope_and_aborts_uncommitted_ingest() + fibers.run(function () + local handle = { + aborted = nil, + append_chunk_op = function () return fibers.always(true, nil) end, + commit_op = function () return fibers.always('artifact-timeout', nil) end, + abort_now = function (self, reason) self.aborted = reason; return true end, + } + local body = { + read_chunk_op = function () + return sleep.sleep_op(1):wrap(function () return 'late', nil end) + end, + } + + local st, _, primary = fibers.perform(upload.run_op({ + body_stream = body, + }, { + ingest = ingest_client(handle), + upload_timeout = 0.01, + })) + + assert_eq(st, 'cancelled') + assert_eq(primary, 'timeout') + assert_eq(handle.aborted, 'timeout') + end) +end + + +function tests.test_upload_requires_read_chunk_op_body_contract() + fibers.run(function () + local handle = { + append_chunk_op = function () error('append must not be called') end, + commit_op = function () error('commit must not be called') end, + abort_now = function (self, reason) self.aborted = reason; return true end, + } + + local st, _, primary = fibers.perform(upload.run_op({ + body_stream = { read_op = function () error('compat path must not be used') end }, + }, { + ingest = ingest_client(handle), + })) + + assert_eq(st, 'failed') + assert_eq(primary, 'request body has no read_chunk_op') + assert_eq(handle.aborted, 'request body has no read_chunk_op') + end) +end + +function tests.test_artifact_ingest_boundary_requires_op_methods_and_abort_now() + local ingest = require 'services.ui.update.artifact_ingest' + fibers.run(function () + local handle, err = fibers.perform(ingest.open_ingest_op({ open_ingest = function () return {} end }, {})) + assert_nil(handle) + assert_eq(err, 'artifact ingest client must expose open_ingest_op') + + local ok, aerr = fibers.perform(ingest.append_chunk_op({ append = function () return true end }, 'abc')) + assert_nil(ok) + assert_eq(aerr, 'artifact ingest handle must expose append_chunk_op') + + local artifact, cerr = fibers.perform(ingest.commit_op({ commit = function () return 'artifact' end })) + assert_nil(artifact) + assert_eq(cerr, 'artifact ingest handle must expose commit_op') + end) + + local ok, err = ingest.abort_now({ abort = function () return true end }, 'closed') + assert_nil(ok) + assert_eq(err, 'artifact ingest handle must expose immediate abort_now') + + ok, err = ingest.abort_now({ abort_now = function () return fibers.always(true, nil) end }, 'closed') + assert_nil(ok) + assert_eq(err, 'artifact ingest abort_now must be immediate and must not return an Op') +end + +function tests.test_upload_timeout_while_append_pending_aborts_uncommitted_ingest() + fibers.run(function () + local handle = { + aborted = nil, + append_chunk_op = function () + return sleep.sleep_op(1):wrap(function () return true, nil end) + end, + commit_op = function () error('commit must not be called') end, + abort_now = function (self, reason) self.aborted = reason; return true end, + } + + local st, _, primary = fibers.perform(upload.run_op({ + body_stream = body_from_chunks({ 'abc' }), + }, { + ingest = ingest_client(handle), + upload_timeout = 0.01, + })) + + assert_eq(st, 'cancelled') + assert_eq(primary, 'timeout') + assert_eq(handle.aborted, 'timeout') + end) +end + +return tests diff --git a/tests/unit/ui/test_user_operation.lua b/tests/unit/ui/test_user_operation.lua new file mode 100644 index 00000000..b5044b7b --- /dev/null +++ b/tests/unit/ui/test_user_operation.lua @@ -0,0 +1,65 @@ +-- tests/unit/ui/test_user_operation.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local user_operation = require 'services.ui.user_operation' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got '..tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end + +function tests.test_user_operation_success_disconnects_owned_connection() + fibers.run(function () + local disconnected = false + local conn = { disconnect = function () disconnected = true; return true end } + local result, err = user_operation.run { + principal = 'alice', + connect = function () return conn, nil end, + run = function (_, c) + assert_eq(c, conn) + return { ok = true } + end, + } + assert_not_nil(result, err) + assert_eq(result.ok, true) + assert_eq(disconnected, true) + end) +end + +function tests.test_user_operation_timeout_returns_cancelled_shape_and_disconnects() + fibers.run(function () + local disconnected = false + local conn = { disconnect = function () disconnected = true; return true end } + local result, err, rep = user_operation.run { + principal = 'alice', + connect = function () return conn, nil end, + timeout = 0.01, + run = function () + fibers.perform(sleep.sleep_op(1)) + return { late = true } + end, + } + assert_nil(result) + assert_eq(err, 'timeout') + assert_not_nil(rep) + assert_eq(disconnected, true) + end) +end + +function tests.test_user_operation_borrowed_connection_is_not_disconnected_by_default() + fibers.run(function () + local disconnected = false + local conn = { disconnect = function () disconnected = true; return true end } + local result, err = user_operation.run { + conn = conn, + run = function () return { ok = true } end, + } + assert_not_nil(result, err) + assert_eq(disconnected, false) + end) +end + +return tests diff --git a/tests/unit/update/test_active_policy.lua b/tests/unit/update/test_active_policy.lua new file mode 100644 index 00000000..300757e1 --- /dev/null +++ b/tests/unit/update/test_active_policy.lua @@ -0,0 +1,60 @@ +local policy = require 'services.update.active_policy' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) + if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end +end +local function assert_not_nil(v, msg) + if v == nil then fail(msg or 'expected non-nil') end +end + +function tests.test_artifact_missing_stage_resume_moves_mcu_job_to_reconcile() + local job = { + job_id = 'j1', + component = 'mcu', + state = 'staging', + expected_image_id = 'img-1', + adoption = { action = 'resume_active_intent' }, + active_intent = { phase = 'stage' }, + created_seq = 1, + updated_seq = 1, + } + + policy.apply_completion(job, { + status = 'failed', + phase = 'stage', + primary = 'not_found', + }, 2) + + assert_eq(job.state, 'awaiting_return') + assert_eq(job.next_step, 'reconcile') + assert_not_nil(job.commit_result) + assert_eq(job.commit_result.tag, 'artifact_missing_reconcile') + assert_eq(job.commit_result.expected_image_id, 'img-1') + assert_eq(job.error, nil) +end + +function tests.test_artifact_missing_stage_resume_without_identity_still_fails() + local job = { + job_id = 'j2', + component = 'mcu', + state = 'staging', + adoption = { action = 'resume_active_intent' }, + active_intent = { phase = 'stage' }, + created_seq = 1, + updated_seq = 1, + } + + policy.apply_completion(job, { + status = 'failed', + phase = 'stage', + primary = 'not_found', + }, 2) + + assert_eq(job.state, 'failed') + assert_eq(job.error, 'not_found') +end + +return tests diff --git a/tests/unit/update/test_active_runtime.lua b/tests/unit/update/test_active_runtime.lua new file mode 100644 index 00000000..e45dd45f --- /dev/null +++ b/tests/unit/update/test_active_runtime.lua @@ -0,0 +1,233 @@ +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local op = require 'fibers.op' +local active = require 'services.update.active_runtime' +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v,msg) if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end end +local function assert_nil(v,msg) if v ~= nil then fail(msg or ('expected nil, got '..tostring(v))) end end +local function assert_not_nil(v,msg) if v == nil then fail(msg or 'expected non-nil') end end + +local function stage_backend(result) + return { stage_op = function () return op.always(result or { ok = true }, nil) end } +end +local function transition_handle(result) + return { outcome_op = function () return op.always(result or { status = 'persisted' }, nil) end } +end +function tests.test_claim_rejects_busy_and_completion_releases_slot() + fibers.run(function (scope) + local state = active.new_state(); local done_tx, done_rx = mailbox.new(8, { full='reject_newest' }) + local lease = assert(active.claim(state, { job_id='j1', generation=1, phase='stage' })) + local second, err = active.claim(state, { job_id='j2', generation=1, phase='stage' }); assert_nil(second); assert_eq(err, 'slot_busy') + local handle = assert(active.start_work(scope, state, { lease=lease, done_tx=done_tx, job={ job_id='j1', component='cm5' }, backend=stage_backend({ ok=true }) })) + assert_not_nil(handle) + local ev = fibers.perform(done_rx:recv_op()); assert_eq(ev.kind, 'active_job_done'); assert_eq(ev.status, 'ok') + assert_true(active.apply_completion(state, ev)); assert_not_nil(state.active); assert_eq(state.active.status, 'completed_pending_persist') + local busy, busy_err = active.claim(state, { job_id='j2', generation=1, phase='stage' }); assert_nil(busy); assert_eq(busy_err, 'slot_busy') + assert_true(active.release_completed(state, ev.token)); assert_nil(state.active) + assert_not_nil(active.claim(state, { job_id='j2', generation=1, phase='stage' })) + end) +end + +function tests.test_lease_release_clears_unstarted_slot_only() + local state = active.new_state() + local lease = assert(active.claim(state, { job_id='j1', generation=1, phase='stage' })) + local ok = assert(lease:release('never_started')) + assert_eq(ok, true) + assert_nil(state.active) + assert_eq(state.stats.released, 1) + assert_not_nil(active.claim(state, { job_id='j2', generation=1, phase='stage' })) +end + +function tests.test_handed_off_lease_cannot_release_running_slot() + local state = active.new_state() + local lease = assert(active.claim(state, { job_id='j1', generation=1, phase='stage' })) + assert_true(lease:handoff()) + local ok, err = lease:release('too_late') + assert_eq(ok, false) + assert_eq(err, 'transferred') + assert_not_nil(state.active) +end + +function tests.test_stale_completion_does_not_release_current_slot() + local state = active.new_state(); assert(active.claim(state, { job_id='j1', generation=1, phase='stage' })) + local ok, err = active.apply_completion(state, { kind='active_job_done', job_id='j1', generation=1, phase='stage', token='wrong', status='ok', result={ tag='staged' } }) + assert_eq(ok, false); assert_eq(err, 'stale'); assert_not_nil(state.active); assert_eq(state.stats.stale, 1) +end + +function tests.test_local_observer_is_notified_after_authoritative_completion_admission() + fibers.run(function (scope) + local state = active.new_state() + local done_tx, done_rx = mailbox.new(8, { full='reject_newest' }) + local observer_scope = assert(scope:child()) + local lease = assert(active.claim(state, { job_id='j1', generation=1, phase='stage' })) + local handle = assert(active.start_work(scope, state, { + lease = lease, + done_tx = done_tx, + local_observer_scope = observer_scope, + job={ job_id='j1', component='cm5' }, + backend=stage_backend({ ok=true }), + })) + + local authoritative = fibers.perform(done_rx:recv_op()) + assert_eq(authoritative.kind, 'active_job_done') + assert_eq(authoritative.status, 'ok') + + local observed = fibers.perform(handle:outcome_op()) + assert_eq(observed.kind, 'active_job_done') + assert_eq(observed.token, authoritative.token) + end) +end + + +function tests.test_component_stores_completion_before_reporting_to_service() + fibers.run(function (scope) + local service_tx, service_rx = mailbox.new(8, { full = 'reject_newest' }) + local fake_jobs = { admit_transition = function () return { outcome_op = function () return fibers.never() end }, nil end } + local component = assert(active.start_component(scope, { + service_id = 'update', + done_tx = service_tx, + work_scope = scope, + jobs = fake_jobs, + })) + local lease = assert(component:claim({ job_id='j1', generation=1, phase='stage' })) + assert(component:start_work({ + lease = lease, + job={ job_id='j1', component='cm5' }, + backend=stage_backend({ ok=true }), + })) + local ev = fibers.perform(service_rx:recv_op()) + assert_eq(ev.kind, 'active_runtime_changed') + assert_eq(ev.reason, 'active_job_completed') + assert_not_nil(active.completion(component:state(), ev.token), 'completion should be stored before report') + assert_not_nil(component:state().active) + assert_eq(component:state().active.status, 'completed_pending_persist') + component:release_completed(ev.token, 'test persisted') + assert_nil(component:state().active) + component:cancel('test complete') + end) +end + +function tests.test_component_apply_start_failure_keeps_stored_completion_and_fails_component() + fibers.run(function (scope) + local service_tx, service_rx = mailbox.new(8, { full = 'reject_newest' }) + local component = assert(active.start_component(scope, { + service_id = 'update', + done_tx = service_tx, + work_scope = scope, + jobs = {}, + })) + + local lease = assert(component:claim({ job_id = 'j1', generation = 1, phase = 'stage' })) + assert(component:start_work({ + lease = lease, + job={ job_id='j1', component='cm5' }, + backend=stage_backend({ ok=true }), + })) + + local completed = fibers.perform(service_rx:recv_op()) + assert_eq(completed.kind, 'active_runtime_changed') + assert_eq(completed.reason, 'active_job_completed') + assert_not_nil(active.completion(component:state(), completed.token), 'completion should be stored before apply starts') + assert_not_nil(component:state().active) + assert_eq(component:state().active.status, 'completed_pending_persist') + + local failed = fibers.perform(service_rx:recv_op()) + assert_eq(failed.kind, 'component_done') + assert_eq(failed.component, 'active_runtime') + assert_eq(failed.status, 'failed') + assert_eq(failed.primary, 'job_runtime_unavailable') + assert_not_nil(active.completion(component:state(), completed.token), 'stored completion should remain accounted for') + assert_not_nil(component:state().active, 'slot should not be silently released when apply cannot start') + end) +end + +function tests.test_component_apply_failure_after_start_keeps_completion_and_reports_failure() + fibers.run(function (scope) + local service_tx, service_rx = mailbox.new(8, { full = 'reject_newest' }) + local fake_jobs = { + admit_transition = function () + return transition_handle({ status = 'failed', reason = 'save_failed' }), nil + end, + } + local component = assert(active.start_component(scope, { + service_id = 'update', + done_tx = service_tx, + work_scope = scope, + jobs = fake_jobs, + })) + + local lease = assert(component:claim({ job_id = 'j1', generation = 1, phase = 'stage' })) + assert(component:start_work({ + lease = lease, + job={ job_id='j1', component='cm5' }, + backend=stage_backend({ ok=true }), + })) + + local completed = fibers.perform(service_rx:recv_op()) + assert_eq(completed.kind, 'active_runtime_changed') + assert_eq(completed.reason, 'active_job_completed') + assert_not_nil(active.completion(component:state(), completed.token), 'completion should be stored before durable apply') + + local apply_failed = fibers.perform(service_rx:recv_op()) + assert_eq(apply_failed.kind, 'active_runtime_changed') + assert_eq(apply_failed.reason, 'active_job_apply_failed') + assert_eq(apply_failed.error, 'save_failed') + assert_eq(component:state().active, nil, 'failed durable apply policy should release completed slot explicitly') + assert_not_nil(active.completion(component:state(), completed.token), 'stored completion should remain available after apply failure') + + local failed = fibers.perform(service_rx:recv_op()) + assert_eq(failed.kind, 'component_done') + assert_eq(failed.component, 'active_runtime') + assert_eq(failed.status, 'failed') + assert_eq(failed.primary, 'save_failed') + end) +end + +function tests.test_component_auto_commit_policy_admits_commit_after_stage_apply() + fibers.run(function (scope) + local service_tx, service_rx = mailbox.new(16, { full = 'reject_newest' }) + local seen = {} + local fake_jobs = { + admit_transition = function (_, cmd) + seen[#seen + 1] = cmd + if cmd.kind == 'apply_active_result' then + return transition_handle({ + status = 'persisted', + job = { job_id = 'j1', component = 'cm5', generation = 1, state = 'awaiting_commit', policy = { commit = 'auto' } }, + }), nil + elseif cmd.kind == 'start_job' then + return transition_handle({ status = 'persisted', job_id = cmd.job_id, phase = cmd.phase, token = 'commit-token', job = { job_id = cmd.job_id, component = 'cm5', state = 'committing' } }), nil + end + return transition_handle({ status = 'rejected', reason = 'unexpected' }), nil + end, + list = function () return {} end, + } + local component = assert(active.start_component(scope, { + service_id = 'update', + done_tx = service_tx, + work_scope = scope, + jobs = fake_jobs, + })) + local lease = assert(component:claim({ job_id = 'j1', generation = 1, phase = 'stage' })) + assert(component:start_work({ + lease = lease, + job = { job_id='j1', component='cm5', generation = 1, policy = { commit = 'auto' } }, + backend = stage_backend({ ok=true }), + })) + local completed = fibers.perform(service_rx:recv_op()) + assert_eq(completed.reason, 'active_job_completed') + local applied = fibers.perform(service_rx:recv_op()) + assert_eq(applied.reason, 'active_job_applied') + local auto = fibers.perform(service_rx:recv_op()) + assert_eq(auto.reason, 'policy_auto_commit_started') + assert_eq(seen[1].kind, 'apply_active_result') + assert_eq(seen[2].kind, 'start_job') + assert_eq(seen[2].phase, 'commit') + assert_eq(seen[2].reason, 'policy_auto_commit') + component:cancel('test complete') + end) +end + +return tests diff --git a/tests/unit/update/test_architecture.lua b/tests/unit/update/test_architecture.lua new file mode 100644 index 00000000..0e3a9732 --- /dev/null +++ b/tests/unit/update/test_architecture.lua @@ -0,0 +1,358 @@ +-- tests/unit/update/test_architecture.lua + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end + +local function read_file(path) + local f = assert(io.open(path, 'r')) + local s = f:read('*a') + f:close() + return s +end + +local function list_update_files() + local p = io.popen("find ../src/services/update -type f -name '*.lua' | sort") + local out = {} + for line in p:lines() do out[#out + 1] = line end + p:close() + out[#out + 1] = '../src/services/update.lua' + return out +end + + +function tests.test_update_service_defaults_to_control_store_job_store() + local svc = read_file('../src/services/update/service.lua') + if not svc:find("services.update.job_store_control_store", 1, true) then + fail('service.lua should import the control-store-backed job store') + end + if not svc:find("control_store_jobs.new", 1, true) then + fail('service.lua should build the control-store job store by default') + end + if not svc:find("services.update.job_store_memory", 1, true) then + fail('service.lua should import strict memory job store for explicit tests/harness use') + end + if not svc:find("job_store_kind == 'memory'", 1, true) then + fail('memory job store should be an explicit test/harness opt-in') + end + if svc:find("services.update.job_store_cap", 1, true) then + fail('service.lua should not use job_store_cap compatibility wrapper') + end + if svc:find("services.update.artifacts.store_cap", 1, true) then + fail('service.lua should not use artifact store compatibility wrapper') + end +end + +function tests.test_update_service_code_does_not_use_perform_raw() + for _, path in ipairs(list_update_files()) do + local s = read_file(path) + if s:find('perform_raw', 1, true) then + fail('perform_raw found in ' .. path) + end + end +end + +function tests.test_update_service_code_does_not_call_join_op() + for _, path in ipairs(list_update_files()) do + local s = read_file(path) + if s:find('join_op', 1, true) then + fail('join_op found in ' .. path) + end + end +end + +function tests.test_update_finalisers_do_not_call_close_op() + for _, path in ipairs(list_update_files()) do + local s = read_file(path) + if s:find('close_op', 1, true) then + fail('close_op found in ' .. path) + end + end +end + +function tests.test_priority_event_is_used_for_service_event_selection() + local s = read_file('../src/services/update/events.lua') + if not s:find("devicecode.support.priority_event", 1, true) then + fail('update events should use devicecode.support.priority_event') + end +end + + +function tests.test_service_does_not_handoff_public_manager_endpoint_to_generation() + local s = read_file('../src/services/update/service.lua') + if s:find('manager_rx = self._manager_ep', 1, true) then + fail('service must route manager requests through a private generation queue') + end +end + +function tests.test_generation_does_not_own_active_runtime_state() + local s = read_file('../src/services/update/generation.lua') + if s:find("require 'services.update.active_runtime'", 1, true) + or s:find('_active_runtime', 1, true) + then + fail('generation must not own active runtime state') + end +end + +function tests.test_service_owns_active_runtime_state() + local s = read_file('../src/services/update/service.lua') + if not s:find("require 'services.update.active_runtime'", 1, true) + or not s:find('_active_component = active_component', 1, true) + or not s:find('_active_runtime = active_component:state()', 1, true) + then + fail('service should own active runtime state') + end +end + +function tests.test_generation_is_composition_not_role_mixing() + local s = read_file('../src/services/update/generation.lua') + local forbidden = { + "services.update.manager_requests", + "devicecode.support.scoped_work", + "devicecode.support.request_owner", + "devicecode.support.priority_event", + "services.update.job_repository", + } + for _, needle in ipairs(forbidden) do + if s:find(needle, 1, true) then + fail('generation.lua should not directly depend on ' .. needle) + end + end + local required = { + "services.update.manager", + "services.update.generation_events", + } + for _, needle in ipairs(required) do + if not s:find(needle, 1, true) then + fail('generation.lua should compose ' .. needle) + end + end +end + +function tests.test_generation_events_is_the_priority_boundary() + local s = read_file('../src/services/update/generation_events.lua') + if not s:find("devicecode.support.priority_event", 1, true) then + fail('generation_events.lua should use support.priority_event') + end + if not s:find('ingest_terminal', 1, true) or not s:find('manager', 1, true) then + fail('generation_events.lua should name generation semantic sources') + end +end + +function tests.test_service_owns_durable_job_runtime() + local svc = read_file('../src/services/update/service.lua') + if not svc:find("services.update.job_runtime", 1, true) + or not svc:find('_jobs = jobs', 1, true) + then + fail('service.lua should own the durable job runtime') + end + local gen = read_file('../src/services/update/generation.lua') + if gen:find("services.update.job_runtime", 1, true) then + fail('generation.lua should consume a service-owned job projection, not own job_runtime') + end + if gen:find('save_job_op', 1, true) then + fail('generation.lua should not perform durable job saves inline') + end + local rt = read_file('../src/services/update/job_runtime.lua') + if not rt:find('save_job_op', 1, true) or not rt:find('job_transition_done', 1, true) then + fail('job_runtime.lua should own durable transition work and transition completions') + end +end + +function tests.test_manager_router_owns_request_scoped_work() + local gen = read_file('../src/services/update/generation.lua') + if gen:find('manager_requests', 1, true) or gen:find('request_owner', 1, true) then + fail('generation.lua should not implement manager request bodies') + end + local mgr = read_file('../src/services/update/manager.lua') + if not mgr:find('services.update.manager_requests', 1, true) + or not mgr:find('scoped_work.start', 1, true) + then + fail('manager.lua should own manager request routing into scoped work') + end +end + +function tests.test_active_completion_policy_is_extracted() + local gen = read_file('../src/services/update/generation.lua') + if gen:find('mark_awaiting_commit', 1, true) + or gen:find('mark_awaiting_return', 1, true) + or gen:find('mark_terminal', 1, true) + then + fail('generation.lua should not encode active job state transition details') + end + local policy = read_file('../src/services/update/active_policy.lua') + if not policy:find('mark_awaiting_commit', 1, true) + or not policy:find('apply_completion', 1, true) + then + fail('active_policy.lua should own active completion interpretation') + end +end + +function tests.test_manager_requests_do_not_start_active_work() + local s = read_file('../src/services/update/manager_requests.lua') + if s:find('start_active', 1, true) or s:find('start_gate', 1, true) or s:find('active lease required', 1, true) then + fail('manager request scopes must persist active intent, not start active work') + end +end + +function tests.test_job_runtime_exposes_immediate_admission_and_active_intent() + local s = read_file('../src/services/update/job_runtime.lua') + if not s:find('function Runtime:admit_transition', 1, true) then + fail('job_runtime should expose immediate admit_transition') + end + if s:find('submit_transition_op', 1, true) or s:find('function Runtime:transition_op', 1, true) then + fail('job_runtime should not expose transition Op compatibility helpers') + end + local body = s:match('function Runtime:admit_transition%(.+function Runtime:terminate') or '' + if body:find('op.guard', 1, true) or body:find('fibers.perform', 1, true) then + fail('admit_transition must allocate and admit immediately without waiting') + end + if not s:find('active_intent', 1, true) then + fail('job_runtime should persist active_intent') + end + if s:find('cancel_handle', 1, true) or s:find('cell.cancelled', 1, true) then + fail('caller cancellation must not cancel admitted job transitions') + end +end + +function tests.test_active_runtime_launches_active_work_from_job_runtime_state() + local svc = read_file('../src/services/update/service.lua') + if svc:find('launch_next_active_from_jobs', 1, true) or svc:find('route_active_completion_to_generation', 1, true) then + fail('service should not own active launch or route active completions to generation') + end + local rt = read_file('../src/services/update/active_runtime.lua') + if not rt:find('function Component:consider_jobs', 1, true) + or not rt:find('start_intent', 1, true) + or not rt:find('admit_transition', 1, true) + then + fail('active_runtime should launch persisted active intents and apply completions through job_runtime') + end + if rt:find('active_runner', 1, true) or rt:find('spec.runner', 1, true) then + fail('active_runtime must not expose an active_runner bypass') + end +end + +function tests.test_update_models_use_terminate_not_close() + local s = read_file('../src/services/update/model.lua') + if s:find('function Model:close', 1, true) then + fail('update model should expose terminate(reason), not close(reason)') + end + if not s:find('function Model:terminate', 1, true) then + fail('update model should expose terminate(reason)') + end +end + +function tests.test_update_observer_uses_terminate_not_close() + local s = read_file('../src/services/update/observe.lua') + if s:find('function Observer:close', 1, true) then + fail('observer should expose terminate(reason), not close(reason)') + end + if not s:find('function Observer:terminate', 1, true) then + fail('observer should expose terminate(reason)') + end +end + +function tests.test_update_artifact_lifetime_uses_canonical_terminate_ownership() + local s = read_file('../src/services/update/artifacts/lifetime.lua') + for _, needle in ipairs({ + 'close_now', + 'abandon_now', + 'abort_now', + 'release_now', + 'terminate_now', + 'cleanup_now', + 'mark_transferred', + 'cleanup_method', + }) do + if s:find(needle, 1, true) then + fail('artifact lifetime should not use legacy cleanup name: ' .. needle) + end + end + if not s:find('resource.owned', 1, true) then + fail('artifact lifetime should delegate ownership to devicecode.support.resource') + end +end + +function tests.test_update_service_code_finalises_request_owners_from_owning_scope() + local saw_finaliser = false + for _, path in ipairs(list_update_files()) do + local s = read_file(path) + if s:find('request_owner', 1, true) then + if s:find('owner:terminate(', 1, true) + or s:find('request_owner.new%([^\n]-%):terminate%s*%(') then + fail('update service code should not use request_owner:terminate(reason): ' .. path) + end + end + if s:find('finalise_unresolved', 1, true) then + saw_finaliser = true + end + end + if not saw_finaliser then + fail('update service code should finalise unresolved request owners from owning-scope finalisers') + end +end + +function tests.test_update_production_code_uses_scoped_work_not_direct_target_spawn() + for _, path in ipairs(list_update_files()) do + local s = read_file(path) + if s:find(':spawn(function', 1, true) then + fail('direct target-scope spawn found in update service code: ' .. path) + end + end +end + +function tests.test_create_job_does_not_use_artifact_resolver_compatibility() + for _, path in ipairs(list_update_files()) do + local s = read_file(path) + if s:find('services.update.artifacts.resolver', 1, true) + or s:find('services.update.artifacts.preflight', 1, true) + or s:find('payload.artifact_source', 1, true) + or s:find('artifact_preflight', 1, true) then + fail('update create-job artifact compatibility remains in: ' .. path) + end + end +end +function tests.test_update_service_generation_boundary_uses_events_not_callbacks() + local svc = read_file('../src/services/update/service.lua') + local gen = read_file('../src/services/update/generation.lua') + local mgr = read_file('../src/services/update/manager.lua') + for label, s in pairs({ service = svc, generation = gen, manager = mgr }) do + for _, needle in ipairs({ 'params.on_snapshot', 'params.on_model', '_on_snapshot', '_on_model', 'on_changed = function', 'ctx.on_changed', 'active_snapshot = function', 'ctx.snapshot()' }) do + if s:find(needle, 1, true) then + fail(label .. ' should use event ports rather than callback seam ' .. needle) + end + end + end + if not svc:find('generation_snapshot', 1, true) or not gen:find('events_tx', 1, true) then + fail('generation snapshots should be reported through service events') + end +end + + +function tests.test_bus_request_scoped_work_uses_caller_cancel_op() + local mgr = read_file('../src/services/update/manager.lua') + if not mgr:find('cancel_op = owner:caller_cancel_op()', 1, true) then + fail('update manager scoped request work should use caller_cancel_op') + end + if not mgr:find('request_owner = owner', 1, true) then + fail('update manager should pass its canonical request owner to request workers') + end + + local reqs = read_file('../src/services/update/manager_requests.lua') + if not reqs:find('params.request_owner', 1, true) then + fail('manager_requests should reuse the manager-owned request owner') + end + + local ingest = read_file('../src/services/update/ingest.lua') + if not ingest:find('owner = request_owner.new(req)', 1, true) then + fail('ingest should create request owners at queue admission') + end + if not ingest:find('cancel_op = entry.owner and entry.owner:caller_cancel_op()', 1, true) then + fail('ingest scoped work should use caller_cancel_op') + end + if not ingest:find('entry_abandoned', 1, true) then + fail('ingest should skip requests abandoned while queued') + end +end + +return tests diff --git a/tests/unit/update/test_artifact_store_update_adapters.lua b/tests/unit/update/test_artifact_store_update_adapters.lua new file mode 100644 index 00000000..2d96b3d2 --- /dev/null +++ b/tests/unit/update/test_artifact_store_update_adapters.lua @@ -0,0 +1,163 @@ +local fibers = require 'fibers' +local op = require 'fibers.op' +local runfibers = require 'tests.support.run_fibers' + +local store_bus = require 'services.update.artifacts.store_bus' +local component_backend = require 'services.update.backends.component' + +local T = {} + +local function assert_eq(a, b, msg) + if a ~= b then error(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a)), 2) end +end + +local function assert_true(v, msg) + if v ~= true then error(msg or ('expected true, got ' .. tostring(v)), 2) end +end + +function T.artifact_store_bus_unwraps_hal_reply_envelopes() + runfibers.run(function() + local sink = { terminated = 0 } + function sink:append_op(_) return op.always(true, nil) end + function sink:commit_op() return op.always({ ref = 'artifact-1' }, nil) end + function sink:terminate(reason) self.terminated = self.terminated + 1; self.reason = reason; return true, nil end + + local source = { read_count = 0 } + function source:read_chunk_op() self.read_count = self.read_count + 1; return op.always(nil, nil) end + + local artifact = {} + function artifact:describe() return { artifact_ref = 'artifact-1', size = 10 } end + function artifact:open_source_op() return op.always(true, source) end + + local conn = { + call_op = function(_, topic, payload) + local method = topic[5] + if method == 'create-sink' then + assert_eq(payload.policy, 'prefer_durable') + return op.always({ ok = true, reason = sink }, nil) + elseif method == 'open' then + assert_eq(payload.artifact_ref, 'artifact-1') + return op.always({ ok = true, reason = artifact }, nil) + elseif method == 'delete' then + return op.always({ ok = true, reason = nil }, nil) + elseif method == 'status' then + return op.always({ ok = true, reason = { available = true } }, nil) + end + return op.always({ ok = false, reason = 'bad method' }, nil) + end, + } + + local store = store_bus.new(conn) + local got_sink, sink_err = fibers.perform(store:create_sink_op({ meta = { component = 'mcu' }, policy = 'prefer_durable' })) + assert_eq(got_sink, sink, tostring(sink_err)) + + local got_source, source_err = fibers.perform(store:open_source_op('artifact-1')) + assert_eq(got_source, source, tostring(source_err)) + + local ok_delete, del_err = fibers.perform(store:delete_op('artifact-1')) + assert_true(ok_delete, tostring(del_err)) + + local status, st_err = fibers.perform(store:status_op()) + assert_true(status and status.available, tostring(st_err)) + end) +end + +function T.component_backend_commit_timeout_is_uncertain() + runfibers.run(function() + local conn = { + call_op = function(_, topic, payload) + assert_eq(topic[5], 'commit-update') + assert_eq(payload.commit_token, 'tok-1') + return op.always(nil, 'timeout') + end, + } + local backend = component_backend.new({ conn = conn, component = 'mcu', rpc_retry = { commit_attempts = 1 } }) + local result, err = fibers.perform(backend:commit_op({ job_id = 'job-1', component = 'mcu', expected_image_id = 'img-new' }, { commit_token = 'tok-1' })) + assert_eq(err, nil) + assert_true(result and result.accepted, 'timeout commit should be accepted as uncertain') + assert_true(result.uncertain, 'timeout commit should be marked uncertain') + end) +end + +function T.component_backend_commit_link_not_ready_is_not_uncertain() + runfibers.run(function() + local conn = { + call_op = function(_, topic, payload) + assert_eq(topic[5], 'commit-update') + return op.always(nil, 'link_not_ready') + end, + } + local backend = component_backend.new({ conn = conn, component = 'mcu', rpc_retry = { commit_attempts = 1 } }) + local result, err = fibers.perform(backend:commit_op({ job_id = 'job-1', component = 'mcu', expected_image_id = 'img-new' }, { commit_token = 'tok-1' })) + assert_eq(result, nil) + assert_eq(err, 'link_not_ready') + end) +end + +function T.component_backend_stage_op_runs_preflight_prepare_and_stage() + runfibers.run(function() + local source = {} + function source:read_chunk_op() return op.always(nil, nil) end + + local artifact = {} + function artifact:describe() + return { + artifact_ref = 'artifact-1', + size = 12, + digest_alg = 'xxhash32', + digest = 'abcd', + meta = { image_id = 'img-new', format = 'dcmcu-v1' }, + } + end + function artifact:open_source_op() + -- Exercise the direct source,err adapter shape as well as the HAL true,source shape. + return op.always(source, nil) + end + + local artifact_store = { + open_op = function(_, ref) + assert_eq(ref, 'artifact-1') + return op.always(artifact, nil) + end, + open_source_op = function(_, ref) + assert_eq(ref, 'artifact-1') + return op.always(source, nil) + end, + } + + local seen_payload + local seen_prepare + local conn = { + call_op = function(_, topic, payload) + assert_eq(topic[1], 'cap') + assert_eq(topic[2], 'component') + assert_eq(topic[4], 'rpc') + if topic[5] == 'prepare-update' then + seen_prepare = payload + assert_eq(payload.target, 'mcu') + return op.always({ ok = true }, nil) + end + if topic[5] == 'stage-update' then + seen_payload = payload + return op.always({ ok = true, public_status = 'succeeded', value = { transferred = true } }, nil) + end + return op.always({ ok = true }, nil) + end, + } + + local backend = component_backend.new({ conn = conn, artifact_store = artifact_store, component = 'mcu' }) + local job = { job_id = 'job-1', component = 'mcu', artifact_ref = 'artifact-1', expected_image_id = 'img-new', metadata = { format = 'dcmcu-v1' } } + + local staged, serr = fibers.perform(backend:stage_op(job, {})) + assert_eq(type(staged), 'table', tostring(serr)) + assert_true(staged.staged) + assert_eq(seen_prepare.target, 'mcu') + assert_eq(staged.preflight.size, 12) + assert_eq(staged.transfer.size, 12) + assert_eq(seen_payload.source, source) + assert_eq(seen_payload.size, 12) + assert_eq(seen_payload.digest, 'abcd') + end) +end + +return T diff --git a/tests/unit/update/test_bundled_probe.lua b/tests/unit/update/test_bundled_probe.lua new file mode 100644 index 00000000..b93f137f --- /dev/null +++ b/tests/unit/update/test_bundled_probe.lua @@ -0,0 +1,403 @@ +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' +local op = require 'fibers.op' +local bundled_probe = require 'services.update.bundled_probe' +local bundled_apply = require 'services.update.bundled_apply' +local bundled = require 'services.update.bundled' +local blob_source = require 'devicecode.blob_source' +local dcmcu_fixture = require 'tests.support.dcmcu_fixture' + +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v,msg) if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end end + + +local function import_store(ref, image_id) + return { + import_path_op = function (_, path, meta, opts) + return op.always({ artifact_ref = ref or 'artifact-1', path = path, meta = meta, policy = opts and opts.policy }, nil) + end, + open_source_op = function (_, artifact_ref) + assert_eq(artifact_ref, ref or 'artifact-1') + return op.always(blob_source.from_string(dcmcu_fixture.make(image_id or 'mcu-image-new')), nil) + end, + } +end + +local function fake_jobs() + local state = { jobs = {}, transitions = {} } + function state:ready_op() + return op.always(true, nil) + end + function state:get(job_id) + return self.jobs[job_id] + end + function state:admit_transition(cmd) + self.transitions[#self.transitions + 1] = cmd + local result + if cmd.kind == 'create_job' then + local payload = cmd.payload or {} + if self.jobs[payload.job_id] then + result = { status = 'rejected', reason = 'job_exists', job_id = payload.job_id } + else + local job = { + job_id = payload.job_id, + component = payload.component, + expected_image_id = payload.expected_image_id, + artifact_ref = payload.artifact_ref, + artifact = payload.artifact, + attempt = payload.attempt, + metadata = payload.metadata, + policy = payload.policy, + state = 'created', + next_step = 'start', + } + self.jobs[job.job_id] = job + result = { status = 'persisted', job_id = job.job_id, job = job } + end + elseif cmd.kind == 'start_job' then + local job = assert(self.jobs[cmd.job_id], 'start missing job') + local phase = cmd.phase or 'stage' + job.state = phase == 'commit' and 'committing' or 'staging' + job.active_intent = { phase = phase, state = 'pending' } + result = { status = 'persisted', job_id = job.job_id, phase = phase, job = job } + elseif cmd.kind == 'discard_job' then + local job = assert(self.jobs[cmd.job_id], 'discard missing job') + self.jobs[cmd.job_id] = nil + result = { status = 'persisted', job_id = cmd.job_id, job = job } + else + result = { status = 'rejected', reason = 'unsupported' } + end + return { + outcome_op = function () return op.always(result, nil) end, + }, nil + end + return state +end + +local function probe_store() + return { + probe_op = function (_, source) + return op.always({ identity = source.identity or 'desired-1' }, nil) + end, + } +end + +function tests.test_bundled_probe_reports_completion_without_inline_policy() + fibers.run(function (scope) + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local handle = assert(bundled_probe.start({ + lifetime_scope = scope, + report_scope = scope, + service_id = 'update', + generation = 7, + component = 'cm5', + artifact_store = probe_store(), + source = { identity = 'image-a' }, + done_tx = tx, + })) + assert_true(handle ~= nil) + local ev = fibers.perform(rx:recv_op()) + assert_eq(ev.kind, 'bundled_probe_done') + assert_eq(ev.status, 'ok') + assert_eq(ev.result.desired.identity, 'image-a') + end) +end + +function tests.test_bundled_coordinator_starts_probe_and_applies_stored_result() + fibers.run(function (scope) + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local co = bundled.new({ service_id = 'update', generation = 3 }) + assert(co:start_probe({ + lifetime_scope = scope, + report_scope = scope, + component = 'cm5', + artifact_store = probe_store(), + source = { identity = 'desired-cm5' }, + done_tx = tx, + })) + local ev = fibers.perform(rx:recv_op()) + assert_true(co:handle_probe_done(ev)) + local snap = co:snapshot() + assert_eq(snap.state.cm5, 'desired_known') + assert_eq(snap.desired.cm5.identity, 'desired-cm5') + end) +end + + +function tests.test_bundled_probe_imports_fixed_file_to_artifact_ref() + fibers.run(function (scope) + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + assert(bundled_probe.start({ + lifetime_scope = scope, + report_scope = scope, + service_id = 'update', + generation = 8, + component = 'mcu', + artifact_store = import_store('mcu-artifact'), + source = { kind = 'file', path = '/artifacts/mcu.dcmcu', policy = 'transient_only' }, + done_tx = tx, + })) + local ev = fibers.perform(rx:recv_op()) + assert_eq(ev.kind, 'bundled_probe_done') + assert_eq(ev.status, 'ok') + assert_eq(ev.result.artifact_ref, 'mcu-artifact') + assert_eq(ev.result.desired.expected_image_id, 'mcu-image-new') + assert_eq(ev.result.desired.path, '/artifacts/mcu.dcmcu') + assert_eq(ev.result.desired.policy, 'transient_only') + end) +end + +function tests.test_bundled_apply_creates_and_optionally_starts_normal_job() + fibers.run(function (scope) + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local jobs = fake_jobs() + assert(bundled_apply.start({ + lifetime_scope = scope, + report_scope = scope, + service_id = 'update', + generation = 9, + component = 'mcu', + jobs = jobs, + spec = { component = 'mcu', source = { metadata = { format = 'dcmcu-v1' } }, job = { start = 'auto', commit = 'auto' } }, + desired = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new', artifact = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new' } }, + done_tx = tx, + })) + local ev = fibers.perform(rx:recv_op()) + assert_eq(ev.kind, 'bundled_apply_done') + assert_eq(ev.status, 'ok') + assert_eq(ev.result.job_id, 'bundled-mcu') + assert_eq(jobs.jobs['bundled-mcu'].artifact_ref, 'mcu-artifact') + assert_eq(jobs.jobs['bundled-mcu'].expected_image_id, 'mcu-image-new') + assert_eq(jobs.jobs['bundled-mcu'].policy.commit, 'auto') + assert_eq(jobs.jobs['bundled-mcu'].attempt, 1) + assert_eq(jobs.jobs['bundled-mcu'].metadata.bundled_attempt, 1) + assert_eq(jobs.jobs['bundled-mcu'].state, 'staging') + assert_eq(#jobs.transitions, 2) + end) +end + +function tests.test_bundled_coordinator_probe_then_apply_policy() + fibers.run(function (scope) + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local jobs = fake_jobs() + local co = bundled.new({ + service_id = 'update', + generation = 10, + config = { + enabled = true, + components = { + mcu = { + component = 'mcu', + source = { kind = 'file', path = 'mcu.dcmcu' }, + job = { create_if = 'image_differs', start = 'auto', commit = 'auto', max_attempts = 3 }, + }, + }, + }, + }) + assert(co:start_missing_probes({ + lifetime_scope = scope, + report_scope = scope, + artifact_store = import_store('mcu-artifact'), + done_tx = tx, + })) + local probe_ev = fibers.perform(rx:recv_op()) + assert_true(co:handle_probe_done(probe_ev)) + assert(co:start_ready_applies({ + lifetime_scope = scope, + report_scope = scope, + jobs = jobs, + observer_snapshot = { by_id = { mcu = { state = { software = { image_id = 'mcu-image-old' } } } } }, + done_tx = tx, + })) + local apply_ev = fibers.perform(rx:recv_op()) + assert_true(co:handle_apply_done(apply_ev)) + local snap = co:snapshot() + assert_eq(snap.state.mcu, 'applied') + assert_eq(jobs.jobs['bundled-mcu'].state, 'staging') + assert_eq(jobs.jobs['bundled-mcu'].expected_image_id, 'mcu-image-new') + end) +end + + +function tests.test_bundled_apply_skips_when_running_image_matches_artifact() + fibers.run(function (scope) + local tx = mailbox.new(4, { full = 'reject_newest' }) + local jobs = fake_jobs() + local co = bundled.new({ service_id = 'update', generation = 11, config = { enabled = true, components = { mcu = { + component = 'mcu', + source = { kind = 'file', path = '/artifacts/mcu.dcmcu', policy = 'transient_only' }, + job = { create_if = 'image_differs', start = 'auto', commit = 'auto', max_attempts = 3 }, + } } } }) + assert_true(co:handle_probe_done({ + kind = 'bundled_probe_done', generation = 11, component = 'mcu', status = 'ok', + result = { desired = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new' } }, + })) + local started = assert(co:start_ready_applies({ + lifetime_scope = scope, + report_scope = scope, + jobs = jobs, + observer_snapshot = { by_id = { mcu = { state = { software = { image_id = 'mcu-image-new' } } } } }, + done_tx = tx, + })) + assert_eq(#started, 0) + assert_eq(co:snapshot().state.mcu, 'already_current') + assert_eq(#jobs.transitions, 0) + end) +end + +function tests.test_bundled_apply_waits_for_current_image_before_creating_job() + fibers.run(function (scope) + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local jobs = fake_jobs() + local co = bundled.new({ service_id = 'update', generation = 12, config = { enabled = true, components = { mcu = { + component = 'mcu', + source = { kind = 'file', path = '/artifacts/mcu.dcmcu', policy = 'transient_only' }, + job = { create_if = 'image_differs', start = 'auto', commit = 'auto', max_attempts = 3 }, + } } } }) + assert_true(co:handle_probe_done({ + kind = 'bundled_probe_done', generation = 12, component = 'mcu', status = 'ok', + result = { desired = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new' } }, + })) + local none = assert(co:start_ready_applies({ lifetime_scope = scope, report_scope = scope, jobs = jobs, observer_snapshot = {}, done_tx = tx })) + assert_eq(#none, 0) + assert_eq(co:snapshot().state.mcu, 'pending_current_image') + local started = assert(co:start_ready_applies({ + lifetime_scope = scope, + report_scope = scope, + jobs = jobs, + observer_snapshot = { by_id = { mcu = { state = { software = { image_id = 'mcu-image-old' } } } } }, + done_tx = tx, + })) + assert_eq(#started, 1) + local ev = fibers.perform(rx:recv_op()) + assert_true(co:handle_apply_done(ev)) + assert_eq(jobs.jobs['bundled-mcu'].state, 'staging') + end) +end + + +function tests.test_bundled_apply_supersedes_terminal_changed_image_job() + fibers.run(function (scope) + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local jobs = fake_jobs() + jobs.jobs['bundled-mcu'] = { job_id = 'bundled-mcu', component = 'mcu', expected_image_id = 'mcu-image-old', state = 'failed' } + assert(bundled_apply.start({ + lifetime_scope = scope, + report_scope = scope, + service_id = 'update', + generation = 13, + component = 'mcu', + jobs = jobs, + spec = { component = 'mcu', source = { metadata = { format = 'dcmcu-v1' } }, job = { job_id = 'bundled-mcu', start = 'manual', supersede = 'same_job_if_image_changed' } }, + desired = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new', artifact = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new' } }, + done_tx = tx, + })) + local ev = fibers.perform(rx:recv_op()) + assert_eq(ev.kind, 'bundled_apply_done') + assert_eq(ev.status, 'ok') + assert_eq(jobs.transitions[1].kind, 'discard_job') + assert_eq(jobs.transitions[2].kind, 'create_job') + assert_eq(jobs.jobs['bundled-mcu'].expected_image_id, 'mcu-image-new') + assert_eq(jobs.jobs['bundled-mcu'].state, 'created') + end) +end + + +function tests.test_bundled_apply_retries_terminal_same_image_until_attempt_limit() + fibers.run(function (scope) + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local jobs = fake_jobs() + jobs.jobs['bundled-mcu'] = { + job_id = 'bundled-mcu', component = 'mcu', expected_image_id = 'mcu-image-new', + state = 'failed', attempt = 1, metadata = { bundled_attempt = 1 }, + } + assert(bundled_apply.start({ + lifetime_scope = scope, + report_scope = scope, + service_id = 'update', + generation = 14, + component = 'mcu', + jobs = jobs, + spec = { component = 'mcu', source = { metadata = { format = 'dcmcu-v1' } }, job = { job_id = 'bundled-mcu', start = 'auto', commit = 'auto', max_attempts = 3, supersede = 'same_job_if_image_changed' } }, + desired = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new', artifact = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new' } }, + done_tx = tx, + })) + local ev = fibers.perform(rx:recv_op()) + assert_eq(ev.kind, 'bundled_apply_done') + assert_eq(ev.status, 'ok') + assert_eq(jobs.transitions[1].kind, 'discard_job') + assert_eq(jobs.transitions[2].kind, 'create_job') + assert_eq(jobs.transitions[3].kind, 'start_job') + assert_eq(jobs.jobs['bundled-mcu'].expected_image_id, 'mcu-image-new') + assert_eq(jobs.jobs['bundled-mcu'].attempt, 2) + assert_eq(jobs.jobs['bundled-mcu'].metadata.bundled_attempt, 2) + assert_eq(jobs.jobs['bundled-mcu'].policy.attempt, 2) + assert_eq(jobs.jobs['bundled-mcu'].state, 'staging') + end) +end + + +function tests.test_bundled_apply_resets_attempts_after_previous_success_for_same_image() + fibers.run(function (scope) + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local jobs = fake_jobs() + jobs.jobs['bundled-mcu'] = { + job_id = 'bundled-mcu', component = 'mcu', expected_image_id = 'mcu-image-new', + state = 'succeeded', attempt = 30, metadata = { bundled_attempt = 30 }, + } + assert(bundled_apply.start({ + lifetime_scope = scope, + report_scope = scope, + service_id = 'update', + generation = 16, + component = 'mcu', + jobs = jobs, + spec = { component = 'mcu', source = { metadata = { format = 'dcmcu-v1' } }, job = { job_id = 'bundled-mcu', start = 'auto', commit = 'auto', max_attempts = 30, supersede = 'same_job_if_image_changed' } }, + desired = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new', artifact = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new' } }, + done_tx = tx, + })) + local ev = fibers.perform(rx:recv_op()) + assert_eq(ev.kind, 'bundled_apply_done') + assert_eq(ev.status, 'ok') + assert_eq(jobs.transitions[1].kind, 'discard_job') + assert_eq(jobs.transitions[2].kind, 'create_job') + assert_eq(jobs.transitions[3].kind, 'start_job') + assert_eq(jobs.jobs['bundled-mcu'].expected_image_id, 'mcu-image-new') + assert_eq(jobs.jobs['bundled-mcu'].attempt, 1) + assert_eq(jobs.jobs['bundled-mcu'].metadata.bundled_attempt, 1) + assert_eq(jobs.jobs['bundled-mcu'].policy.attempt, 1) + assert_eq(jobs.jobs['bundled-mcu'].state, 'staging') + end) +end + +function tests.test_bundled_apply_stops_terminal_same_image_after_attempt_limit() + fibers.run(function (scope) + local tx, rx = mailbox.new(4, { full = 'reject_newest' }) + local jobs = fake_jobs() + jobs.jobs['bundled-mcu'] = { + job_id = 'bundled-mcu', component = 'mcu', expected_image_id = 'mcu-image-new', + state = 'failed', attempt = 3, metadata = { bundled_attempt = 3 }, + } + assert(bundled_apply.start({ + lifetime_scope = scope, + report_scope = scope, + service_id = 'update', + generation = 15, + component = 'mcu', + jobs = jobs, + spec = { component = 'mcu', source = { metadata = { format = 'dcmcu-v1' } }, job = { job_id = 'bundled-mcu', start = 'auto', commit = 'auto', max_attempts = 3, supersede = 'same_job_if_image_changed' } }, + desired = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new', artifact = { artifact_ref = 'mcu-artifact', expected_image_id = 'mcu-image-new' } }, + done_tx = tx, + })) + local ev = fibers.perform(rx:recv_op()) + assert_eq(ev.kind, 'bundled_apply_done') + assert_eq(ev.status, 'failed') + assert_eq(ev.primary, 'bundled_retry_exhausted:3/3') + assert_eq(#jobs.transitions, 0) + assert_eq(jobs.jobs['bundled-mcu'].state, 'failed') + end) +end + +return tests diff --git a/tests/unit/update/test_config.lua b/tests/unit/update/test_config.lua new file mode 100644 index 00000000..82798fbf --- /dev/null +++ b/tests/unit/update/test_config.lua @@ -0,0 +1,167 @@ +-- tests/unit/update/test_config.lua + +local config = require 'services.update.config' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil value') end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end +local function assert_contains(s, needle, msg) + if tostring(s or ''):find(tostring(needle), 1, true) == nil then + fail(msg or ('expected ' .. tostring(s) .. ' to contain ' .. tostring(needle))) + end +end + +function tests.test_default_config_normalises() + local cfg, err = config.normalise({}) + assert_not_nil(cfg, err) + assert_eq(cfg.schema, config.SCHEMA) + assert_eq(cfg.service_id, 'update') + assert_eq(cfg.namespace, 'default') + assert_eq(cfg.summary.component_count, 0) + assert_true(cfg.publish.enabled) +end + +function tests.test_components_array_is_normalised_to_map() + local cfg, err = config.normalise({ + components = { + { component = 'cm5', backend = 'swupdate', stage_timeout_s = '12.5' }, + { component = 'mcu', backend = 'mcu' }, + }, + }) + assert_not_nil(cfg, err) + assert_eq(cfg.components.cm5.component, 'cm5') + assert_eq(cfg.components.cm5.stage_timeout_s, 12.5) + assert_eq(cfg.components.mcu.component, 'mcu') + assert_eq(cfg.summary.component_count, 2) +end + + +function tests.test_components_array_requires_component_identifier_field() + local cfg, err = config.normalise({ + components = { + { id = 'cm5', backend = 'swupdate' }, + }, + }) + if cfg ~= nil then fail('expected id alias to be rejected') end + assert_not_nil(err) +end + +function tests.test_unsupported_schema_is_rejected() + local cfg, err = config.normalise({ schema = 'wrong/schema' }) + if cfg ~= nil then fail('expected config to be rejected') end + assert_not_nil(err) +end + +function tests.test_component_phase_timeouts_must_be_positive_numbers() + local cfg, err = config.normalise({ + components = { + mcu = { component = 'mcu', stage_timeout_s = 0 }, + }, + }) + if cfg ~= nil then fail('expected zero stage timeout to be rejected') end + assert_not_nil(err) + assert_true(tostring(err):find('stage_timeout_s', 1, true) ~= nil, err) +end + +function tests.test_extract_payload_accepts_config_service_shape() + local raw, rev = config.extract_payload({ rev = 7, data = { schema = config.SCHEMA, namespace = 'prod' } }) + assert_eq(rev, 7) + assert_eq(raw.namespace, 'prod') +end + +function tests.test_retention_policy_normalises() + local cfg, err = config.normalise({ + retention = { + prune_on_startup = true, + terminal_max_count = '50', + terminal_max_age_s = '604800', + }, + }) + assert_not_nil(cfg, err) + assert_true(cfg.retention.prune_on_startup) + assert_eq(cfg.retention.mode, 'single_job') + assert_eq(cfg.retention.max_jobs, 1) + assert_true(cfg.retention.scrub_legacy_on_startup) + assert_eq(cfg.retention.terminal_max_count, 50) + assert_eq(cfg.retention.terminal_max_age_s, 604800) +end + + +function tests.test_bundled_job_policy_normalises() + local cfg, err = config.normalise({ + bundled = { + enabled = true, + max_attempts = 7, + components = { + mcu = { + component = 'mcu', + source = { kind = 'file', path = '/artifacts/mcu.dcmcu', policy = 'transient_only' }, + job = { + job_id = 'bundled-mcu', + create_if = 'image_differs', + start = 'auto', + commit = 'auto', + reconcile = 'required', + supersede = 'same_job_if_image_changed', + max_attempts = 5, + }, + }, + }, + }, + }) + assert_not_nil(cfg, err) + assert_eq(cfg.bundled.max_attempts, 7) + local mcu = cfg.bundled.components.mcu + assert_eq(mcu.job.job_id, 'bundled-mcu') + assert_eq(mcu.job.create_if, 'image_differs') + assert_eq(mcu.job.start, 'auto') + assert_eq(mcu.job.commit, 'auto') + assert_eq(mcu.job.reconcile, 'required') + assert_eq(mcu.job.supersede, 'same_job_if_image_changed') + assert_eq(mcu.job.max_attempts, 5) + assert_nil(mcu.auto_create) + assert_nil(mcu.auto_start) +end + +function tests.test_bundled_legacy_auto_flags_map_to_job_policy() + local cfg, err = config.normalise({ + bundled = { + enabled = true, + max_attempts = 9, + components = { + mcu = { + component = 'mcu', + source = { kind = 'file', path = '/artifacts/mcu.dcmcu' }, + auto_create = true, + auto_start = true, + }, + }, + }, + }) + assert_not_nil(cfg, err) + local mcu = cfg.bundled.components.mcu + assert_eq(mcu.job.job_id, 'bundled-mcu') + assert_eq(mcu.job.create_if, 'always') + assert_eq(mcu.job.start, 'auto') + assert_eq(mcu.job.commit, 'manual') + assert_eq(mcu.job.max_attempts, 9) + assert_nil(mcu.auto_create) + assert_nil(mcu.auto_start) +end + +function tests.test_bundled_enabled_requires_attempt_budget() + local cfg, err = config.normalise({ + bundled = { + enabled = true, + components = {}, + }, + }) + assert_nil(cfg) + assert_contains(err, 'bundled.max_attempts') +end + +return tests diff --git a/tests/unit/update/test_dcmcu_artifact_identity.lua b/tests/unit/update/test_dcmcu_artifact_identity.lua new file mode 100644 index 00000000..cce0e2cb --- /dev/null +++ b/tests/unit/update/test_dcmcu_artifact_identity.lua @@ -0,0 +1,34 @@ +local fibers = require 'fibers' +local blob_source = require 'devicecode.blob_source' +local dcmcu = require 'services.update.artifacts.dcmcu' +local fixture = require 'tests.support.dcmcu_fixture' + +local T = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil') end end + +function T.parses_dcmcu_v1_identity_from_source() + fibers.run(function () + local raw = fixture.make('mcu-image-new', 'abc', { version = '15.0', build_id = 'build-15' }) + local identity, err = fibers.perform(dcmcu.identity_from_source_op(blob_source.from_string(raw))) + assert_not_nil(identity, tostring(err)) + assert_eq(identity.format, 'dcmcu-v1') + assert_eq(identity.component, 'mcu') + assert_eq(identity.image_id, 'mcu-image-new') + assert_eq(identity.version, '15.0') + assert_eq(identity.build_id, 'build-15') + assert_eq(identity.payload_length, 3) + end) +end + +function T.rejects_non_dcmcu_payload() + fibers.run(function () + local identity, err = fibers.perform(dcmcu.identity_from_source_op(blob_source.from_string('not a dcmcu'))) + assert_eq(identity, nil) + assert_not_nil(err) + end) +end + +return T diff --git a/tests/unit/update/test_events.lua b/tests/unit/update/test_events.lua new file mode 100644 index 00000000..0f367236 --- /dev/null +++ b/tests/unit/update/test_events.lua @@ -0,0 +1,80 @@ +-- tests/unit/update/test_events.lua + +local fibers = require 'fibers' +local mailbox = require 'fibers.mailbox' + +local events = require 'services.update.events' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end + +function tests.test_next_event_prefers_generation_done_over_manager_when_both_ready() + fibers.run(function () + local done_tx, done_rx = mailbox.new(4, { full = 'reject_newest' }) + local mgr_tx, mgr_rx = mailbox.new(4, { full = 'reject_newest' }) + + done_tx:send({ kind = 'generation_done', generation = 1 }) + mgr_tx:send({ fake = 'request' }) + + local ev = fibers.perform(events.next_service_event_op({ + pending = {}, + done_rx = done_rx, + manager_rx = mgr_rx, + })) + + assert_eq(ev.kind, 'generation_done') + end) +end + +function tests.test_next_event_rechecks_priority_after_wake() + fibers.run(function (scope) + local done_tx, done_rx = mailbox.new(4, { full = 'reject_newest' }) + local mgr_tx, mgr_rx = mailbox.new(4, { full = 'reject_newest' }) + + local state = { + pending = {}, + done_rx = done_rx, + manager_rx = mgr_rx, + } + + local seen + local ok, err = scope:spawn(function () + seen = fibers.perform(events.next_service_event_op(state)) + end) + if not ok then fail(err) end + + -- Wake via manager, then make a higher-priority completion ready before + -- the selector commits. The helper stores the wake and re-runs priority. + mgr_tx:send({ fake = 'request' }) + done_tx:send({ kind = 'generation_done', generation = 2 }) + + while seen == nil do fibers.perform(require('fibers.sleep').sleep_op(0.001)) end + assert_eq(seen.kind, 'generation_done') + assert_eq(seen.generation, 2) + end) +end + +function tests.test_map_config_event_accepts_shared_config_watch_shape() + local ev = events.map_config_event({ + kind = 'config_changed', + rev = 7, + generation = 3, + record = { rev = 7, data = { namespace = 'shared' } }, + msg = { origin = { kind = 'local' } }, + }) + assert_eq(ev.kind, 'config_changed') + assert_eq(ev.rev, 7) + assert_eq(ev.generation, 3) + assert_eq(ev.payload.data.namespace, 'shared') + assert_eq(ev.origin.kind, 'local') +end + +function tests.test_map_config_event_accepts_shared_config_watch_closed_shape() + local ev = events.map_config_event({ kind = 'config_watch_closed', err = 'closed_for_test' }) + assert_eq(ev.kind, 'config_watch_closed') + assert_eq(ev.err, 'closed_for_test') +end + +return tests diff --git a/tests/unit/update/test_generation_refactor.lua b/tests/unit/update/test_generation_refactor.lua new file mode 100644 index 00000000..007d9c35 --- /dev/null +++ b/tests/unit/update/test_generation_refactor.lua @@ -0,0 +1,512 @@ +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local cond = require 'fibers.cond' +local op = require 'fibers.op' +local mailbox = require 'fibers.mailbox' +local busmod = require 'bus' + +local service = require 'services.update.service' +local topics = require 'services.update.topics' +local store_mod = require 'services.update.job_store_memory' +local active_runtime = require 'services.update.active_runtime' +local probe = require 'tests.support.bus_probe' +local queue = require 'devicecode.support.queue' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil value') end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end + +local function start_service(root_scope, params) + local bus = busmod.new() + local svc_conn = bus:connect() + local caller = bus:connect() + local cfg_conn = bus:connect() + local child = assert(root_scope:child()) + + local ok, err = child:spawn(function (scope) + params = params or {} + params.conn = svc_conn + params.service_id = params.service_id or 'update' + if params.job_store == nil and params.job_store_kind == nil then + params.job_store_kind = 'memory' + end + service.run(scope, params) + end) + assert_true(ok, err) + + fibers.perform(sleep.sleep_op(0.02)) + return child, caller, cfg_conn, bus +end + + +local function wait_job_visible(caller, job_id, timeout) + local status + local ok = probe.wait_until(function () + status = caller:call(topics.update_manager_rpc('status'), {}, { timeout = 0.05 }) + return status and status.snapshot and status.snapshot.jobs and status.snapshot.jobs.by_id[job_id] ~= nil + end, { timeout = timeout or 0.5, interval = 0.01 }) + return ok, status +end + +local function config_payload(namespace) + return { + rev = namespace == 'ns2' and 2 or 1, + data = { + schema = 'devicecode.update/1', + namespace = namespace or 'ns1', + components = { + { component = 'cm5' }, + }, + }, + } +end + +local function blocking_store_for_create() + local gate = cond.new() + local state = { + save_started = false, + saved = {}, + } + + local backend = {} + function backend:load_all_op() + return op.always({ jobs = {}, order = {} }, nil) + end + function backend:save_job_op(job) + state.save_started = true + state.pending_job = job + return gate:wait_op():wrap(function () + state.saved[job.job_id] = job + return true, nil + end) + end + + state.gate = gate + return backend, state +end + +function tests.test_admitted_old_generation_create_completion_remains_durable_after_replacement() + fibers.run(function (root_scope) + local store, store_state = blocking_store_for_create() + local child, caller, cfg_conn = start_service(root_scope, { + job_store = store, + config = config_payload('ns1'), + }) + + local reply, reply_err + local ok_spawn = root_scope:spawn(function () + reply, reply_err = caller:call(topics.update_manager_rpc('create-job'), { + job_id = 'old-job', + component = 'cm5', + artifact_ref = 'artifact-old-job', + }, { timeout = 1.0 }) + end) + assert_true(ok_spawn) + + assert_true(probe.wait_until(function () return store_state.save_started end, { timeout = 0.5, interval = 0.01 }), 'expected old create to reach store save') + + cfg_conn:retain(topics.config(), config_payload('ns2')) + + assert_true(probe.wait_until(function () return reply_err ~= nil end, { timeout = 0.8, interval = 0.01 }), 'expected old request to be finalised by generation cancellation') + assert_eq(reply, nil) + assert_eq(reply_err, 'config_changed') + + store_state.gate:signal() + fibers.perform(sleep.sleep_op(0.05)) + + local status = assert(caller:call(topics.update_manager_rpc('status'), {}, { timeout = 0.2 })) + assert_eq(status.snapshot.config.namespace, 'ns2') + assert_not_nil(status.snapshot.jobs.by_id['old-job'], 'admitted durable transition should still apply after generation replacement') + assert_eq(status.snapshot.jobs.by_id['old-job'].state, 'created') + + child:cancel('test complete') + end) +end + +function tests.test_generation_replacement_does_not_cancel_accepted_active_work() + fibers.run(function (root_scope) + local active_started = cond.new() + local active_finalised = false + local active_finalised_cond = cond.new() + local release_active = cond.new() + + local backend = {} + function backend:stage_op(job) + active_started:signal() + return release_active:wait_op():wrap(function () + active_finalised = true + active_finalised_cond:signal() + return { job_id = job.job_id } + end) + end + + local child, caller, cfg_conn = start_service(root_scope, { + config = config_payload('ns1'), + backend = backend, + }) + + assert(caller:call(topics.update_manager_rpc('create-job'), { job_id = 'j1', component = 'cm5', artifact_ref = 'artifact-j1' }, { timeout = 0.5 })) + fibers.perform(sleep.sleep_op(0.05)) + assert(caller:call(topics.update_manager_rpc('start-job'), { job_id = 'j1' }, { timeout = 0.5 })) + fibers.perform(active_started:wait_op()) + + cfg_conn:retain(topics.config(), config_payload('ns2')) + fibers.perform(sleep.sleep_op(0.05)) + assert_eq(active_finalised, false, 'accepted active work must be service-owned, not cancelled by generation replacement') + + release_active:signal() + fibers.perform(active_finalised_cond:wait_op()) + assert_eq(active_finalised, true) + + local status + assert_true(probe.wait_until(function () + status = caller:call(topics.update_manager_rpc('status'), {}, { timeout = 0.05 }) + return status + and status.snapshot + and status.snapshot.config + and status.snapshot.config.namespace == 'ns2' + and status.snapshot.jobs + and status.snapshot.jobs.by_id + and status.snapshot.jobs.by_id.j1 + and status.snapshot.jobs.by_id.j1.state == 'awaiting_commit' + end, { timeout = 0.8, interval = 0.01 }), 'expected current generation to observe active completion') + + child:cancel('test complete') + end) +end + +function tests.test_start_job_request_finalises_on_generation_cancellation() + fibers.run(function (root_scope) + local gate = cond.new() + local start_save_seen = false + local backend = { jobs = {} } + + function backend:load_all_op() + return op.always({ jobs = self.jobs, order = {} }, nil) + end + function backend:save_job_op(job) + if job.state == 'staging' then + start_save_seen = true + return gate:wait_op():wrap(function () + self.jobs[job.job_id] = job + return true, nil + end) + end + self.jobs[job.job_id] = job + return op.always(true, nil) + end + + local child, caller, cfg_conn = start_service(root_scope, { + config = config_payload('ns1'), + job_store = backend, + }) + + assert(caller:call(topics.update_manager_rpc('create-job'), { job_id = 'j1', component = 'cm5', artifact_ref = 'artifact-j1' }, { timeout = 0.5 })) + fibers.perform(sleep.sleep_op(0.05)) + + local reply, reply_err + assert_true(root_scope:spawn(function () + reply, reply_err = caller:call(topics.update_manager_rpc('start-job'), { job_id = 'j1' }, { timeout = 1.0 }) + end)) + + assert_true(probe.wait_until(function () return start_save_seen end, { timeout = 0.5, interval = 0.01 }), 'expected start-job save to be in progress') + cfg_conn:retain(topics.config(), config_payload('ns2')) + + assert_true(probe.wait_until(function () return reply_err ~= nil end, { timeout = 0.8, interval = 0.01 }), 'expected start request to be finalised') + assert_nil(reply) + assert_eq(reply_err, 'config_changed') + + gate:signal() + child:cancel('test complete') + end) +end + +function tests.test_accepted_start_is_saved_durably_before_reply() + fibers.run(function (root_scope) + local save_log = {} + local backend = { jobs = {} } + + function backend:load_all_op() + return op.always({ jobs = self.jobs, order = {} }, nil) + end + function backend:save_job_op(job) + save_log[#save_log + 1] = { job_id = job.job_id, state = job.state } + self.jobs[job.job_id] = job + return op.always(true, nil) + end + + local child, caller = start_service(root_scope, { + config = config_payload('ns1'), + job_store = backend, + backend = { stage_op = function (_, job) return op.always({ job_id = job.job_id }, nil) end }, + }) + + assert(caller:call(topics.update_manager_rpc('create-job'), { job_id = 'j1', component = 'cm5', artifact_ref = 'artifact-j1' }, { timeout = 0.5 })) + fibers.perform(sleep.sleep_op(0.05)) + local reply = assert(caller:call(topics.update_manager_rpc('start-job'), { job_id = 'j1' }, { timeout = 0.5 })) + assert_eq(reply.accepted, true) + + local saw_staging_save = false + for _, row in ipairs(save_log) do + if row.job_id == 'j1' and row.state == 'staging' then + saw_staging_save = true + end + end + assert_eq(saw_staging_save, true) + + child:cancel('test complete') + end) +end + +function tests.test_active_completion_and_ready_start_cannot_double_own_slot() + fibers.run(function (root_scope) + local run_count = 0 + local backend = {} + function backend:stage_op(job) + run_count = run_count + 1 + return op.always({ job_id = job.job_id }, nil) + end + + local child, caller = start_service(root_scope, { + config = config_payload('ns1'), + backend = backend, + }) + + assert(caller:call(topics.update_manager_rpc('create-job'), { job_id = 'j1', component = 'cm5', artifact_ref = 'artifact-j1' }, { timeout = 0.5 })) + assert(caller:call(topics.update_manager_rpc('start-job'), { job_id = 'j1' }, { timeout = 0.5 })) + assert_true(probe.wait_until(function () return run_count >= 1 end, { timeout = 2.0, interval = 0.02 }), 'expected first job to run') + + assert(caller:call(topics.update_manager_rpc('create-job'), { job_id = 'j2', component = 'cm5', artifact_ref = 'artifact-j2' }, { timeout = 0.5 })) + local reply, err + assert_true(probe.wait_until(function () + reply, err = caller:call(topics.update_manager_rpc('start-job'), { job_id = 'j2' }, { timeout = 0.05 }) + return reply and reply.accepted == true + end, { timeout = 2.0, interval = 0.02 }), err or 'expected second start to be admitted after first completion') + + assert_true(probe.wait_until(function () return run_count >= 2 end, { timeout = 2.0, interval = 0.02 }), 'expected both sequential jobs to run') + + child:cancel('test complete') + end) +end + + +function tests.test_generation_event_selector_rechecks_completion_after_manager_wake() + local priority_event = require 'devicecode.support.priority_event' + + local function staged_rx(name, item, bus, wakes_done) + local rx = { + _name = name, + _item = item, + _consumed = false, + _closed = false, + _why = nil, + } + + function rx:why() + return self._why + end + + function rx:recv_op() + return op.new_primitive(nil, function () + if self._consumed then + return false + end + + if self._closed then + return true, nil + end + + if bus.ready[self._name] then + self._consumed = true + self._closed = true + self._why = 'closed' + return true, self._item + end + + return false + end, function (suspension, wrap_fn) + if wakes_done and not bus.fired then + bus.fired = true + bus.ready.done = true + bus.ready.manager = true + self._consumed = true + self._closed = true + self._why = 'closed' + if suspension:waiting() then + suspension:complete(wrap_fn, self._item) + end + end + end) + end + + return rx + end + + fibers.run(function () + local bus = { ready = {}, fired = false } + local pending = {} + local done_rx = staged_rx('done', { kind = 'active_job_done', job_id = 'j1' }, bus, false) + local manager_rx = staged_rx('manager', { id = 'start-j2' }, bus, true) + + local function map_done(ev) + if ev == nil then return { kind = 'completion_queue_closed' } end + return ev + end + + local function map_manager(req) + if req == nil then return { kind = 'manager_closed' } end + return { kind = 'manager_request', request = req } + end + + local function try_recv_now(rx, map) + local item = queue.try_now(rx:recv_op(), NOT_READY) + if item == NOT_READY then return nil end + return map(item) + end + + local function next_event_op() + return priority_event.sources_op { + label = 'update.generation.next_event.test', + pending = pending, + sources = { + { + name = 'done', + try_now = function () return try_recv_now(done_rx, map_done) end, + recv_op = function () return done_rx:recv_op():wrap(map_done) end, + }, + { + name = 'manager', + try_now = function () return try_recv_now(manager_rx, map_manager) end, + recv_op = function () return manager_rx:recv_op():wrap(map_manager) end, + }, + }, + } + end + + local first = fibers.perform(next_event_op()) + assert_eq(first.kind, 'active_job_done') + assert_eq(first.job_id, 'j1') + + local second = fibers.perform(next_event_op()) + assert_eq(second.kind, 'manager_request') + assert_eq(second.request.id, 'start-j2') + end) +end + +function tests.test_report_failure_does_not_lose_stored_active_completion() + fibers.run(function (root_scope) + local state = active_runtime.new_state() + local done_tx = mailbox.new(0, { full = 'reject_newest' }) + local report_scope = assert(root_scope:child()) + local lease = assert(active_runtime.claim(state, { job_id = 'j1', generation = 1, phase = 'stage' })) + + local handle = assert(active_runtime.start_work(root_scope, state, { + lease = lease, + done_tx = done_tx, + report_scope = report_scope, + job = { job_id = 'j1', component = 'cm5', artifact_ref = 'artifact-j1' }, + backend = { stage_op = function () return op.always({ ok = true }, nil) end }, + })) + + local ev = fibers.perform(handle:outcome_op()) + assert_eq(ev.kind, 'active_job_done') + assert_eq(ev.status, 'ok') + assert_eq(ev.job_id, 'j1') + + report_scope:cancel('test complete') + end) +end + + +function tests.test_active_completion_starts_durable_job_save() + fibers.run(function (root_scope) + local save_log = {} + local backend = { jobs = {} } + + function backend:load_all_op() + return op.always({ jobs = self.jobs, order = {} }, nil) + end + + function backend:save_job_op(job) + save_log[#save_log + 1] = { job_id = job.job_id, state = job.state } + self.jobs[job.job_id] = job + return op.always(true, nil) + end + + local child, caller = start_service(root_scope, { + config = config_payload('ns1'), + job_store = backend, + backend = { stage_op = function (_, job) return op.always({ job_id = job.job_id }, nil) end }, + }) + + assert(caller:call(topics.update_manager_rpc('create-job'), { job_id = 'j1', component = 'cm5', artifact_ref = 'artifact-j1' }, { timeout = 0.5 })) + fibers.perform(sleep.sleep_op(0.05)) + assert(caller:call(topics.update_manager_rpc('start-job'), { job_id = 'j1' }, { timeout = 0.5 })) + + local ok_wait = probe.wait_until(function () + for _, row in ipairs(save_log) do + if row.job_id == 'j1' and row.state == 'awaiting_commit' then + return true + end + end + return false + end, { timeout = 0.5, interval = 0.01 }) + + assert_true(ok_wait, 'expected active completion to trigger durable awaiting_commit save') + child:cancel('test complete') + end) +end + +function tests.test_service_owns_active_completion_persistence_after_generation_replacement() + fibers.run(function (root_scope) + local saves = {} + local store = { + load_all_op = function () return op.always({ jobs = {}, order = {} }, nil) end, + save_job_op = function (_, job) + saves[#saves + 1] = job + return op.always(true, nil) + end, + } + local release_active = cond.new() + local active_started = cond.new() + local backend = {} + function backend:stage_op() + active_started:signal() + return release_active:wait_op():wrap(function () + return { accepted = true } + end) + end + + local child, caller, cfg_conn = start_service(root_scope, { + job_store = store, + config = config_payload('ns1'), + backend = backend, + }) + + assert(caller:call(topics.update_manager_rpc('create-job'), { job_id = 'persist-active', component = 'cm5', artifact_ref = 'artifact-persist-active' }, { timeout = 0.5 })) + fibers.perform(sleep.sleep_op(0.05)) + assert(caller:call(topics.update_manager_rpc('start-job'), { job_id = 'persist-active' }, { timeout = 0.5 })) + fibers.perform(active_started:wait_op()) + + cfg_conn:retain(topics.config(), config_payload('ns2')) + fibers.perform(sleep.sleep_op(0.05)) + release_active:signal() + + assert_true(probe.wait_until(function () + for _, job in ipairs(saves) do + if job.job_id == 'persist-active' and job.state == 'awaiting_commit' then + return true + end + end + return false + end, { timeout = 0.8, interval = 0.01 }), 'expected service-owned active completion save after generation replacement') + + child:cancel('test complete') + end) +end + +return tests diff --git a/tests/unit/update/test_ingest_artifacts.lua b/tests/unit/update/test_ingest_artifacts.lua new file mode 100644 index 00000000..3c7388b6 --- /dev/null +++ b/tests/unit/update/test_ingest_artifacts.lua @@ -0,0 +1,322 @@ +local fibers = require 'fibers' +local op = require 'fibers.op' +local cond = require 'fibers.cond' +local mailbox = require 'fibers.mailbox' +local ingest = require 'services.update.ingest' +local lifetime = require 'services.update.artifacts.lifetime' + +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v,msg) if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end end + +local function new_sink() + local sink = { terminated = 0, appended = {}, committed = false } + function sink:append_op(chunk) + self.appended[#self.appended + 1] = chunk + return op.always(true, nil) + end + function sink:commit_op() + self.committed = true + return op.always({ ref = 'artifact-1' }, nil) + end + function sink:terminate(reason) + self.terminated = self.terminated + 1 + self.reason = reason + return true, nil + end + return sink +end + +function tests.test_ingest_scope_terminates_uncommitted_sink_from_finaliser() + local sink = new_sink() + fibers.run(function () + local st, rep, snap = fibers.run_scope(function (scope) + local inst = assert(ingest.new_instance(scope, { + ingest_id = 'i1', + component = 'cm5', + sink = sink, + })) + return inst:snapshot() + end) + assert_eq(st, 'ok') + assert_eq(snap.ingest_id, 'i1') + end) + assert_eq(sink.terminated, 1) +end + +function tests.test_ingest_commit_transfers_sink_cleanup_ownership() + local sink = new_sink() + fibers.run(function () + local st, rep, result = fibers.run_scope(function (scope) + local inst = assert(ingest.new_instance(scope, { + ingest_id = 'i1', + component = 'cm5', + sink = sink, + })) + local appended = assert(fibers.perform(assert(inst:append_op('abc')))) + assert_eq(appended.bytes, 3) + assert(inst:begin_commit()) + return assert(inst:commit_worker(scope)) + end) + assert_eq(st, 'ok') + assert_eq(result.artifact.ref, 'artifact-1') + end) + assert_true(sink.committed) + assert_eq(sink.terminated, 0) +end + +function tests.test_artifact_lifetime_terminate_uses_immediate_terminate() + local sink = new_sink() + fibers.run(function (scope) + local owned = assert(lifetime.own(scope, sink)) + local ok = assert(owned:terminate('manual')) + assert_eq(ok, true) + end) + assert_eq(sink.terminated, 1) + assert_eq(sink.reason, 'manual') +end + + +local function new_req(payload) + return { + _update_method = payload and payload.method, + payload = payload, + reply_value = nil, + fail_reason = nil, + reply = function(self, v) self.reply_value = v; return true end, + fail = function(self, r) self.fail_reason = r; return true end, + } +end + +function tests.test_ingest_terminal_request_is_selected_before_append() + fibers.run(function (scope) + local state = ingest.new_state(scope, { queue_len = 4 }) + local append_req = new_req({ method='ingest_append', ingest_id='i1', chunk='abc' }) + local commit_req = new_req({ method='ingest_commit', ingest_id='i1' }) + assert(state:submit(append_req)) + assert(state:submit(commit_req)) + local ev = state:try_terminal_now() + assert_eq(ev.kind, 'ingest_request') + assert_eq(ev.request, commit_req) + local ev2 = state:try_request_now() + assert_eq(ev2.request, append_req) + end) +end + +function tests.test_artifact_lifetime_rejects_close_only_resources() + fibers.run(function (scope) + local closed = 0 + local close_only = { close = function () closed = closed + 1; return true end } + local owned, err = lifetime.own(scope, close_only) + assert_eq(owned, nil) + assert_eq(err, 'resource has no terminate(reason) method') + assert_eq(closed, 0) + end) +end + +function tests.test_artifact_lifetime_rejects_async_close_without_immediate_cleanup() + fibers.run(function (scope) + local async_only = { ['close' .. '_op'] = function () return op.always(true, nil) end } + local owned, err = lifetime.own(scope, async_only) + assert_eq(owned, nil) + assert_eq(err, 'resource exposes close_op but no terminate(reason) method') + end) +end + + +function tests.test_ingest_instance_has_own_child_scope() + local sink = new_sink() + fibers.run(function () + local st, _, snap = fibers.run_scope(function (scope) + local inst = assert(ingest.new_instance(scope, { + ingest_id = 'i-scoped', + component = 'cm5', + sink = sink, + })) + if inst._scope == nil or inst._scope == scope then + fail('ingest instance should own a child scope') + end + return inst:snapshot() + end) + assert_eq(st, 'ok') + assert_eq(snap.ingest_id, 'i-scoped') + end) + assert_eq(sink.terminated, 1) +end + +function tests.test_ingest_instance_scope_finalises_pending_requests() + local sink = new_sink() + local req = new_req({ method = 'ingest_append', ingest_id = 'i1', chunk = 'abc' }) + fibers.run(function () + local st = fibers.run_scope(function (scope) + local inst = assert(ingest.new_instance(scope, { + ingest_id = 'i1', + component = 'cm5', + sink = sink, + })) + inst.pending[#inst.pending + 1] = { req = req, category = 'append' } + end) + assert_eq(st, 'ok') + end) + assert_eq(req.fail_reason, 'ingest_instance_closed') +end + +function tests.test_ingest_operations_are_serialised_per_instance_and_terminal_closes_append_admission() + fibers.run(function (scope) + local append_gate = cond.new() + local sink = { appended = {}, commits = 0, terminated = 0 } + function sink:append_op(chunk) + return append_gate:wait_op():wrap(function () + self.appended[#self.appended + 1] = chunk + return true, nil + end) + end + function sink:commit_op() + self.commits = self.commits + 1 + return op.always({ ref = 'artifact-serial' }, nil) + end + function sink:terminate(reason) + self.terminated = self.terminated + 1 + self.reason = reason + return true, nil + end + + local done_tx, done_rx = mailbox.new(8, { full = 'reject_newest' }) + local state = ingest.new_state(scope, { queue_len = 8 }) + local inst = assert(ingest.new_instance(scope, { + ingest_id = 'i1', + component = 'cm5', + sink = sink, + })) + state._instances.i1 = inst + local ctx = { scope = scope, request_root = scope, service_id = 'update', generation = 1, done_tx = done_tx } + + local append_req = new_req({ method = 'ingest_append', ingest_id = 'i1', chunk = 'abc' }) + local commit_req = new_req({ method = 'ingest_commit', ingest_id = 'i1' }) + local late_append = new_req({ method = 'ingest_append', ingest_id = 'i1', chunk = 'late' }) + + state:handle_event(ctx, { kind = 'ingest_request', request = append_req }) + state:handle_event(ctx, { kind = 'ingest_request', request = commit_req }) + state:handle_event(ctx, { kind = 'ingest_request', request = late_append }) + + assert_eq(sink.commits, 0, 'commit must not run while append is active') + assert_eq(late_append.fail_reason, 'ingest_closing') + + append_gate:signal() + local append_done = fibers.perform(done_rx:recv_op()) + assert_eq(append_done.status, 'ok', 'append should complete before terminal work starts') + assert_eq(append_done.result.tag, 'ingest_appended') + state:handle_done(ctx, append_done) + + -- handle_done admits the next queued instance operation, but does not + -- synchronously run worker code inside the coordinator branch. The commit + -- worker runs when the scheduler next gets control. + assert(inst.active_request_id ~= nil, 'commit should be admitted after append completion') + assert_eq(sink.commits, 0, 'coordinator branch must not synchronously run commit work') + + local commit_done = fibers.perform(done_rx:recv_op()) + assert_eq(sink.commits, 1, 'commit should run after scheduler progress') + state:handle_done(ctx, commit_done) + + assert_eq(#sink.appended, 1) + assert_eq(sink.appended[1], 'abc') + assert_eq(append_req.reply_value.ok, true) + assert_eq(commit_req.reply_value.ok, true) + assert_eq(inst.state, 'committed') + assert_eq(inst.closed, true) + end) +end + + +function tests.test_ingest_commit_construction_does_not_mutate_instance_state() + local sink = { terminated = 0 } + function sink:commit_op() + return op.always({ ref = 'artifact-constructed' }, nil) + end + function sink:terminate(reason) + self.terminated = self.terminated + 1 + self.reason = reason + return true, nil + end + fibers.run(function () + fibers.run_scope(function (scope) + local inst = assert(ingest.new_instance(scope, { + ingest_id = 'i-construct', + component = 'cm5', + sink = sink, + })) + assert_eq(inst.commit_op, nil, 'ingest instances must not expose state-mutating Op construction') + assert_eq(inst.state, 'open') + assert_eq(inst.closed, false) + local ok_begin = assert(inst:begin_commit()) + assert_eq(ok_begin, true) + assert_eq(inst.state, 'committing') + local result = assert(inst:commit_worker(scope)) + assert_eq(result.artifact.ref, 'artifact-constructed') + end) + end) +end + + +function tests.test_ingest_create_defaults_artifact_policy_to_transient_only() + fibers.run(function (scope) + local seen_policy + local artifact_store = { + create_sink_op = function (_, opts) + seen_policy = opts and opts.policy + return op.always(new_sink(), nil) + end, + } + local done_tx, done_rx = mailbox.new(4, { full = 'reject_newest' }) + local state = ingest.new_state(scope, { queue_len = 4 }) + local ctx = { + scope = scope, + request_root = scope, + service_id = 'update', + generation = 1, + done_tx = done_tx, + artifact_store = artifact_store, + } + local req = new_req({ method = 'ingest_create', ingest_id = 'i-default', component = 'mcu' }) + state:handle_event(ctx, { kind = 'ingest_request', request = req }) + local ev = fibers.perform(done_rx:recv_op()) + assert_eq(ev.kind, 'ingest_create_done') + assert_eq(ev.status, 'ok') + state:handle_done(ctx, ev) + assert_eq(seen_policy, 'transient_only') + assert_eq(req.reply_value.ok, true) + end) +end + +function tests.test_ingest_create_honours_explicit_artifact_policy_override() + fibers.run(function (scope) + local seen_policy + local artifact_store = { + create_sink_op = function (_, opts) + seen_policy = opts and opts.policy + return op.always(new_sink(), nil) + end, + } + local done_tx, done_rx = mailbox.new(4, { full = 'reject_newest' }) + local state = ingest.new_state(scope, { queue_len = 4 }) + local ctx = { + scope = scope, + request_root = scope, + service_id = 'update', + generation = 1, + done_tx = done_tx, + artifact_store = artifact_store, + } + local req = new_req({ method = 'ingest_create', ingest_id = 'i-durable', component = 'mcu', policy = 'prefer_durable' }) + state:handle_event(ctx, { kind = 'ingest_request', request = req }) + local ev = fibers.perform(done_rx:recv_op()) + assert_eq(ev.kind, 'ingest_create_done') + assert_eq(ev.status, 'ok') + state:handle_done(ctx, ev) + assert_eq(seen_policy, 'prefer_durable') + assert_eq(req.reply_value.ok, true) + end) +end + +return tests diff --git a/tests/unit/update/test_job_repository.lua b/tests/unit/update/test_job_repository.lua new file mode 100644 index 00000000..c8f13ebf --- /dev/null +++ b/tests/unit/update/test_job_repository.lua @@ -0,0 +1,92 @@ +local repo = require 'services.update.job_repository' +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v,msg) if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end end +local function assert_not_nil(v,msg) if v == nil then fail(msg or 'expected non-nil') end end +function tests.test_new_job_normalises_and_snapshots_are_copies() + local state = assert(repo.new_state()) + local job = assert(repo.new_job({ job_id='j1', component='cm5', artifact_ref='a1' }, { generation=7, seq=1 })) + assert(repo.upsert(state, job)) + assert_eq(job.phase, nil) + assert_eq(job.stage, nil) + local snap = repo.snapshot(state) + assert_eq(snap.count, 1); assert_eq(snap.by_id.j1.component, 'cm5') + assert_eq(snap.by_id.j1.phase, nil) + assert_eq(snap.by_id.j1.stage, nil) + snap.by_id.j1.component = 'mutated' + assert_eq(repo.get(state, 'j1').component, 'cm5') +end + +function tests.test_job_repository_rejects_phase_and_stage_aliases() + local state = assert(repo.new_state()) + local job, err = repo.new_job({ job_id='j1', component='cm5' }, { seq=repo.next_sequence(state) }) + assert_not_nil(job, err) + job.phase = 'created' + local ok_job, jerr = repo.upsert(state, job) + assert_eq(ok_job, nil) + assert_not_nil(jerr) + + job.phase = nil + assert(repo.upsert(state, job)) + local _, perr = repo.patch(state.jobs.j1, { phase = 'stage' }, { seq=repo.next_sequence(state) }) + assert_not_nil(perr) +end + +function tests.test_lifecycle_helpers_do_not_perform_work() + local state = assert(repo.new_state()) + local job = assert(repo.new_job({ job_id='j1', component='cm5' }, { seq=repo.next_sequence(state) })) + repo.upsert(state, job); local stored = state.jobs.j1 + repo.mark_staging(stored, { seq=repo.next_sequence(state), reason='start' }); assert_eq(stored.state, 'staging'); assert_eq(stored.phase, nil); assert_eq(stored.stage, nil) + repo.mark_awaiting_commit(stored, { image='ok' }, { seq=repo.next_sequence(state) }); assert_eq(stored.state, 'awaiting_commit'); assert_eq(stored.next_step, 'commit'); assert_eq(stored.phase, nil); assert_eq(stored.stage, nil) + repo.mark_terminal(stored, 'failed', 'boom', nil, { seq=repo.next_sequence(state) }); assert_true(repo.is_terminal(stored.state)); assert_not_nil(stored.last_event) +end +function tests.test_terminal_compaction_drops_operational_internals() + local job = repo.compact_job({ + job_id='j1', component='cm5', state='succeeded', expected_image_id='img', + created_seq=1, updated_seq=2, next_step='commit', generation=9, + active_token='tok', active_intent={ token='tok', phase='stage' }, active={ token='tok', phase='stage' }, + adoption={ action='kept_committable' }, commit_attempt={ token='ct' }, + stage_result={ reply={ transfer={ xfer_id='x1', sent_bytes=12 } }, preflight={ metadata={ large=true } } }, + result={ ok=true, tag='ok' }, history={ { seq=2, state='succeeded', reason='done' } }, + }) + assert_eq(job.next_step, nil) + assert_eq(job.generation, nil) + assert_eq(job.active_token, nil) + assert_eq(job.active_intent, nil) + assert_eq(job.active, nil) + assert_eq(job.adoption, nil) + assert_not_nil(job.commit_attempt) + assert_eq(job.commit_attempt.token, 'ct') + assert_eq(job.stage_result, nil) + assert_not_nil(job.transfer) + assert_eq(job.transfer.xfer_id, 'x1') + assert_eq(job.result.ok, true) + assert_eq(job.last_event.reason, 'done') +end + +function tests.test_active_compaction_preserves_policy_for_auto_commit() + local job = repo.compact_job({ + job_id='j1', component='mcu', state='awaiting_commit', expected_image_id='img', + created_seq=1, updated_seq=2, next_step='commit', generation=9, + policy={ + job_id='j1', create_if='image_differs', start='auto', commit='auto', + reconcile='required', supersede='same_job_if_image_changed', + }, + }) + assert_not_nil(job.policy) + assert_eq(job.policy.commit, 'auto') + assert_eq(job.policy.start, 'auto') + assert_eq(job.policy.reconcile, 'required') +end + +function tests.test_new_job_preserves_policy_for_active_lifecycle() + local job = assert(repo.new_job({ + job_id='j1', component='mcu', expected_image_id='img', artifact_ref='a1', + policy={ start='auto', commit='auto', reconcile='required' }, + }, { seq=1 })) + assert_not_nil(job.policy) + assert_eq(job.policy.commit, 'auto') +end + +return tests diff --git a/tests/unit/update/test_job_runtime.lua b/tests/unit/update/test_job_runtime.lua new file mode 100644 index 00000000..b4da0694 --- /dev/null +++ b/tests/unit/update/test_job_runtime.lua @@ -0,0 +1,566 @@ +local fibers = require 'fibers' +local op = require 'fibers.op' +local job_runtime = require 'services.update.job_runtime' +local store_mod = require 'services.update.job_store_memory' + +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v,msg) if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end end +local function assert_not_nil(v,msg) if v == nil then fail(msg or 'expected non-nil') end end + +local function start_runtime(scope, params) + local rt = assert(job_runtime.start(scope, params or {})) + local ready, err = fibers.perform(rt:ready_op()) + assert_true(ready, err) + return rt +end + +local function perform_transition(rt, cmd) + local handle, admit_err = rt:admit_transition(cmd) + assert_not_nil(handle, admit_err) + return assert(fibers.perform(handle:outcome_op())) +end + +function tests.test_transition_lifecycle_records_persisted_and_rejected_states() + fibers.run(function () + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { service_id = 'update', store = store_mod.new() }) + local created = perform_transition(rt, { + kind = 'create_job', + generation = 1, + payload = { job_id = 'j1', component = 'cm5' }, + }) + assert_eq(created.status, 'persisted') + local rejected = perform_transition(rt, { + kind = 'start_job', + generation = 1, + job_id = 'missing', + }) + assert_eq(rejected.status, 'rejected') + local transitions = rt:transition_snapshot() + rt:cancel('test complete') + return { created = created, rejected = rejected, transitions = transitions } + end) + assert_eq(st, 'ok') + local created_rec = result.transitions.by_id[result.created.transition_id] + local rejected_rec = result.transitions.by_id[result.rejected.transition_id] + assert_not_nil(created_rec, 'created transition should be recorded') + assert_not_nil(rejected_rec, 'rejected transition should be recorded') + assert_eq(created_rec.state, 'persisted') + assert_eq(created_rec.plan_kind, 'save_job') + assert_eq(rejected_rec.state, 'rejected') + assert_eq(rejected_rec.error, 'not_found') + assert_eq(rejected_rec.admitted, nil, 'rejected-before-admission must not retain admitted=true') + end) +end + +function tests.test_single_job_startup_scrub_keeps_newest_recoverable_and_blitzes_others() + fibers.run(function () + local saves = {} + local initial = { jobs = { + staging = { job_id = 'staging', component = 'cm5', state = 'staging', created_seq = 1, updated_seq = 1 }, + active = { job_id = 'active', component = 'cm5', state = 'staging', created_seq = 4, updated_seq = 4, active_token = 'tok-stage', active_intent = { token = 'tok-stage', phase = 'stage' } }, + commit = { job_id = 'commit', component = 'cm5', state = 'awaiting_commit', created_seq = 2, updated_seq = 2 }, + ret = { job_id = 'ret', component = 'cm5', state = 'awaiting_return', created_seq = 3, updated_seq = 3 }, + }, order = { 'staging', 'commit', 'ret', 'active' }, next_seq = 10 } + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function (_, job) saves[#saves + 1] = job; return op.always(true, nil) end, + delete_job_op = function (_, _) return op.always(true, nil) end, + } + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { service_id = 'update', store = store }) + local adoption = rt:adoption() + local active = rt:get('active') + local staging = rt:get('staging') + local commit = rt:get('commit') + local ret = rt:get('ret') + rt:cancel('test complete') + return { adoption = adoption, active = active, staging = staging, commit = commit, ret = ret } + end) + assert_eq(st, 'ok') + assert_eq(result.staging, nil) + assert_eq(result.commit, nil) + assert_eq(result.ret, nil) + assert_eq(result.active.state, 'staging') + assert_eq(result.active.adoption.action, 'resume_active_intent') + assert_eq(result.adoption.single_job.kept_job_id, 'active') + assert_eq(result.adoption.single_job.deleted_count, 3) + assert_eq(#result.adoption.pruned, 3) + assert_eq(#result.adoption.active_intent, 1) + assert_true(#saves >= 1) + end) +end + +function tests.test_restart_adoption_fails_active_intent_after_configured_restart_limit() + fibers.run(function () + local saves = {} + local initial = { jobs = { + active = { + job_id = 'active', component = 'cm5', state = 'staging', created_seq = 1, updated_seq = 1, + active_token = 'tok-stage', + active_intent = { token = 'tok-stage', phase = 'stage', restart_count = 1 }, + }, + }, order = { 'active' }, next_seq = 10 } + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function (_, job) saves[#saves + 1] = job; return op.always(true, nil) end, + } + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { + service_id = 'update', + store = store, + retention = { active_intent_restart_max = 1 }, + }) + local job = rt:get('active') + local adoption = rt:adoption() + rt:cancel('test complete') + return { job = job, adoption = adoption } + end) + assert_eq(st, 'ok') + assert_eq(result.job.state, 'failed') + assert_eq(result.job.error, 'active_intent_restart_limit_exceeded') + assert_eq(result.job.active_token, nil) + assert_eq(result.job.active_intent, nil) + assert_eq(result.job.active, nil) + assert_eq(result.job.adoption.action, 'failed_restart_limit') + assert_eq(result.job.adoption.restart_attempts, 2) + assert_eq(#result.adoption.failed, 1) + assert_true(#saves >= 1) + end) +end + +function tests.test_restart_adoption_ignores_legacy_history_after_compaction() + fibers.run(function () + local saves = {} + local initial = { jobs = { + active = { + job_id = 'active', component = 'cm5', state = 'staging', created_seq = 1, updated_seq = 1, + active_token = 'tok-stage', + active_intent = { token = 'tok-stage', phase = 'stage' }, + history = { + { seq = 1, state = 'staging', reason = 'restart_adoption_active_intent' }, + }, + }, + }, order = { 'active' }, next_seq = 10 } + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function (_, job) saves[#saves + 1] = job; return op.always(true, nil) end, + } + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { + service_id = 'update', + store = store, + retention = { active_intent_restart_max = 1 }, + }) + local job = rt:get('active') + rt:cancel('test complete') + return { job = job } + end) + assert_eq(st, 'ok') + assert_eq(result.job.state, 'staging') + assert_eq(result.job.history, nil) + assert_eq(result.job.adoption.action, 'resume_active_intent') + assert_eq(result.job.adoption.restart_attempts, 1) + assert_true(#saves >= 1) + end) +end + +function tests.test_restart_adoption_tolerates_missing_restart_counters() + fibers.run(function () + local saves = {} + local initial = { jobs = { + active = { + job_id = 'active', component = 'cm5', state = 'staging', created_seq = 1, updated_seq = 1, + active_token = 'tok-stage', + active_intent = { token = 'tok-stage', phase = 'stage' }, + runtime = {}, + adoption = {}, + }, + }, order = { 'active' }, next_seq = 10 } + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function (_, job) saves[#saves + 1] = job; return op.always(true, nil) end, + } + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { + service_id = 'update', + store = store, + retention = { active_intent_restart_max = 1 }, + }) + local job = rt:get('active') + rt:cancel('test complete') + return { job = job } + end) + assert_eq(st, 'ok') + assert_eq(result.job.state, 'staging') + assert_eq(result.job.adoption.action, 'resume_active_intent') + assert_eq(result.job.active_intent.restart_count, 1) + assert_eq(result.job.active_intent.attempt, 1) + assert_eq(result.job.active.attempt, 1) + assert_eq(result.job.adoption.attempt, 1) + assert_true(#saves >= 1) + end) +end + + +function tests.test_restart_adoption_reads_legacy_attempt_field() + fibers.run(function () + local saves = {} + local initial = { jobs = { + active = { + job_id = 'active', component = 'cm5', state = 'staging', created_seq = 1, updated_seq = 1, + active_token = 'tok-stage', + active_intent = { token = 'tok-stage', phase = 'stage', attempt = 1 }, + }, + }, order = { 'active' }, next_seq = 10 } + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function (_, job) saves[#saves + 1] = job; return op.always(true, nil) end, + } + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { + service_id = 'update', + store = store, + retention = { active_intent_restart_max = 1 }, + }) + local job = rt:get('active') + rt:cancel('test complete') + return { job = job } + end) + assert_eq(st, 'ok') + assert_eq(result.job.state, 'failed') + assert_eq(result.job.error, 'active_intent_restart_limit_exceeded') + assert_eq(result.job.adoption.restart_attempts, 2) + assert_eq(result.job.adoption.attempt, 2) + assert_eq(result.job.result.restart_attempts, 2) + assert_eq(result.job.result.active_intent, nil) + assert_true(#saves >= 1) + end) +end + +function tests.test_failed_after_admission_keeps_admitted_lifecycle_marker() + fibers.run(function () + local initial = { jobs = { + j1 = { job_id = 'j1', component = 'cm5', state = 'created', created_seq = 1, updated_seq = 1 }, + }, order = { 'j1' }, next_seq = 10 } + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function () return op.always(nil, 'save_failed') end, + } + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { service_id = 'update', store = store }) + local failed = perform_transition(rt, { + kind = 'start_job', + generation = 1, + job_id = 'j1', + }) + local transitions = rt:transition_snapshot() + rt:cancel('test complete') + return { failed = failed, transitions = transitions } + end) + assert_eq(st, 'ok') + assert_eq(result.failed.status, 'failed') + local rec = result.transitions.by_id[result.failed.transition_id] + assert_eq(rec.state, 'failed') + assert_eq(rec.error, 'save_failed') + assert_true(rec.admitted, 'failed-after-admission should remain distinguishable from rejection') + end) +end + +function tests.test_submit_transition_rejects_before_ready_without_recording_admission() + fibers.run(function () + local st, _, result = fibers.run_scope(function (scope) + local rt = assert(job_runtime.start(scope, { service_id = 'update', store = store_mod.new() })) + local handle, admit_err = rt:admit_transition { + kind = 'create_job', + generation = 1, + payload = { job_id = 'j1', component = 'cm5' }, + } + assert_eq(handle, nil) + local rejected = { status = 'rejected', reason = admit_err } + local transitions = rt:transition_snapshot() + rt:cancel('test complete') + return { rejected = rejected, transitions = transitions } + end) + assert_eq(st, 'ok') + assert_eq(result.rejected.status, 'rejected') + assert_eq(result.rejected.reason, 'job_runtime_not_ready') + assert_eq(result.transitions.count, 0, 'pre-ready submission is not admitted into durable lifecycle') + end) +end + + +function tests.test_job_runtime_rejects_stale_generation_before_admission() + fibers.run(function () + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { + service_id = 'update', + store = store_mod.new(), + current_generation = 2, + }) + local rejected = perform_transition(rt, { + kind = 'create_job', + generation = 1, + payload = { job_id = 'old', component = 'cm5' }, + }) + local transitions = rt:transition_snapshot() + rt:cancel('test complete') + return { rejected = rejected, transitions = transitions } + end) + assert_eq(st, 'ok') + assert_eq(result.rejected.status, 'rejected') + assert_eq(result.rejected.reason, 'stale_generation') + local rec = result.transitions.by_id[result.rejected.transition_id] + assert_not_nil(rec, 'stale transition should be recorded as rejected') + assert_eq(rec.state, 'rejected') + assert_eq(rec.error, 'stale_generation') + assert_eq(rec.admitted, nil) + end) +end + +function tests.test_job_runtime_allows_old_generation_active_apply_by_token() + fibers.run(function () + local initial = { jobs = { + j1 = { + job_id = 'j1', + component = 'cm5', + state = 'staging', + created_seq = 1, + updated_seq = 1, + active_token = 'tok1', + active_intent = { token = 'tok1', phase = 'stage', generation = 1 }, + }, + }, order = { 'j1' }, next_seq = 10 } + local saved = {} + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function (_, job) saved[#saved + 1] = job; return op.always(true, nil) end, + } + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { + service_id = 'update', + store = store, + current_generation = 2, + }) + local applied = perform_transition(rt, { + kind = 'apply_active_result', + generation = 1, + job_id = 'j1', + phase = 'stage', + token = 'tok1', + event = { + kind = 'active_job_done', + generation = 1, + job_id = 'j1', + phase = 'stage', + token = 'tok1', + status = 'ok', + result = { tag = 'staged', staged = { component = 'cm5' } }, + }, + }) + local job = rt:get('j1') + rt:cancel('test complete') + return { applied = applied, job = job } + end) + assert_eq(st, 'ok') + assert_eq(result.applied.status, 'persisted') + assert_eq(result.job.state, 'awaiting_commit') + assert_eq(result.job.active_token, nil) + assert_true(#saved >= 2) + end) +end + +function tests.test_job_runtime_transition_admission_is_not_an_op_builder() + local f = assert(io.open('../src/services/update/job_runtime.lua', 'r')) + local src = f:read('*a') + f:close() + if src:find('submit_transition_op', 1, true) or src:find('transition_op', 1, true) then + fail('job_runtime should expose immediate admit_transition, not transition Op compatibility helpers') + end + if not src:find('function Runtime:admit_transition', 1, true) then + fail('job_runtime should expose admit_transition') + end + local body = src:match('function Runtime:admit_transition%(.+function Runtime:terminate') or '' + if body:find('op.guard', 1, true) or body:find('fibers.perform', 1, true) then + fail('admit_transition must remain strictly immediate') + end +end + + +function tests.test_begin_commit_attempt_persists_awaiting_return_before_backend_commit() + fibers.run(function () + local saved = {} + local initial = { jobs = { + j1 = { + job_id = 'j1', component = 'cm5', state = 'committing', + created_seq = 1, updated_seq = 1, active_token = 'active-token', + active_intent = { token = 'active-token', phase = 'commit', commit_token = 'commit-token', commit_policy = 'idempotent_by_token' }, + commit_attempt = { token = 'commit-token', policy = 'idempotent_by_token', acceptance = 'unknown' }, + }, + }, order = { 'j1' }, next_seq = 10 } + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function (_, job) saved[#saved + 1] = job; return op.always(true, nil) end, + } + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { service_id = 'update', store = store }) + local begun = perform_transition(rt, { + kind = 'begin_commit_attempt', + job_id = 'j1', + phase = 'commit', + token = 'active-token', + commit_token = 'commit-token', + commit_policy = 'idempotent_by_token', + }) + local job = rt:get('j1') + rt:cancel('test complete') + return { begun = begun, job = job } + end) + assert_eq(st, 'ok') + assert_eq(result.begun.status, 'persisted') + assert_eq(result.begun.commit_token, 'commit-token') + assert_eq(result.job.state, 'awaiting_return') + assert_eq(result.job.next_step, 'reconcile') + assert_eq(result.job.active_token, nil) + assert_eq(result.job.active_intent, nil) + assert_eq(result.job.commit_attempt.token, 'commit-token') + assert_eq(result.job.commit_attempt.policy, 'idempotent_by_token') + assert_eq(result.job.commit_attempt.acceptance, 'unknown') + assert_eq(result.job.commit_result.tag, 'commit_pending') + assert_eq(result.job.commit_result.commit_token, 'commit-token') + assert_eq(#saved >= 2, true, 'adoption and awaiting_return commit attempt should be durably saved') + end) +end + +function tests.test_single_job_startup_scrub_keeps_newest_uncertain_commit_only() + fibers.run(function () + local initial = { jobs = { + idem = { job_id = 'idem', component = 'cm5', state = 'committing', created_seq = 1, updated_seq = 1, active_token = 'tok-idem', active_intent = { token = 'tok-idem', phase = 'commit', commit_token = 'ct-idem', commit_policy = 'idempotent_by_token' }, commit_attempt = { token = 'ct-idem', policy = 'idempotent_by_token' } }, + nodup = { job_id = 'nodup', component = 'cm5', state = 'committing', created_seq = 2, updated_seq = 2, active_token = 'tok-nodup', active_intent = { token = 'tok-nodup', phase = 'commit', commit_token = 'ct-nodup', commit_policy = 'no_duplicate' }, commit_attempt = { token = 'ct-nodup', policy = 'no_duplicate' } }, + missing = { job_id = 'missing', component = 'cm5', state = 'committing', created_seq = 3, updated_seq = 3, active_token = 'tok-missing', active_intent = { token = 'tok-missing', phase = 'commit', commit_token = 'ct-missing' } }, + }, order = { 'idem', 'nodup', 'missing' }, next_seq = 10 } + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function (_, _) return op.always(true, nil) end, + delete_job_op = function (_, _) return op.always(true, nil) end, + } + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { service_id = 'update', store = store }) + local adoption = rt:adoption() + local idem = rt:get('idem') + local nodup = rt:get('nodup') + local missing = rt:get('missing') + rt:cancel('test complete') + return { adoption = adoption, idem = idem, nodup = nodup, missing = missing } + end) + assert_eq(st, 'ok') + assert_eq(result.idem, nil) + assert_eq(result.nodup, nil) + assert_not_nil(result.missing) + assert_eq(result.missing.state, 'failed') + assert_eq(result.missing.error, 'restart_commit_policy_missing') + assert_eq(result.adoption.single_job.kept_job_id, 'missing') + assert_eq(result.adoption.single_job.deleted_count, 2) + assert_eq(#result.adoption.failed, 1) + end) +end + +function tests.test_job_runtime_blitzes_legacy_jobs_and_keeps_newest_terminal() + fibers.run(function () + local initial = { jobs = { + old1 = { job_id = 'old1', component = 'cm5', state = 'succeeded', created_seq = 1, updated_seq = 1, history = { { seq = 1, state = 'succeeded', reason = 'old' } } }, + old2 = { job_id = 'old2', component = 'cm5', state = 'failed', created_seq = 2, updated_seq = 2 }, + keep1 = { job_id = 'keep1', component = 'cm5', state = 'succeeded', created_seq = 3, updated_seq = 3 }, + keep2 = { job_id = 'keep2', component = 'cm5', state = 'cancelled', created_seq = 4, updated_seq = 4 }, + live = { job_id = 'live', component = 'cm5', state = 'created', created_seq = 5, updated_seq = 5 }, + }, order = { 'old1', 'old2', 'keep1', 'keep2', 'live' }, next_seq = 10 } + local store = store_mod.new(initial) + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { service_id = 'update', store = store }) + local snapshot = rt:snapshot() + local adoption = rt:adoption() + rt:cancel('test complete') + return { snapshot = snapshot, adoption = adoption } + end) + assert_eq(st, 'ok') + assert_eq(result.snapshot.count, 1) + assert_not_nil(result.snapshot.by_id.keep2) + assert_eq(result.snapshot.by_id.keep2.history, nil) + assert_eq(#result.adoption.pruned, 4) + end) +end + +function tests.test_job_runtime_keeps_newest_recoverable_before_terminal() + fibers.run(function () + local initial = { jobs = { + terminal = { job_id = 'terminal', component = 'cm5', state = 'succeeded', created_seq = 10, updated_seq = 10 }, + awaiting = { job_id = 'awaiting', component = 'cm5', state = 'awaiting_return', created_seq = 2, updated_seq = 2, commit_attempt = { token = 'ct' } }, + created = { job_id = 'created', component = 'cm5', state = 'created', created_seq = 99, updated_seq = 99 }, + }, order = { 'awaiting', 'terminal', 'created' }, next_seq = 100 } + local store = store_mod.new(initial) + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { service_id = 'update', store = store }) + local snapshot = rt:snapshot() + local adoption = rt:adoption() + rt:cancel('test complete') + return { snapshot = snapshot, adoption = adoption } + end) + assert_eq(st, 'ok') + assert_eq(result.snapshot.count, 1) + assert_not_nil(result.snapshot.by_id.awaiting) + assert_eq(result.adoption.single_job.kept_job_id, 'awaiting') + assert_eq(result.adoption.single_job.deleted_count, 2) + end) +end + +function tests.test_single_job_startup_scrub_prefers_terminal_over_plain_awaiting_commit() + fibers.run(function () + local initial = { jobs = { + terminal = { job_id = 'terminal', component = 'cm5', state = 'succeeded', created_seq = 10, updated_seq = 10 }, + awaiting = { job_id = 'awaiting', component = 'cm5', state = 'awaiting_commit', created_seq = 99, updated_seq = 99 }, + }, order = { 'terminal', 'awaiting' }, next_seq = 100 } + local store = store_mod.new(initial) + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { service_id = 'update', store = store }) + local snapshot = rt:snapshot() + local adoption = rt:adoption() + rt:cancel('test complete') + return { snapshot = snapshot, adoption = adoption } + end) + assert_eq(st, 'ok') + assert_eq(result.snapshot.count, 1) + assert_not_nil(result.snapshot.by_id.terminal) + assert_eq(result.snapshot.by_id.awaiting, nil) + assert_eq(result.adoption.single_job.kept_job_id, 'terminal') + end) +end + +function tests.test_restart_adoption_does_not_resave_plain_awaiting_commit() + fibers.run(function () + local saves = 0 + local initial = { jobs = { + awaiting = { job_id = 'awaiting', component = 'cm5', state = 'awaiting_commit', created_seq = 1, updated_seq = 1 }, + }, order = { 'awaiting' }, next_seq = 10 } + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function () saves = saves + 1; return op.always(true, nil) end, + delete_job_op = function () return op.always(true, nil) end, + } + local st, _, result = fibers.run_scope(function (scope) + local rt = start_runtime(scope, { service_id = 'update', store = store }) + local job = rt:get('awaiting') + local adoption = rt:adoption() + rt:cancel('test complete') + return { job = job, adoption = adoption } + end) + assert_eq(st, 'ok') + assert_eq(result.job.state, 'awaiting_commit') + assert_eq(result.job.adoption, nil) + assert_eq(#result.adoption.awaiting_commit, 1) + assert_eq(saves, 1, 'startup compaction save is allowed; adoption must not add a second save') + end) +end + +return tests diff --git a/tests/unit/update/test_job_store_control_store.lua b/tests/unit/update/test_job_store_control_store.lua new file mode 100644 index 00000000..ae703d9e --- /dev/null +++ b/tests/unit/update/test_job_store_control_store.lua @@ -0,0 +1,122 @@ +local fibers = require 'fibers' +local runfibers = require 'tests.support.run_fibers' +local store_mod = require 'services.update.job_store_control_store' + +local T = {} + +local function fake_conn() + local data = {} + local calls = {} + return { + calls = calls, + data = data, + call_op = function(_, topic, payload) + calls[#calls + 1] = { topic = topic, payload = payload } + local method = topic[5] + if topic[1] ~= 'cap' or topic[2] ~= 'control-store' or topic[3] ~= 'update' or topic[4] ~= 'rpc' then + return require('fibers.op').always({ ok = false, reason = 'bad_topic' }, nil) + end + if method == 'list' then + local prefix = payload and payload.prefix or '' + local out = {} + for k in pairs(data) do + if prefix == '' or k:sub(1, #prefix) == prefix then out[#out + 1] = k end + end + table.sort(out) + return require('fibers.op').always({ ok = true, reason = out }, nil) + elseif method == 'get' then + if data[payload.key] == nil then return require('fibers.op').always({ ok = false, reason = 'not found' }, nil) end + return require('fibers.op').always({ ok = true, reason = data[payload.key] }, nil) + elseif method == 'put' then + data[payload.key] = payload.data + return require('fibers.op').always({ ok = true, reason = nil }, nil) + elseif method == 'delete' then + data[payload.key] = nil + return require('fibers.op').always({ ok = true, reason = nil }, nil) + end + return require('fibers.op').always({ ok = false, reason = 'bad_method' }, nil) + end, + } +end + +function T.save_load_and_delete_round_trip() + runfibers.run(function() + local conn = fake_conn() + local store = store_mod.new(conn) + + local ok_save, save_err = fibers.perform(store:save_job_op({ + job_id = 'job-1', + component = 'mcu', + expected_image_id = 'mcu-image-test', + state = 'created', + created_seq = 1, + updated_seq = 1, + history = {}, + })) + assert(ok_save == true, tostring(save_err)) + assert(conn.data['update-job-job-1'] ~= nil) + + local snapshot, load_err = fibers.perform(store:load_all_op()) + assert(snapshot ~= nil, tostring(load_err)) + assert(snapshot.jobs['job-1'].component == 'mcu') + assert(snapshot.order[1] == 'job-1') + + local ok_delete, delete_err = fibers.perform(store:delete_job_op('job-1')) + assert(ok_delete == true, tostring(delete_err)) + assert(conn.data['update-job-job-1'] == nil) + end) +end + +function T.uses_control_store_update_capability() + runfibers.run(function() + local conn = fake_conn() + local store = store_mod.new(conn) + local ok_save = fibers.perform(store:save_job_op({ job_id = 'job-2', component = 'mcu', expected_image_id = 'mcu-image-test' })) + assert(ok_save == true) + local t = conn.calls[1].topic + assert(t[1] == 'cap') + assert(t[2] == 'control-store') + assert(t[3] == 'update') + assert(t[4] == 'rpc') + assert(t[5] == 'put') + end) +end + +function T.accepts_hal_void_success_replies_for_put_and_delete() + runfibers.run(function() + local stored = {} + local conn = { + call_op = function(_, topic, payload) + local method = topic[5] + if method == 'put' then + stored[payload.key] = payload.data + return require('fibers.op').always({ ok = true, reason = nil }, nil) + elseif method == 'delete' then + stored[payload.key] = nil + return require('fibers.op').always({ ok = true, reason = nil }, nil) + elseif method == 'list' then + local out = {} + for key in pairs(stored) do out[#out + 1] = key end + table.sort(out) + return require('fibers.op').always({ ok = true, reason = out }, nil) + elseif method == 'get' then + return require('fibers.op').always({ ok = true, reason = stored[payload.key] }, nil) + end + return require('fibers.op').always({ ok = false, reason = 'bad method' }, nil) + end, + } + + local store = store_mod.new(conn) + local ok_save, save_err = fibers.perform(store:save_job_op({ job_id = 'job-hal', component = 'mcu', expected_image_id = 'mcu-image-test', state = 'created' })) + assert(ok_save == true, tostring(save_err)) + + local snapshot, load_err = fibers.perform(store:load_all_op()) + assert(snapshot ~= nil, tostring(load_err)) + assert(snapshot.jobs['job-hal'].component == 'mcu') + + local ok_delete, delete_err = fibers.perform(store:delete_job_op('job-hal')) + assert(ok_delete == true, tostring(delete_err)) + end) +end + +return T diff --git a/tests/unit/update/test_job_store_memory.lua b/tests/unit/update/test_job_store_memory.lua new file mode 100644 index 00000000..02036fe5 --- /dev/null +++ b/tests/unit/update/test_job_store_memory.lua @@ -0,0 +1,26 @@ +local fibers = require 'fibers' +local store_mod = require 'services.update.job_store_memory' +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v,msg) if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end end +function tests.test_memory_store_exposes_operation_shaped_api() + fibers.run(function () + local store = store_mod.new() + assert_true(fibers.perform(store:save_job_op({ job_id='j1', component='cm5', state='created' }))) + local loaded = assert(fibers.perform(store:load_all_op())) + assert_eq(loaded.jobs.j1.component, 'cm5') + assert_true(fibers.perform(store:delete_job_op('j1'))) + loaded = assert(fibers.perform(store:load_all_op())) + assert_eq(loaded.jobs.j1, nil) + end) +end +function tests.test_store_snapshots_are_copied() + fibers.run(function () + local store = store_mod.new(); local job = { job_id='j1', component='cm5', nested={a=1} } + assert_true(fibers.perform(store:save_job_op(job))); job.nested.a = 99 + local loaded = assert(fibers.perform(store:load_all_op())) + assert_eq(loaded.jobs.j1.nested.a, 1) + end) +end +return tests diff --git a/tests/unit/update/test_manager_requests.lua b/tests/unit/update/test_manager_requests.lua new file mode 100644 index 00000000..8895e6bf --- /dev/null +++ b/tests/unit/update/test_manager_requests.lua @@ -0,0 +1,284 @@ +local fibers = require 'fibers' +local cond = require 'fibers.cond' +local op = require 'fibers.op' +local manager_requests = require 'services.update.manager_requests' +local store_mod = require 'services.update.job_store_memory' +local job_runtime = require 'services.update.job_runtime' + +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v,msg) if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end end +local function assert_nil(v,msg) if v ~= nil then fail(msg or ('expected nil, got '..tostring(v))) end end +local function assert_not_nil(v,msg) if v == nil then fail(msg or 'expected non-nil') end end + +local function request(payload) + local c = cond.new(); local req = { payload=payload, done=false } + function req:reply(v) self.done=true; self.ok=true; self.value=v; c:signal(); return true end + function req:fail(e) self.done=true; self.ok=false; self.err=e; c:signal(); return true end + function req:wait_op() return op.guard(function() if self.done then return op.always(self.ok,self.value,self.err) end; return c:wait_op():wrap(function() return self.ok,self.value,self.err end) end) end + return req +end + +local function start_jobs(scope, store, initial) + local jobs = assert(job_runtime.start(scope, { + service_id = 'update', + store = store or store_mod.new(initial), + initial_jobs = initial, + })) + local ready, err = fibers.perform(jobs:ready_op()) + assert_true(ready, err) + return jobs +end + +local function initial_with(job) + return { jobs = { [job.job_id] = job }, order = { job.job_id }, next_seq = 10 } +end + +function tests.test_create_job_scope_replies_and_returns_completion_fact() + fibers.run(function () + local req = request({ method='create_job', job_id='j1', component='cm5', artifact_ref='a1' }) + local st, rep, result = fibers.run_scope(function (scope) + local jobs = start_jobs(scope, store_mod.new()) + local out = manager_requests.create_job(scope, { request=req, jobs=jobs, config={ components={ cm5={component='cm5'} } }, generation=3 }) + jobs:cancel('test complete') + return out + end) + assert_eq(st, 'ok') + assert_eq(result.status, 'persisted') + assert_eq(result.tag, 'job_created') + local ok, value = fibers.perform(req:wait_op()); assert_true(ok); assert_eq(value.job.job_id, 'j1') + end) +end + +function tests.test_create_job_rejects_unknown_component_without_throwing() + fibers.run(function () + local req = request({ method='create_job', job_id='j1', component='mcu' }) + local st, rep, result = fibers.run_scope(function (scope) return manager_requests.create_job(scope, { request=req, config={ components={ cm5={component='cm5'} } }, seq=1 }) end) + assert_eq(st, 'ok'); assert_eq(result.tag, 'manager_request_rejected') + local ok, value, err = fibers.perform(req:wait_op()); assert_eq(ok, false); assert_eq(err, 'unknown_component') + end) +end + +function tests.test_start_job_persists_active_intent_without_starting_active_work() + fibers.run(function () + local saves = {} + local initial = initial_with({ job_id = 'j1', component = 'cm5', state = 'created' }) + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function (_, job) saves[#saves + 1] = job; return op.always(true, nil) end, + } + + local req = request({ method = 'start_job', job_id = 'j1' }) + local st, _, result = fibers.run_scope(function (scope) + local jobs = start_jobs(scope, store) + local out = manager_requests.start_job(scope, { + request = req, + jobs = jobs, + job_id = 'j1', + phase = 'stage', + generation = 1, + }) + jobs:cancel('test complete') + return out + end) + + assert_eq(st, 'ok') + assert_eq(result.status, 'persisted') + assert_eq(result.tag, 'job_started') + assert_eq(#saves, 1, 'start request should durably save active intent') + assert_eq(saves[1].state, 'staging') + assert_not_nil(saves[1].active_intent, 'active intent should be durable') + assert_eq(saves[1].active_intent.phase, 'stage') + local ok, value = fibers.perform(req:wait_op()) + assert_eq(ok, true) + assert_eq(value.accepted, true) + assert_eq(value.token, result.token) + end) +end + +function tests.test_second_start_rejected_while_durable_active_intent_exists() + fibers.run(function () + local initial = { jobs = { + j1 = { job_id = 'j1', component = 'cm5', state = 'staging', active_token='tok-1', active_intent={ token='tok-1', phase='stage' }, created_seq=1, updated_seq=1 }, + j2 = { job_id = 'j2', component = 'cm5', state = 'created', created_seq=2, updated_seq=2 }, + }, order = { 'j1', 'j2' }, next_seq = 10 } + local req = request({ method = 'start_job', job_id = 'j2' }) + local st, _, result = fibers.run_scope(function (scope) + local jobs = start_jobs(scope, store_mod.new(initial), initial) + local out = manager_requests.start_job(scope, { + request = req, + jobs = jobs, + job_id = 'j2', + phase = 'stage', + generation = 1, + }) + jobs:cancel('test complete') + return out + end) + assert_eq(st, 'ok') + assert_eq(result.tag, 'manager_request_rejected') + assert_eq(result.reason, 'slot_busy') + local ok, _, err = fibers.perform(req:wait_op()) + assert_eq(ok, false) + assert_eq(err, 'slot_busy') + end) +end + +function tests.test_start_job_caller_cancellation_after_transition_admission_does_not_cancel_durable_transition() + fibers.run(function () + local save_entered = cond.new() + local save_release = cond.new() + local saves = {} + local initial = initial_with({ job_id = 'j1', component = 'cm5', state = 'created' }) + local store = { + load_all_op = function () return op.always(initial, nil) end, + save_job_op = function (_, job) + saves[#saves + 1] = job + save_entered:signal() + return save_release:wait_op():wrap(function () return true, nil end) + end, + } + + local req = request({ method = 'start_job', job_id = 'j1' }) + req.reply_count = 0 + req.fail_count = 0 + local original_reply = req.reply + local original_fail = req.fail + function req:reply(v) self.reply_count = self.reply_count + 1; return original_reply(self, v) end + function req:fail(e) self.fail_count = self.fail_count + 1; return original_fail(self, e) end + + local st, _, result = fibers.run_scope(function (scope) + local jobs = start_jobs(scope, store) + local request_scope = assert(scope:child()) + local ok, spawn_err = request_scope:spawn(function (rs) + manager_requests.start_job(rs, { + request = req, + jobs = jobs, + job_id = 'j1', + phase = 'stage', + generation = 1, + }) + end) + assert_true(ok, spawn_err) + + fibers.perform(save_entered:wait_op()) + local transitions_at_admission = jobs:transition_snapshot() + local transition_id = transitions_at_admission.order[1] + local admitted = transitions_at_admission.by_id[transition_id] + assert_eq(admitted.state, 'persisting') + assert_true(admitted.admitted, 'transition should be admitted before caller cancellation') + + request_scope:cancel('caller_cancelled') + local cst = fibers.perform(request_scope:join_op()) + assert_eq(cst, 'cancelled') + assert_eq(req.fail_count, 1) + assert_eq(req.reply_count, 0) + assert_eq(req.err, 'caller_cancelled') + + local seen = jobs:version() + save_release:signal() + for _ = 1, 8 do + local job = jobs:get('j1') + if job and job.state == 'staging' then break end + local version = fibers.perform(jobs:changed_op(seen)) + seen = version or seen + end + + local job = jobs:get('j1') + local transitions = jobs:transition_snapshot() + local outcome = jobs:transition_outcome(transition_id) + jobs:cancel('test complete') + return { job = job, transitions = transitions, transition_id = transition_id, outcome = outcome, saves = saves } + end) + + assert_eq(st, 'ok') + assert_eq(result.job.state, 'staging') + assert_not_nil(result.job.active_intent, 'durable active intent should be persisted despite caller cancellation') + assert_eq(result.transitions.by_id[result.transition_id].state, 'persisted') + assert_eq(result.outcome.status, 'persisted') + assert_eq(#result.saves, 1) + end) +end + + +function tests.test_create_job_requires_artifact_ref() + fibers.run(function () + local req = request({ method='create_job', job_id='j-missing-artifact', component='cm5' }) + local st, _, result = fibers.run_scope(function (scope) + local jobs = start_jobs(scope, store_mod.new()) + local out = manager_requests.create_job(scope, { + request = req, + jobs = jobs, + config = { components = { cm5 = { component = 'cm5' } } }, + generation = 1, + }) + jobs:cancel('test complete') + return out + end) + assert_eq(st, 'ok') + assert_eq(result.tag, 'manager_request_rejected') + assert_eq(result.reason, 'artifact_ref_required') + local ok, _, err = fibers.perform(req:wait_op()) + assert_eq(ok, false) + assert_eq(err, 'artifact_ref_required') + end) +end + +function tests.test_create_mcu_job_resolves_expected_image_id_from_dcmcu_artifact() + fibers.run(function () + local dcmcu_fixture = require 'tests.support.dcmcu_fixture' + local blob_source = require 'devicecode.blob_source' + local req = request({ method='create_job', job_id='j-mcu', component='mcu', artifact_ref='artifact-mcu' }) + local artifact_store = { + open_source_op = function (_, ref) + assert_eq(ref, 'artifact-mcu') + return op.always(blob_source.from_string(dcmcu_fixture.make('mcu-image-new')), nil) + end, + } + local st, _, result = fibers.run_scope(function (scope) + local jobs = start_jobs(scope, store_mod.new()) + local out = manager_requests.create_job(scope, { + request = req, + jobs = jobs, + config = { components = { mcu = { component = 'mcu' } } }, + artifact_store = artifact_store, + generation = 3, + }) + jobs:cancel('test complete') + return out + end) + assert_eq(st, 'ok') + assert_eq(result.status, 'persisted') + assert_eq(result.job.expected_image_id, 'mcu-image-new') + local ok, value = fibers.perform(req:wait_op()) + assert_true(ok) + assert_eq(value.job.expected_image_id, 'mcu-image-new') + end) +end + +function tests.test_create_mcu_job_rejects_caller_supplied_expected_image_id() + fibers.run(function () + local req = request({ method='create_job', job_id='j-mcu', component='mcu', artifact_ref='artifact-mcu', expected_image_id='caller-value' }) + local st, _, result = fibers.run_scope(function (scope) + local jobs = start_jobs(scope, store_mod.new()) + local out = manager_requests.create_job(scope, { + request = req, + jobs = jobs, + config = { components = { mcu = { component = 'mcu' } } }, + artifact_store = { open_source_op = function () error('must not inspect artifact after rejecting caller expected_image_id') end }, + generation = 3, + }) + jobs:cancel('test complete') + return out + end) + assert_eq(st, 'ok') + assert_eq(result.tag, 'manager_request_rejected') + assert_eq(result.reason, 'expected_image_id_must_be_resolved_from_artifact') + local ok, _, err = fibers.perform(req:wait_op()) + assert_eq(ok, false) + assert_eq(err, 'expected_image_id_must_be_resolved_from_artifact') + end) +end + +return tests diff --git a/tests/unit/update/test_model.lua b/tests/unit/update/test_model.lua new file mode 100644 index 00000000..f82df201 --- /dev/null +++ b/tests/unit/update/test_model.lua @@ -0,0 +1,83 @@ +-- tests/unit/update/test_model.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local mailbox = require 'fibers.mailbox' + +local model_mod = require 'services.update.model' +local queue = require 'devicecode.support.queue' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_false(v, msg) if v ~= false then fail(msg or ('expected false, got ' .. tostring(v))) end end +local function assert_nil(v, msg) if v ~= nil then fail(msg or ('expected nil, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil value') end end + +function tests.test_snapshot_returns_deep_copy() + fibers.run(function () + local m = model_mod.new({ state = 'idle', nested = { n = 1 } }) + local snap = m:snapshot() + snap.nested.n = 99 + assert_eq(m:snapshot().nested.n, 1) + end) +end + +function tests.test_unchanged_snapshot_does_not_signal() + fibers.run(function () + local m = model_mod.new({ a = 1, b = { c = 2 } }) + local v0 = m:version() + local changed, v = m:set_snapshot({ a = 1, b = { c = 2 } }) + assert_false(changed) + assert_eq(v, v0) + end) +end + +function tests.test_changed_op_returns_snapshot() + fibers.run(function (scope) + local m = model_mod.new({ state = 'idle' }) + local seen = m:version() + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = scope:spawn(function () + local version, snap, cerr = fibers.perform(m:changed_op(seen)) + queue.assert_admit_required(tx, { version = version, snap = snap, err = cerr }, 'update_model_changed') + end) + assert_true(ok, err) + + fibers.perform(sleep.sleep_op(0.001)) + m:set_snapshot({ state = 'ready' }) + + local result = fibers.perform(rx:recv_op()) + assert_not_nil(result) + assert_eq(result.version, seen + 1) + assert_eq(result.snap.state, 'ready') + assert_nil(result.err) + end) +end + +function tests.test_terminate_wakes_observer() + fibers.run(function (scope) + local m = model_mod.new({ state = 'idle' }) + local seen = m:version() + local tx, rx = mailbox.new(1, { full = 'reject_newest' }) + + local ok, err = scope:spawn(function () + local version, snap, cerr = fibers.perform(m:changed_op(seen)) + queue.assert_admit_required(tx, { version = version, snap = snap, err = cerr }, 'update_model_closed') + end) + assert_true(ok, err) + + fibers.perform(sleep.sleep_op(0.001)) + m:terminate('closed for test') + + local result = fibers.perform(rx:recv_op()) + assert_nil(result.version) + assert_nil(result.snap) + assert_eq(result.err, 'closed for test') + end) +end + +return tests diff --git a/tests/unit/update/test_observe_reconcile.lua b/tests/unit/update/test_observe_reconcile.lua new file mode 100644 index 00000000..dfd28e9d --- /dev/null +++ b/tests/unit/update/test_observe_reconcile.lua @@ -0,0 +1,219 @@ +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local op = require 'fibers.op' +local observe = require 'services.update.observe' +local active_job = require 'services.update.active_job' + +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v,msg) if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end end + +function tests.test_observer_changed_op_wakes_with_snapshot_copy() + fibers.run(function () + local obs = observe.new({ components = { cm5 = { component = 'cm5' } } }) + local seen = obs:version() + local got + fibers.spawn(function () + local version, snapshot = fibers.perform(obs:changed_op(seen)) + got = { version = version, snapshot = snapshot } + end) + obs:update_component('cm5', { boot_id = 'b1' }) + fibers.perform(sleep.sleep_op(0.01)) + assert_true(got ~= nil, 'expected observer wait to wake') + assert_eq(got.snapshot.by_id.cm5.state.boot_id, 'b1') + got.snapshot.by_id.cm5.state.boot_id = 'mutated' + assert_eq(obs:snapshot().by_id.cm5.state.boot_id, 'b1') + end) +end + +function tests.test_reconcile_worker_waits_on_component_observer() + fibers.run(function (scope) + local obs = observe.new({ components = { cm5 = { component = 'cm5' } } }) + local backend = {} + function backend:evaluate_reconcile(job, snapshot) + local cm5 = snapshot and snapshot.by_id and snapshot.by_id.cm5 + if cm5 and cm5.state and cm5.state.version == 'new' then + return { done = true, tag = 'reconciled_success', observed = cm5.state } + end + return { done = false } + end + fibers.spawn(function () + fibers.perform(sleep.sleep_op(0.02)) + obs:update_component('cm5', { version = 'new' }) + end) + local result = active_job.reconcile(scope, { + backend = backend, + job = { job_id = 'j1', component = 'cm5' }, + observer = obs, + deadline = fibers.now() + 1, + }) + assert_eq(result.tag, 'reconciled_success') + assert_eq(result.job_id, 'j1') + assert_eq(result.observed.version, 'new') + end) +end + + +function tests.test_stage_worker_runs_single_stage_op() + fibers.run(function (scope) + local called = 0 + local backend = {} + function backend:stage_op(job, ctx) + called = called + 1 + assert_eq(ctx.phase, 'stage') + return op.always({ staged = true, preflight = { compatible = true }, prepared = { handle = 'prepared' } }, nil) + end + + local result = active_job.stage(scope, { + backend = backend, + job = { job_id = 'j1', component = 'cm5' }, + }) + + assert_eq(result.tag, 'staged') + assert_eq(result.staged.staged, true) + assert_eq(result.staged.preflight.compatible, true) + assert_eq(result.staged.prepared.handle, 'prepared') + assert_eq(called, 1) + end) +end +function tests.test_reconcile_worker_returns_observer_closed_result() + fibers.run(function (scope) + local obs = observe.new({ components = { cm5 = { component = 'cm5' } } }) + local backend = {} + function backend:evaluate_reconcile() + return { done = false } + end + + fibers.spawn(function () + fibers.perform(sleep.sleep_op(0.02)) + obs:terminate('observer_lost') + end) + + local result = active_job.reconcile(scope, { + backend = backend, + job = { job_id = 'j1', component = 'cm5' }, + observer = obs, + deadline = fibers.now() + 1, + }) + + assert_eq(result.tag, 'reconcile_observer_closed') + assert_eq(result.reason, 'observer_lost') + end) +end + +function tests.test_reconcile_worker_returns_deadline_result() + fibers.run(function (scope) + local backend = {} + function backend:evaluate_reconcile() + return { done = false } + end + + local result = active_job.reconcile(scope, { + backend = backend, + job = { job_id = 'j1', component = 'cm5' }, + deadline = fibers.now() + 0.02, + poll_s = 0.005, + }) + + assert_eq(result.tag, 'reconcile_timeout') + assert_eq(result.job_id, 'j1') + end) +end + + +function tests.test_commit_worker_persists_awaiting_return_before_backend_commit_and_passes_token() + fibers.run(function (scope) + local order = {} + local jobs = {} + function jobs:admit_transition(cmd) + if cmd.kind == 'begin_commit_attempt' then + order[#order + 1] = 'begin_attempt' + assert_eq(cmd.token, 'active-token') + assert_eq(cmd.commit_token, 'active-token') + assert_eq(cmd.commit_policy, 'idempotent_by_token') + elseif cmd.kind == 'commit_accepted' then + order[#order + 1] = 'commit_accepted' + assert_eq(cmd.commit_token, 'active-token') + assert_eq(cmd.commit_policy, 'idempotent_by_token') + else + error('unexpected transition '..tostring(cmd.kind), 0) + end + return { outcome_op = function () return op.always({ status = 'persisted', commit_token = cmd.commit_token, commit_policy = cmd.commit_policy }, nil) end }, nil + end + local backend = {} + function backend:commit_capabilities() + return { policy = 'idempotent_by_token' } + end + function backend:commit_op(job, ctx) + assert_eq(order[#order], 'begin_attempt') + order[#order + 1] = 'backend_commit' + assert_eq(ctx.commit_token, 'active-token') + assert_eq(ctx.commit_policy, 'idempotent_by_token') + return op.always({ accepted = true, token = ctx.commit_token }, nil) + end + local result = active_job.commit(scope, { + backend = backend, + jobs = jobs, + lease = { token = 'active-token', generation = 1 }, + job = { job_id = 'j1', component = 'cm5', state = 'committing', active_token = 'active-token' }, + }) + assert_eq(result.tag, 'commit_started') + assert_eq(result.commit_token, 'active-token') + assert_eq(table.concat(order, ','), 'begin_attempt,backend_commit,commit_accepted') + end) +end + +function tests.test_commit_worker_rejects_backend_without_commit_policy() + fibers.run(function (scope) + local backend = { commit_op = function () return op.always({ accepted = true }, nil) end } + local ok, err = pcall(function () + active_job.commit(scope, { + backend = backend, + jobs = { admit_transition = function () return { outcome_op = function () return op.always({ status = 'persisted' }, nil) end }, nil end }, + lease = { token = 'active-token', generation = 1 }, + job = { job_id = 'j1', component = 'cm5', state = 'committing', active_token = 'active-token' }, + }) + end) + assert_eq(ok, false) + assert_true(tostring(err):find('commit policy', 1, true) ~= nil, 'expected explicit commit policy error') + end) +end + + +function tests.test_commit_worker_surfaces_critical_inconsistent_outcome_after_backend_acceptance() + fibers.run(function (scope) + local calls = 0 + local jobs = {} + function jobs:admit_transition(cmd) + calls = calls + 1 + if cmd.kind == 'begin_commit_attempt' then + return { outcome_op = function () return op.always({ status = 'persisted', commit_token = cmd.commit_token, commit_policy = cmd.commit_policy }, nil) end }, nil + end + if cmd.kind == 'commit_accepted' then + return { outcome_op = function () return op.always({ status = 'failed', reason = 'save_failed' }, nil) end }, nil + end + error('unexpected transition '..tostring(cmd.kind), 0) + end + local backend = {} + function backend:commit_capabilities() + return { policy = 'idempotent_by_token' } + end + function backend:commit_op(_, ctx) + return op.always({ accepted = true, token = ctx.commit_token }, nil) + end + local ok, err = pcall(function () + active_job.commit(scope, { + backend = backend, + jobs = jobs, + lease = { token = 'active-token', generation = 1 }, + job = { job_id = 'j1', component = 'cm5', state = 'committing', active_token = 'active-token' }, + }) + end) + assert_eq(ok, false) + assert_true(tostring(err):find('critical_inconsistent_commit_acceptance', 1, true) ~= nil, 'expected critical inconsistent-outcome failure') + assert_eq(calls, 2) + end) +end + +return tests diff --git a/tests/unit/update/test_publisher.lua b/tests/unit/update/test_publisher.lua new file mode 100644 index 00000000..8b18881a --- /dev/null +++ b/tests/unit/update/test_publisher.lua @@ -0,0 +1,114 @@ +-- tests/unit/update/test_publisher.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local busmod = require 'bus' + +local model_mod = require 'services.update.model' +local publisher = require 'services.update.publisher' +local topics = require 'services.update.topics' +local probe = require 'tests.support.bus_probe' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil value') end end + +function tests.test_publisher_retains_initial_and_changed_state() + fibers.run(function (scope) + local bus = busmod.new() + local conn = bus:connect() + local view_conn = bus:connect() + local m = model_mod.new(model_mod.service_initial('update', 0)) + + local pub, err = publisher.start(scope, { conn = conn, model = m }) + assert_not_nil(pub, err) + + local view = view_conn:retained_view(topics.update_summary()) + local payload = probe.wait_retained_payload(view_conn, topics.update_summary(), { timeout = 0.2, view = view }) + assert_eq(payload.service, 'update') + assert_eq(payload.state, 'starting') + + m:update(function (s) + s.state = 'running' + s.ready = true + return s + end) + + local ok_update = probe.wait_until(function () + local msg = view:get(topics.update_summary()) + return msg and msg.payload and msg.payload.state == 'running' + end, { timeout = 0.3 }) + + assert_true(ok_update, 'expected retained update') + local updated = view:get(topics.update_summary()).payload + assert_eq(updated.ready, true) + view:close() + + pub:stop('test complete') + fibers.perform(sleep.sleep_op(0.001)) + end) +end + + +function tests.test_publisher_publishes_compact_component_and_no_workflow_timeline() + fibers.run(function (scope) + local bus = busmod.new() + local conn = bus:connect() + local m = model_mod.new(model_mod.service_initial('update', 1)) + + local pub, err = publisher.start(scope, { conn = conn, model = m }) + assert_not_nil(pub, err) + + m:update(function (snap) + snap.ready = true + snap.state = 'running' + snap.config = { components = { mcu = { component = 'mcu' } } } + snap.jobs = { + count = 1, + order = { 'job-1' }, + by_id = { + ['job-1'] = { + job_id = 'job-1', + component = 'mcu', expected_image_id = 'mcu-image-test', + state = 'awaiting_commit', + artifact_ref = 'artifact-1', + updated_seq = 2, + last_event = { seq = 2, state = 'awaiting_commit', reason = 'stage_complete' }, + stage_result = { + transfer = { digest_alg = 'xxhash32', digest = '12345678', size = 5 }, + reply = { transfer = { xfer_id = 'xfer-1', request_id = 'req-1', link_id = 'link-a', sent_bytes = 5 } }, + }, + }, + }, + } + return snap + end) + + local component = probe.wait_retained_payload(conn, topics.update_component('mcu'), { timeout = 0.3 }) + assert_eq(component.kind, 'update.component') + assert_eq(component.current_job.job_id, 'job-1') + assert_eq(component.current_job.state, 'awaiting_commit') + assert_eq(component.jobs, nil) + + local summary = probe.wait_retained_payload(conn, topics.update_summary(), { timeout = 0.3 }) + assert_eq(summary.jobs.count, 1) + assert_eq(summary.jobs.by_id, nil) + assert_eq(summary.jobs.order, nil) + assert_eq(summary.jobs.last.history, nil) + assert_eq(summary.job.job_id, 'job-1') + + fibers.perform(sleep.sleep_op(0.05)) + local view = conn:retained_view({ 'state', 'workflow', '#' }) + assert_eq(view:get(topics.workflow_update_job_timeline('job-1')), nil) + assert_eq(view:get(topics.workflow_update_job('job-1')), nil) + view:close() + + pub:stop('test complete') + fibers.perform(sleep.sleep_op(0.001)) + end) +end + +return tests diff --git a/tests/unit/update/test_service.lua b/tests/unit/update/test_service.lua new file mode 100644 index 00000000..bb98b7d8 --- /dev/null +++ b/tests/unit/update/test_service.lua @@ -0,0 +1,242 @@ +-- tests/unit/update/test_service.lua + +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local busmod = require 'bus' + +local update = require 'services.update' +local service = require 'services.update.service' +local topics = require 'services.update.topics' +local probe = require 'tests.support.bus_probe' + +local tests = {} + +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a, b, msg) if a ~= b then fail(msg or ('expected ' .. tostring(b) .. ', got ' .. tostring(a))) end end +local function assert_true(v, msg) if v ~= true then fail(msg or ('expected true, got ' .. tostring(v))) end end +local function assert_not_nil(v, msg) if v == nil then fail(msg or 'expected non-nil value') end end + +local function run_service_once(params) + return fibers.run_scope(function (scope) + return service.run(scope, params) + end) +end + +function tests.test_public_update_module_exports_start_and_run() + assert_eq(type(update.start), 'function') + assert_eq(type(update.run), 'function') + assert_not_nil(update.config) + assert_not_nil(update.service) +end + +function tests.test_service_run_completes_with_injected_generation_runner() + fibers.run(function () + local st, rep, result = run_service_once { + publish = false, + service_id = 'update', + job_store_kind = 'memory', + watch_config = false, + generation_runner = function (_, params) + return { + role = 'fake_generation', + generation = params.generation, + } + end, + } + + assert_eq(st, 'ok') + assert_eq(#rep.extra_errors, 0) + assert_eq(result.role, 'update_service') + assert_eq(result.snapshot.state, 'stopped') + + end) +end + +function tests.test_manager_status_request_replies_immediately() + fibers.run(function (root_scope) + local bus = busmod.new() + local svc_conn = bus:connect() + local caller = bus:connect() + + local child, cerr = root_scope:child() + assert_not_nil(child, cerr) + local ok, err = child:spawn(function (scope) + service.run(scope, { + conn = svc_conn, + service_id = 'update', + job_store_kind = 'memory', + watch_config = false, + }) + end) + assert_true(ok, err) + + -- Let the manager endpoint bind. + fibers.perform(sleep.sleep_op(0.01)) + + local reply, call_err = caller:call(topics.update_manager_rpc('status'), {}, { timeout = 0.2 }) + assert_not_nil(reply, call_err) + assert_eq(reply.ok, true) + assert_eq(reply.snapshot.service, 'update') + + child:cancel('test complete') + end) +end + +function tests.test_config_change_replaces_generation() + fibers.run(function (root_scope) + local bus = busmod.new() + local svc_conn = bus:connect() + local cfg_conn = bus:connect() + local caller = bus:connect() + + local child = assert(root_scope:child()) + local ok, err = child:spawn(function (scope) + service.run(scope, { + conn = svc_conn, + service_id = 'update', + job_store_kind = 'memory', + }) + end) + assert_true(ok, err) + + local status_ready = probe.wait_until(function () + local reply = caller:call(topics.update_manager_rpc('status'), {}, { timeout = 0.1 }) + return reply and reply.ok == true and reply.snapshot and reply.snapshot.service == 'update' + end, { timeout = 1.0, interval = 0.01 }) + assert_true(status_ready, 'expected update status endpoint to be ready before changing config') + + cfg_conn:retain(topics.config(), { + rev = 2, + data = { + schema = 'devicecode.update/1', + namespace = 'new-ns', + components = { + { component = 'cm5' }, + }, + }, + }) + + local reply + local ok_wait = probe.wait_until(function () + reply = caller:call(topics.update_manager_rpc('status'), {}, { timeout = 0.1 }) + return reply and reply.snapshot and reply.snapshot.config and reply.snapshot.config.namespace == 'new-ns' + end, { timeout = 1.0, interval = 0.01 }) + + assert_true(ok_wait, 'expected config replacement to be visible') + assert_eq(reply.snapshot.config.component_count, 1) + + child:cancel('test complete') + end) +end + + +function tests.test_service_shell_owns_status_and_routes_generation_commands_to_private_queue() + fibers.run(function (root_scope) + local bus = busmod.new() + local svc_conn = bus:connect() + local caller = bus:connect() + local child = assert(root_scope:child()) + local got_private_request = false + local private_method = nil + + local ok, err = child:spawn(function (scope) + service.run(scope, { + conn = svc_conn, + service_id = 'update', + job_store_kind = 'memory', + watch_config = false, + publish = false, + generation_runner = function (_, params) + local req = fibers.perform(params.manager_rx:recv_op()) + got_private_request = req ~= nil + private_method = req and req._update_method or nil + if req then req:reply({ ok = true, generation = params.generation, method = private_method }) end + return { role = 'fake_generation', generation = params.generation } + end, + }) + end) + assert_true(ok, err) + + fibers.perform(sleep.sleep_op(0.02)) + local status, status_err = caller:call(topics.update_manager_rpc('status'), {}, { timeout = 0.5 }) + assert_not_nil(status, status_err) + assert_eq(status.ok, true) + assert_true(status.snapshot ~= nil, 'status should be answered by the service shell') + assert_eq(got_private_request, false) + + local reply, call_err = caller:call(topics.update_manager_rpc('create-job'), { + job_id = 'j-private-route', + component = 'cm5', + artifact_ref = 'artifact-private-route', + }, { timeout = 0.5 }) + assert_not_nil(reply, call_err) + assert_eq(reply.ok, true) + assert_true(got_private_request, 'expected generation-owned command to arrive through private route') + assert_eq(private_method, 'create-job') + + child:cancel('test complete') + end) +end + +function tests.test_publisher_failure_is_supervised_component_failure() + fibers.run(function () + local bus = busmod.new({ + authoriser = function (ctx) + if ctx.action == 'retain' then return false, 'retain_denied' end + return true, nil + end, + }) + local svc_conn = bus:connect() + + local st, _, primary = fibers.run_scope(function (scope) + return service.run(scope, { + conn = svc_conn, + service_id = 'update', + job_store_kind = 'memory', + watch_config = false, + generation_runner = function () + fibers.perform(sleep.sleep_op(10)) + return { role = 'never' } + end, + }) + end) + + assert_eq(st, 'failed') + assert_true(tostring(primary):find('publisher') ~= nil or tostring(primary):find('retain_denied') ~= nil) + end) +end + +function tests.test_service_uses_shared_config_watch_helper() + local source = assert(io.open('../src/services/update/service.lua', 'r')):read('*a') + assert_true(source:find("devicecode.support.config_watch", 1, true) ~= nil, 'update service should require shared config_watch helper') + assert_true(source:find('config_watch.open', 1, true) ~= nil, 'update service should open cfg/update through shared config_watch') + assert_true(source:find('watch_retained(conn, topics.config()', 1, true) == nil, 'update service should not own a bespoke retained config watcher') +end + +function tests.test_service_start_path_allows_injected_config_watch_for_harnesses() + fibers.run(function () + local rx = { + recv_op = function () return sleep.sleep_op(10) end, + } + local st, _, primary = fibers.run_scope(function (scope) + scope:spawn(function () + service.run(scope, { + publish = false, + service_id = 'update', + job_store_kind = 'memory', + config_watch = rx, + generation_runner = function () + fibers.perform(sleep.sleep_op(10)) + return { role = 'never' } + end, + }) + end) + fibers.perform(sleep.sleep_op(0.01)) + scope:cancel('test complete') + end) + assert_eq(st, 'cancelled') + assert_eq(primary, 'test complete') + end) +end + +return tests diff --git a/tests/unit/update/test_service_phase2.lua b/tests/unit/update/test_service_phase2.lua new file mode 100644 index 00000000..b6d26c36 --- /dev/null +++ b/tests/unit/update/test_service_phase2.lua @@ -0,0 +1,517 @@ +local fibers = require 'fibers' +local sleep = require 'fibers.sleep' +local cond = require 'fibers.cond' +local op = require 'fibers.op' +local busmod = require 'bus' +local service = require 'services.update.service' +local topics = require 'services.update.topics' +local probe = require 'tests.support.bus_probe' +local store_mod = require 'services.update.job_store_memory' +local tests = {} +local function fail(msg) error(msg or 'assertion failed', 2) end +local function assert_eq(a,b,msg) if a ~= b then fail(msg or ('expected '..tostring(b)..', got '..tostring(a))) end end +local function assert_true(v,msg) if v ~= true then fail(msg or ('expected true, got '..tostring(v))) end end +local function assert_not_nil(v,msg) if v == nil then fail(msg or 'expected non-nil') end end +local function assert_contains(s, needle, msg) if tostring(s or ''):find(tostring(needle), 1, true) == nil then fail(msg or ('expected '..tostring(s)..' to contain '..tostring(needle))) end end +local function start_service(root_scope, params) + params = params or {} + local bus = params.bus or busmod.new(); local svc_conn = bus:connect(); local caller = bus:connect(); local child = assert(root_scope:child()) + local ok, err = child:spawn(function (scope) params=params or {}; params.conn=svc_conn; params.service_id=params.service_id or 'update'; params.watch_config=false; if params.job_store == nil and params.job_store_kind == nil then params.job_store_kind='memory' end; service.run(scope, params) end) + assert_true(ok, err); fibers.perform(sleep.sleep_op(0.02)); return child, caller, bus +end +function tests.test_manager_create_job_is_scoped_and_updates_status_model() + fibers.run(function (root_scope) + local child, caller = start_service(root_scope, { config={ schema='devicecode.update/1', components={ { component='cm5' } } } }) + local reply, err = caller:call(topics.update_manager_rpc('create-job'), { job_id='j1', component='cm5', artifact_ref='artifact-1' }, { timeout=0.5 }) + assert_not_nil(reply, err); assert_eq(reply.ok, true); assert_eq(reply.job.job_id, 'j1') + local status + local ok_wait = probe.wait_until(function() status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }); return status and status.snapshot and status.snapshot.jobs and status.snapshot.jobs.by_id.j1 ~= nil end, { timeout=0.5, interval=0.01 }) + assert_true(ok_wait, 'expected created job to appear in service status'); assert_eq(status.snapshot.jobs.by_id.j1.component, 'cm5') + child:cancel('test complete') + end) +end +function tests.test_single_job_policy_replaces_previous_created_job() + fibers.run(function (root_scope) + local child, caller = start_service(root_scope, { config={ schema='devicecode.update/1', components={ { component='cm5' } } } }) + assert(caller:call(topics.update_manager_rpc('create-job'), { job_id='j1', component='cm5', artifact_ref='artifact-j1' }, { timeout=0.5 })) + assert(caller:call(topics.update_manager_rpc('create-job'), { job_id='j2', component='cm5', artifact_ref='artifact-j2' }, { timeout=0.5 })) + local status + assert_true(probe.wait_until(function() + status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot and status.snapshot.jobs.by_id.j2 ~= nil + end, { timeout=0.5, interval=0.01 }), 'expected newest job to be visible') + assert_eq(status.snapshot.jobs.count, 1) + assert_eq(status.snapshot.jobs.by_id.j1, nil) + assert_eq(status.snapshot.jobs.by_id.j2.state, 'created') + child:cancel('test complete') + end) +end + + +function tests.test_commit_job_persists_awaiting_return_before_reconcile() + fibers.run(function (root_scope) + local save_log = {} + local backend = { jobs = {} } + function backend:load_all_op() return op.always({ jobs = self.jobs, order = {} }, nil) end + function backend:save_job_op(job) + save_log[#save_log + 1] = { job_id = job.job_id, state = job.state } + self.jobs[job.job_id] = job + return op.always(true, nil) + end + + local active_backend = {} + function active_backend:stage_op(job) return op.always({ job_id=job.job_id }, nil) end + function active_backend:commit_capabilities() return { policy = 'idempotent_by_token' } end + function active_backend:commit_op(job, ctx) return op.always({ accepted=true, token=ctx.commit_token, job_id=job.job_id }, nil) end + function active_backend:evaluate_reconcile() return { done=true, tag='reconciled_success', observed={ ok=true } } end + + local child, caller = start_service(root_scope, { + config={ schema='devicecode.update/1', components={ { component='cm5' } } }, + job_store = backend, + backend = active_backend, + }) + assert(caller:call(topics.update_manager_rpc('create-job'), { job_id='j1', component='cm5', artifact_ref='artifact-j1' }, { timeout=0.5 })) + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot.jobs.by_id.j1 ~= nil + end, { timeout=0.5, interval=0.01 }), 'expected created job') + assert(caller:call(topics.update_manager_rpc('start-job'), { job_id='j1' }, { timeout=0.5 })) + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot.jobs.by_id.j1 and status.snapshot.jobs.by_id.j1.state == 'awaiting_commit' + end, { timeout=0.5, interval=0.01 }), 'expected staged job') + + local reply = assert(caller:call(topics.update_manager_rpc('commit-job'), { job_id='j1' }, { timeout=0.5 })) + assert_eq(reply.accepted, true) + local status + assert_true(probe.wait_until(function() + status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot.jobs.by_id.j1 and status.snapshot.jobs.by_id.j1.state == 'succeeded' + end, { timeout=0.8, interval=0.01 }), 'expected reconcile success after commit') + + local saw_awaiting_return, saw_succeeded = false, false + for _, row in ipairs(save_log) do + if row.job_id == 'j1' and row.state == 'awaiting_return' then saw_awaiting_return = true end + if row.job_id == 'j1' and row.state == 'succeeded' then saw_succeeded = true end + end + assert_true(saw_awaiting_return, 'commit accepted boundary must be durably saved') + assert_true(saw_succeeded, 'reconcile result must be saved') + child:cancel('test complete') + end) +end + +function tests.test_restart_adoption_keeps_awaiting_commit_committable() + fibers.run(function (root_scope) + local initial_jobs = { jobs = { j1 = { job_id='j1', component='cm5', state='awaiting_commit', created_seq=1, updated_seq=1 } } } + local child, caller = start_service(root_scope, { + config={ schema='devicecode.update/1', components={ { component='cm5' } } }, + initial_jobs = initial_jobs, + backend = { + stage_op = function () return op.always({}, nil) end, + commit_capabilities = function () return { policy = 'idempotent_by_token' } end, + commit_op = function (_, job, ctx) return op.always({ accepted=true, token=ctx.commit_token }, nil) end, + evaluate_reconcile = function () return { done=true, tag='reconciled_success', observed={ ok=true } } end, + }, + }) + local reply = assert(caller:call(topics.update_manager_rpc('commit-job'), { job_id='j1' }, { timeout=0.5 })) + assert_eq(reply.accepted, true) + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot.jobs.by_id.j1 and status.snapshot.jobs.by_id.j1.state == 'succeeded' + end, { timeout=0.8, interval=0.01 }), 'awaiting_commit should remain committable after adoption') + child:cancel('test complete') + end) +end + +function tests.test_restart_adoption_starts_reconcile_for_awaiting_return() + fibers.run(function (root_scope) + local ran_reconcile = false + local initial_jobs = { jobs = { j1 = { job_id='j1', component='cm5', state='awaiting_return', created_seq=1, updated_seq=1 } } } + local child, caller = start_service(root_scope, { + config={ schema='devicecode.update/1', components={ { component='cm5' } } }, + initial_jobs = initial_jobs, + backend = { + stage_op = function () return op.always({}, nil) end, + evaluate_reconcile = function () ran_reconcile = true; return { done=true, tag='reconciled_success', observed={ ok=true } } end, + }, + }) + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return ran_reconcile and status and status.snapshot.jobs.by_id.j1 and status.snapshot.jobs.by_id.j1.state == 'succeeded' + end, { timeout=0.8, interval=0.01 }), 'awaiting_return should start reconcile after adoption') + child:cancel('test complete') + end) +end + + +function tests.test_slow_job_runtime_load_keeps_public_service_responsive() + fibers.run(function (root_scope) + local load_gate = cond.new() + local backend = { loaded = false, jobs = {} } + function backend:load_all_op() + return load_gate:wait_op():wrap(function () + self.loaded = true + return { jobs = self.jobs, order = {} }, nil + end) + end + function backend:save_job_op(job) + self.jobs[job.job_id] = job + return op.always(true, nil) + end + local child, caller = start_service(root_scope, { + config={ schema='devicecode.update/1', components={ { component='cm5' } } }, + job_store = backend, + }) + local status = assert(caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.5 })) + assert_eq(status.ok, true) + assert_eq(status.snapshot.state, 'starting') + local created, create_err = caller:call(topics.update_manager_rpc('create-job'), { job_id='j1', component='cm5', artifact_ref='artifact-j1' }, { timeout=0.5 }) + assert_eq(created, nil) + assert_eq(create_err, 'job_runtime_not_ready') + load_gate:signal() + assert_true(probe.wait_until(function() + local s = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return s and s.snapshot and s.snapshot.state == 'running' + end, { timeout=0.6, interval=0.01 }), 'service should become running after job load') + child:cancel('test complete') + end) +end + +local function bind_fake_control_store(scope, bus, backing, opts) + backing = backing or {} + opts = opts or {} + local conn = bus:connect() + local methods = { 'list', 'get', 'put', 'delete' } + for _, method in ipairs(methods) do + local loop_method = method + local ep = assert(conn:bind({ 'cap', 'control-store', 'update', 'rpc', loop_method })) + scope:spawn(function () + while true do + local req = fibers.perform(ep:recv_op()) + if req == nil then return end + local p = req.payload or {} + if loop_method == 'list' then + if opts.list_calls then opts.list_calls.count = (opts.list_calls.count or 0) + 1 end + if opts.on_list then opts.on_list(p) end + if opts.fail_list then + req:reply({ ok = false, reason = opts.fail_list_reason or 'backend_failed' }) + else + local keys = {} + local prefix = p.prefix or '' + for k in pairs(backing) do + if k:sub(1, #prefix) == prefix then keys[#keys + 1] = k end + end + table.sort(keys) + req:reply({ ok = true, reason = keys }) + end + elseif loop_method == 'get' then + if backing[p.key] == nil then req:reply({ ok = false, reason = 'not found' }) else req:reply({ ok = true, reason = backing[p.key] }) end + elseif loop_method == 'put' then + backing[p.key] = p.data + req:reply({ ok = true, reason = nil }) + elseif loop_method == 'delete' then + backing[p.key] = nil + req:reply({ ok = true, reason = nil }) + end + end + end) + end + conn:retain({ 'cap', 'control-store', 'update', 'status' }, { schema='devicecode.cap.status/1', state='available', available=true }) + return backing, conn +end + +local function retain_fake_artifact_store_status(bus, status) + local conn = bus:connect() + conn:retain({ 'cap', 'artifact-store', 'main', 'status' }, { + schema='devicecode.cap.status/1', state=status or 'available', available=(status or 'available') == 'available' + }) + return conn +end + + +function tests.test_control_store_dependency_waits_until_available() + fibers.run(function (root_scope) + local bus = busmod.new() + retain_fake_artifact_store_status(bus, 'available') + local child, caller = start_service(root_scope, { + bus = bus, + job_store_kind = 'control-store', + config = { schema='devicecode.update/1', components={ { component='cm5' } } }, + }) + + local waiting + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + waiting = status and status.snapshot + return waiting and waiting.state == 'waiting_for_job_store' + end, { timeout=0.5, interval=0.01 }), 'expected update to wait for control-store capability') + assert_eq(waiting.ready, false) + assert_eq(waiting.reason, 'job_store_unavailable') + + local view = caller:retained_view(topics.update_summary()) + local retained_waiting = probe.wait_versioned_until('update retained summary includes dependencies while waiting', + function () return view:version() end, + function (seen) return view:changed_op(seen) end, + function () + local msg = view:get(topics.update_summary()) + local payload = msg and msg.payload + return payload and payload.state == 'waiting_for_job_store' + and payload.dependencies and payload.dependencies.job_store + and payload.dependencies.job_store.available == false + and payload or nil + end, + { timeout = 0.5 }) + assert_eq(retained_waiting.dependencies.job_store.status, 'configured') + view:close() + + local control_scope = assert(root_scope:child()) + bind_fake_control_store(control_scope, bus, {}) + + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot and status.snapshot.state == 'running' + end, { timeout=0.7, interval=0.01 }), 'expected update to start after control-store becomes available') + + child:cancel('test complete') + fibers.perform(child:join_op()) + control_scope:cancel('test complete') + fibers.perform(control_scope:join_op()) + end) +end + +function tests.test_control_store_no_route_returns_to_waiting_not_failed() + fibers.run(function (root_scope) + local bus = busmod.new() + retain_fake_artifact_store_status(bus, 'available') + local status_conn = bus:connect() + status_conn:retain({ 'cap', 'control-store', 'update', 'status' }, { schema='devicecode.cap.status/1', state='available', available=true }) + + local child, caller = start_service(root_scope, { + bus = bus, + job_store_kind = 'control-store', + config = { schema='devicecode.update/1', components={ { component='cm5' } } }, + }) + + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot and status.snapshot.state == 'waiting_for_job_store' + end, { timeout=0.7, interval=0.01 }), 'expected no_route job load to return update to waiting') + + local control_scope = assert(root_scope:child()) + bind_fake_control_store(control_scope, bus, {}) + + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot and status.snapshot.state == 'running' + end, { timeout=0.7, interval=0.01 }), 'expected update to restart job runtime after route returns') + + child:cancel('test complete') + fibers.perform(child:join_op()) + control_scope:cancel('test complete') + fibers.perform(control_scope:join_op()) + end) +end + +function tests.test_default_job_store_uses_control_store_and_reloads_after_restart() + fibers.run(function (root_scope) + local bus = busmod.new() + retain_fake_artifact_store_status(bus, 'available') + local control_scope = assert(root_scope:child()) + local backing = bind_fake_control_store(control_scope, bus, {}) + local child, caller = start_service(root_scope, { + bus = bus, + job_store_kind = 'control-store', + config = { schema='devicecode.update/1', components={ { component='cm5' } } }, + }) + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot and status.snapshot.state == 'running' + end, { timeout=0.5, interval=0.01 }), 'expected control-store job runtime to become ready') + local created, create_err = caller:call(topics.update_manager_rpc('create-job'), { job_id='j-persist', component='cm5', artifact_ref='artifact-j-persist' }, { timeout=0.5 }) + assert_not_nil(created, create_err) + assert_eq(created.ok, true) + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot and status.snapshot.jobs.by_id['j-persist'] ~= nil + end, { timeout=0.5, interval=0.01 }), 'expected first service to persist job') + child:cancel('first service complete') + fibers.perform(child:join_op()) + + local saw_key_after_first = false + for k in pairs(backing) do if k:sub(1, 11) == 'update-job-' then saw_key_after_first = true end end + assert_true(saw_key_after_first, 'expected first service to persist job in control-store keyspace') + + local child2, caller2 = start_service(root_scope, { + bus = bus, + job_store_kind = 'control-store', + config = { schema='devicecode.update/1', components={ { component='cm5' } } }, + }) + assert_true(probe.wait_until(function() + local status = caller2:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot and status.snapshot.jobs.by_id['j-persist'] ~= nil + end, { timeout=0.5, interval=0.01 }), 'expected restarted service to reload persisted job') + child2:cancel('test complete') + fibers.perform(child2:join_op()) + control_scope:cancel('test complete') + fibers.perform(control_scope:join_op()) + end) +end + + +function tests.test_control_store_backend_failure_still_fails_update() + fibers.run(function (root_scope) + local bus = busmod.new() + retain_fake_artifact_store_status(bus, 'available') + local control_scope = assert(root_scope:child()) + bind_fake_control_store(control_scope, bus, {}, { + fail_list = true, + fail_list_reason = 'backend_failed', + }) + + local child = start_service(root_scope, { + bus = bus, + job_store_kind = 'control-store', + config = { schema='devicecode.update/1', components={ { component='cm5' } } }, + }) + + local which, st, _, primary = fibers.perform(fibers.named_choice { + done = child:join_op(), + timeout = sleep.sleep_op(0.8), + }) + if which == 'timeout' then + child:cancel('test timeout') + fibers.perform(child:join_op()) + fail('expected update service to fail on real control-store backend failure') + end + assert_eq(st, 'failed') + assert_contains(primary, 'backend_failed') + + control_scope:cancel('test complete') + fibers.perform(control_scope:join_op()) + end) +end + +function tests.test_job_store_dependency_loss_cancels_and_reloads_runtime() + fibers.run(function (root_scope) + local bus = busmod.new() + retain_fake_artifact_store_status(bus, 'available') + local control_scope = assert(root_scope:child()) + local list_calls = { count = 0 } + local _, control_conn = bind_fake_control_store(control_scope, bus, {}, { list_calls = list_calls }) + local child, caller = start_service(root_scope, { + bus = bus, + job_store_kind = 'control-store', + config = { schema='devicecode.update/1', components={ { component='cm5' } } }, + }) + + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot and status.snapshot.state == 'running' + end, { timeout=0.7, interval=0.01 }), 'expected service running before dependency loss') + assert_true(list_calls.count >= 1, 'expected initial job runtime load') + + local created0, create_err0 = caller:call(topics.update_manager_rpc('create-job'), { + job_id='j-before-loss', component='cm5', artifact_ref='artifact-before-loss', + }, { timeout=0.5 }) + assert_not_nil(created0, create_err0) + assert_eq(created0.ok, true) + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot and status.snapshot.jobs.by_id['j-before-loss'] ~= nil + end, { timeout=0.5, interval=0.01 }), 'expected job visible before dependency loss') + local first_load_count = list_calls.count + + control_conn:retain({ 'cap', 'control-store', 'update', 'status' }, { + schema='devicecode.cap.status/1', state='unavailable', available=false, + }) + + local waiting + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + waiting = status and status.snapshot + return waiting and waiting.state == 'waiting_for_job_store' + end, { timeout=0.7, interval=0.01 }), 'expected service to wait when job-store dependency is lost') + assert_eq(waiting.ready, false) + assert_eq(waiting.reason, 'job_store_unavailable') + assert_eq(waiting.dependencies.job_store.available, false) + + local created, create_err = caller:call(topics.update_manager_rpc('create-job'), { + job_id='j-while-unavailable', component='cm5', artifact_ref='artifact-unavailable', + }, { timeout=0.2 }) + assert_eq(created, nil) + assert_eq(create_err, 'job_store_unavailable') + + control_conn:retain({ 'cap', 'control-store', 'update', 'status' }, { + schema='devicecode.cap.status/1', state='available', available=true, + }) + + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot + and status.snapshot.state == 'running' + and list_calls.count > first_load_count + end, { timeout=0.8, interval=0.01 }), 'expected service to reload job runtime from store after job-store returns') + + local created2, create_err2 = caller:call(topics.update_manager_rpc('create-job'), { + job_id='j-after-recovery', component='cm5', artifact_ref='artifact-after-recovery', + }, { timeout=0.5 }) + assert_not_nil(created2, create_err2) + assert_eq(created2.ok, true) + + child:cancel('test complete') + fibers.perform(child:join_op()) + control_scope:cancel('test complete') + fibers.perform(control_scope:join_op()) + end) +end + + + +function tests.test_artifact_store_dependency_gates_generation_after_job_store_ready() + fibers.run(function (root_scope) + local bus = busmod.new() + local control_scope = assert(root_scope:child()) + bind_fake_control_store(control_scope, bus, {}) + + local child, caller = start_service(root_scope, { + bus = bus, + job_store_kind = 'control-store', + config = { schema='devicecode.update/1', components={ { component='cm5' } } }, + }) + + local waiting + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + waiting = status and status.snapshot + return waiting and waiting.state == 'waiting_for_artifact_store' + end, { timeout=0.7, interval=0.01 }), 'expected service to wait for artifact-store after job runtime is available') + assert_eq(waiting.reason, 'artifact_store_unavailable') + assert_eq(waiting.dependencies.artifact_store.available, false) + assert_true(waiting.pending and waiting.pending.runtime and waiting.pending.runtime.dependency == 'artifact_store', + 'expected artifact-store pending runtime projection') + + local listed, list_err = caller:call(topics.update_manager_rpc('list-jobs'), {}, { timeout=0.2 }) + assert_not_nil(listed, list_err) + assert_eq(listed.ok, true) + + local created, create_err = caller:call(topics.update_manager_rpc('create-job'), { + job_id='j-artifact-wait', component='cm5', artifact_ref='artifact-wait', + }, { timeout=0.2 }) + assert_eq(created, nil) + assert_eq(create_err, 'artifact_store_unavailable') + + retain_fake_artifact_store_status(bus, 'available') + + assert_true(probe.wait_until(function() + local status = caller:call(topics.update_manager_rpc('status'), {}, { timeout=0.05 }) + return status and status.snapshot and status.snapshot.state == 'running' + end, { timeout=0.8, interval=0.01 }), 'expected update to admit generation after artifact-store becomes available') + + child:cancel('test complete') + fibers.perform(child:join_op()) + control_scope:cancel('test complete') + fibers.perform(control_scope:join_op()) + end) +end + +return tests diff --git a/tests/unit/wired/test_architecture.lua b/tests/unit/wired/test_architecture.lua new file mode 100644 index 00000000..73b1d168 --- /dev/null +++ b/tests/unit/wired/test_architecture.lua @@ -0,0 +1,62 @@ +local tests = {} + +local function read(path) + local f = assert(io.open(path, 'rb')) + local s = f:read('*a') + f:close() + return s +end + +local function assert_false(v, msg) if v ~= false then error(msg or 'expected false', 2) end end + +function tests.test_wired_service_has_no_os_backend_detail() + local src = read('../src/services/wired/service.lua') + assert_false(src:find('perform_raw', 1, true) ~= nil, 'wired service must not use perform_raw') + assert_false(src:find('join_op', 1, true) ~= nil, 'wired service must not join child scopes') + assert_false(src:find('services.hal.backends', 1, true) ~= nil, 'wired service must not require HAL backends') + assert_false(src:find('/sys/class', 1, true) ~= nil, 'wired service must not read sysfs') +end + + +function tests.test_wired_public_seam_has_no_provider_capability_projection() + for _, path in ipairs({ + '../src/services/wired/topics.lua', + '../src/services/wired/projection.lua', + '../src/services/wired/publisher.lua', + '../src/services/device/projection.lua', + '../src/services/device/publisher.lua', + }) do + local src = read(path) + assert_false(src:find('cap/wired-provider', 1, true) ~= nil, path .. ' must not publish cap/wired-provider') + assert_false(src:find("'state', 'wired', 'provider'", 1, true) ~= nil, path .. ' must not publish state/wired/provider') + end +end + + +function tests.test_switch_observation_path_uses_runtime_and_power_not_telemetry() + for _, path in ipairs({ + '../src/services/hal/backends/wired/providers/rtl8380m_http.lua', + '../src/services/hal/managers/wired.lua', + '../src/services/wired/service.lua', + }) do + local src = read(path) + assert_false(src:find("state', 'telemetry", 1, true) ~= nil, path .. ' must not publish state/telemetry for switch observations') + assert_false(src:find('telemetry.cpu', 1, true) ~= nil, path .. ' must not use telemetry.cpu') + assert_false(src:find('telemetry.mem', 1, true) ~= nil, path .. ' must not use telemetry.mem') + end +end + +function tests.test_switch_and_wired_path_has_no_soft_config_aliases() + local rtl = read('../src/services/hal/backends/wired/providers/rtl8380m_http.lua') + assert_false(rtl:find('config.id', 1, true) ~= nil, 'rtl8380m_http must not accept config.id') + assert_false(rtl:find("or 'main'", 1, true) ~= nil, 'rtl8380m_http must require http.capability') + assert_false(rtl:find("or 'legacy-http1-close'", 1, true) ~= nil, 'rtl8380m_http must require explicit legacy parser') + assert_false(rtl:find('default_surfaces', 1, true) ~= nil, 'rtl8380m_http must not have stub/default surfaces') + + local cfg = read('../src/services/wired/config.lua') + assert_false(cfg:find('rec.label', 1, true) ~= nil, 'cfg/wired must not accept label alias') + assert_false(cfg:find("and 'access' or 'trunk'", 1, true) ~= nil, 'cfg/wired must not infer attachment mode from segment') + assert_false(cfg:find("or { mode = 'none' }", 1, true) ~= nil, 'cfg/wired surfaces must declare attachment.mode') +end + +return tests diff --git a/tests/unit/wired/test_config.lua b/tests/unit/wired/test_config.lua new file mode 100644 index 00000000..5f5b79cf --- /dev/null +++ b/tests/unit/wired/test_config.lua @@ -0,0 +1,110 @@ +local config = require 'services.wired.config' + +local tests = {} +local function assert_true(v, msg) if v ~= true then error(msg or 'expected true', 2) end end +local function assert_not_nil(v, msg) if v == nil then error(msg or 'expected non-nil', 2) end end +local function assert_eq(a,b,msg) if a ~= b then error(msg or ('expected '..tostring(b)..', got '..tostring(a)), 2) end end + +function tests.test_cm5_trunk_config_normalises() + local intent, err = config.normalise({ + schema = config.SCHEMA, + surfaces = { + ['cm5-eth0'] = { + kind = 'direct-nic', + role = 'internal-trunk', + protected = true, + attachment = { mode = 'trunk', required_segments = { 'mgmt', 'switch_control', 'fabric' }, user_segments = 'all-realised-user-segments' }, + }, + }, + }) + assert_not_nil(intent, err) + assert_true(intent.surfaces['cm5-eth0'].protected) + assert_eq(intent.surfaces['cm5-eth0'].attachment.mode, 'trunk') +end + + +function tests.test_protected_surface_cannot_be_disabled() + local intent, err = config.normalise({ + schema = config.SCHEMA, + surfaces = { + ['cm5-eth0'] = { + protected = true, + enabled = false, + attachment = { mode = 'trunk', required_segments = { 'mgmt' } }, + }, + }, + }) + if intent ~= nil then error('expected invalid protected surface config') end + assert_not_nil(err) +end + +function tests.test_protected_surface_must_be_trunk_with_required_segments() + local intent, err = config.normalise({ + schema = config.SCHEMA, + surfaces = { + ['cm5-eth0'] = { + protected = true, + attachment = { mode = 'access', segment = 'lan' }, + }, + }, + }) + if intent ~= nil then error('expected protected non-trunk config to fail') end + assert_not_nil(err) + + intent, err = config.normalise({ + schema = config.SCHEMA, + surfaces = { + ['cm5-eth0'] = { + protected = true, + attachment = { mode = 'trunk' }, + }, + }, + }) + if intent ~= nil then error('expected protected trunk without required_segments to fail') end + assert_not_nil(err) +end + +function tests.test_access_surface_requires_segment() + local intent, err = config.normalise({ + schema = config.SCHEMA, + surfaces = { + ['lan-1'] = { + attachment = { mode = 'access' }, + }, + }, + }) + if intent ~= nil then error('expected invalid config') end + assert_not_nil(err) +end + + +function tests.test_attachment_mode_is_required() + local intent, err = config.normalise({ + schema = config.SCHEMA, + surfaces = { + ['lan-1'] = { + attachment = { segment = 'lan' }, + }, + }, + }) + if intent ~= nil then error('expected attachment without mode to fail') end + assert_not_nil(err) + assert_true(err:find('mode', 1, true) ~= nil, tostring(err)) +end + +function tests.test_surface_label_is_not_accepted() + local intent, err = config.normalise({ + schema = config.SCHEMA, + surfaces = { + ['lan-1'] = { + label = 'LAN 1', + attachment = { mode = 'none' }, + }, + }, + }) + if intent ~= nil then error('expected label alias to fail') end + assert_not_nil(err) + assert_true(err:find('label', 1, true) ~= nil, tostring(err)) +end + +return tests diff --git a/tests/unit/wired/test_dependencies.lua b/tests/unit/wired/test_dependencies.lua new file mode 100644 index 00000000..c5f95854 --- /dev/null +++ b/tests/unit/wired/test_dependencies.lua @@ -0,0 +1,32 @@ +local config = require 'services.wired.config' +local deps = require 'services.wired.dependencies' + +local T = {} + +function T.semantic_surfaces_have_no_capability_dependencies() + local intent = assert(config.normalise({ + schema = config.SCHEMA, + surfaces = { + lan1 = { attachment = { mode = 'access', segment = 'lan' } }, + lan2 = { attachment = { mode = 'access', segment = 'lan' } }, + }, + })) + local specs = deps.observation_dependencies(intent) + assert(#specs == 0) +end + +function T.provider_mapping_is_no_longer_accepted_in_cfg_wired() + local intent, err = config.normalise({ + schema = config.SCHEMA, + surfaces = { + lan1 = { + provider = { capability_id = 'switch-main', provider_surface_id = 'port1' }, + attachment = { mode = 'access', segment = 'lan' }, + }, + }, + }) + assert(intent == nil) + assert(type(err) == 'string' and err:find('provider', 1, true)) +end + +return T diff --git a/tests/unit/wired/test_publisher.lua b/tests/unit/wired/test_publisher.lua new file mode 100644 index 00000000..d7a46831 --- /dev/null +++ b/tests/unit/wired/test_publisher.lua @@ -0,0 +1,118 @@ +local publisher = require 'services.wired.publisher' + +local tests = {} + +local function eq(a, b, msg) + if a ~= b then error((msg or 'assertion failed') .. ': expected ' .. tostring(b) .. ', got ' .. tostring(a), 2) end +end +local function ok(v, msg) if not v then error(msg or 'assertion failed', 2) end return v end + +local function topic_key(t) + local parts = {} + for i = 1, #(t or {}) do parts[#parts + 1] = tostring(t[i]) end + return table.concat(parts, '/') +end + +local function fake_conn() + return { + retains = {}, + unretains = {}, + retain = function(self, topic, payload) + self.retains[#self.retains + 1] = { topic = topic_key(topic), payload = payload } + return true + end, + unretain = function(self, topic) + self.unretains[#self.unretains + 1] = topic_key(topic) + return true + end, + } +end + +local function snapshot(link_state) + return { + service = 'wired', + state = 'running', + ready = true, + generation = 1, + config = {}, + observations = {}, + stats = {}, + surfaces = { + a = { surface_id = 'a', link = { state = link_state or 'up' }, availability = { state = 'available' } }, + }, + topology = { trunks = {}, access = {}, protected_trunks = {} }, + violations = {}, + } +end + +local function topics(retains) + local out = {} + for _, rec in ipairs(retains) do out[#out + 1] = rec.topic end + table.sort(out) + return table.concat(out, ',') +end + +function tests.test_publish_all_suppresses_unchanged_retained_payloads() + local conn = fake_conn() + local published = publisher.new_state() + local dirty = publisher.mark_all(publisher.new_dirty_state()) + local ok1, err1, changed = publisher.publish_dirty_now(conn, snapshot('up'), published, dirty) + ok(ok1, err1) + eq(changed, 4) + eq(#conn.retains, 4) + dirty = publisher.mark_all(dirty) + local ok2, err2, changed2 = publisher.publish_dirty_now(conn, snapshot('up'), published, dirty) + ok(ok2, err2) + eq(changed2, 0) + eq(#conn.retains, 4) +end + +function tests.test_dirty_surface_publish_only_retains_changed_surface() + local conn = fake_conn() + local published = publisher.new_state() + local dirty = publisher.mark_all(publisher.new_dirty_state()) + ok(publisher.publish_dirty_now(conn, snapshot('up'), published, dirty)) + local before = #conn.retains + publisher.mark_surface(dirty, 'a') + local ok2, err2, changed = publisher.publish_dirty_now(conn, snapshot('down'), published, dirty) + ok(ok2, err2) + eq(changed, 1) + eq(#conn.retains, before + 1) + eq(conn.retains[#conn.retains].topic, 'state/wired/surface/a') +end + + +function tests.test_counter_updates_publish_metric_without_republishing_surface() + local conn = fake_conn() + local published = publisher.new_state() + local snap = snapshot('up') + snap.counters = { a = { surface_id = 'a', counters = { rx = { bytes = 1 } } } } + local dirty = publisher.mark_all(publisher.new_dirty_state()) + ok(publisher.publish_dirty_now(conn, snap, published, dirty)) + local before = #conn.retains + local snap2 = snapshot('up') + snap2.counters = { a = { surface_id = 'a', counters = { rx = { bytes = 2 } } } } + publisher.mark_counter(dirty, 'a') + local ok2, err2, changed = publisher.publish_dirty_now(conn, snap2, published, dirty) + ok(ok2, err2) + eq(changed, 1) + eq(#conn.retains, before + 1) + eq(conn.retains[#conn.retains].topic, 'obs/v1/wired/metric/surface_counters/a') +end + +function tests.test_removed_dirty_surface_is_unretained_once() + local conn = fake_conn() + local published = publisher.new_state() + local dirty = publisher.mark_all(publisher.new_dirty_state()) + ok(publisher.publish_dirty_now(conn, snapshot('up'), published, dirty)) + publisher.mark_surface(dirty, 'a') + local snap = snapshot('up') + snap.surfaces = {} + local ok2, err2, changed = publisher.publish_dirty_now(conn, snap, published, dirty) + ok(ok2, err2) + eq(changed, 1) + eq(#conn.unretains, 1) + eq(conn.unretains[1], 'state/wired/surface/a') +end + +return tests diff --git a/tests/unit/wired/test_trunk_validation.lua b/tests/unit/wired/test_trunk_validation.lua new file mode 100644 index 00000000..869c18a9 --- /dev/null +++ b/tests/unit/wired/test_trunk_validation.lua @@ -0,0 +1,241 @@ +local config = require 'services.wired.config' +local service = require 'services.wired.service' + +local tests = {} +local function assert_true(v,msg) if v ~= true then error(msg or 'expected true',2) end end +local function assert_not_nil(v,msg) if v == nil then error(msg or 'expected non-nil',2) end end +local function assert_eq(a,b,msg) if a ~= b then error(msg or ('expected '..tostring(b)..', got '..tostring(a)),2) end end + +local function protected_intent() + local intent, err = config.normalise({ + schema = config.SCHEMA, + surfaces = { + ['switch-uplink-cm5'] = { + kind = 'switch-port', + role = 'internal-trunk', + protected = true, + attachment = { mode = 'trunk', required_segments = { 'mgmt', 'switch_control', 'fabric' } }, + }, + }, + }) + assert_not_nil(intent, err) + return intent +end + +function tests.test_protected_trunk_reports_missing_required_vlan_carriage() + local snap = { + net = { + segments = { + mgmt = { vlan = { id = 10 } }, + switch_control = { vlan = { id = 11 } }, + fabric = { vlan = { id = 12 } }, + }, + }, + config_intent = protected_intent(), + assembly = { surfaces = { ['switch-uplink-cm5'] = { component = 'switch-main', observed_surface = 'GE8' } } }, + observations = { + ['switch-main'] = { + status = { state = 'available', available = true }, + surfaces = { + ['GE8'] = { + attachment = { mode = 'trunk', vlans = { 10, 12 } }, + }, + }, + }, + }, + stats = {}, + } + service._test.rebuild_derived(snap) + local found = false + for _, v in ipairs(snap.violations or {}) do + if v.kind == 'missing_required_segment_carriage' and v.segment == 'switch_control' and v.vlan == 11 then + found = true + assert_eq(v.severity, 'critical') + end + end + assert_true(found, 'expected missing switch_control VLAN carriage violation') + assert_eq(snap.state, 'degraded') +end + +function tests.test_protected_trunk_passes_when_all_required_vlans_are_observed() + local snap = { + net = { + segments = { + mgmt = { vlan = { id = 10 } }, + switch_control = { vlan = { id = 11 } }, + fabric = { vlan = { id = 12 } }, + }, + }, + config_intent = protected_intent(), + assembly = { surfaces = { ['switch-uplink-cm5'] = { component = 'switch-main', observed_surface = 'GE8' } } }, + observations = { + ['switch-main'] = { + status = { state = 'available', available = true }, + surfaces = { + ['GE8'] = { + attachment = { mode = 'trunk', vlans = { 10, 11, 12, 100 } }, + }, + }, + }, + }, + stats = {}, + } + service._test.rebuild_derived(snap) + for _, v in ipairs(snap.violations or {}) do + if v.kind == 'missing_required_segment_carriage' then + error('unexpected carriage violation: '..tostring(v.segment), 2) + end + end + assert_eq(snap.state, 'running') +end + + +function tests.test_protected_trunk_reports_source_missing() + local snap = { + net = { segments = { mgmt = { vlan = { id = 10 } } } }, + config_intent = protected_intent(), + assembly = { surfaces = { ['switch-uplink-cm5'] = { component = 'switch-main', observed_surface = 'GE8' } } }, + observations = {}, + stats = {}, + } + service._test.rebuild_derived(snap) + local found = false + for _, v in ipairs(snap.violations or {}) do + if v.kind == 'protected_source_missing' and v.surface_id == 'switch-uplink-cm5' then + found = true + assert_eq(v.severity, 'critical') + end + end + assert_true(found, 'expected protected source missing violation') +end + +function tests.test_protected_trunk_reports_observed_surface_missing() + local snap = { + net = { segments = { mgmt = { vlan = { id = 10 } }, switch_control = { vlan = { id = 11 } }, fabric = { vlan = { id = 12 } } } }, + config_intent = protected_intent(), + assembly = { surfaces = { ['switch-uplink-cm5'] = { component = 'switch-main', observed_surface = 'GE8' } } }, + observations = { ['switch-main'] = { status = { state = 'available', available = true }, surfaces = {} } }, + stats = {}, + } + service._test.rebuild_derived(snap) + local found = false + for _, v in ipairs(snap.violations or {}) do + if v.kind == 'protected_observed_surface_missing' and v.observed_surface == 'GE8' then + found = true + assert_eq(v.severity, 'critical') + end + end + assert_true(found, 'expected protected observed surface missing violation') +end + +function tests.test_all_realised_user_segments_are_checked_on_protected_trunk() + local intent, err = config.normalise({ + schema = config.SCHEMA, + surfaces = { + ['switch-uplink-cm5'] = { + protected = true, + attachment = { mode = 'trunk', required_segments = { 'mgmt' }, user_segments = 'all-realised-user-segments' }, + }, + }, + }) + assert_not_nil(intent, err) + local snap = { + net = { + segments = { + mgmt = { protected = true, kind = 'system', vlan = { id = 10 } }, + lan = { kind = 'user', vlan = { id = 100 } }, + guest = { kind = 'guest', vlan = { id = 101 } }, + }, + }, + config_intent = intent, + assembly = { surfaces = { ['switch-uplink-cm5'] = { component = 'switch-main', observed_surface = 'GE8' } } }, + observations = { ['switch-main'] = { status = { state = 'available', available = true }, surfaces = { ['GE8'] = { attachment = { mode = 'trunk', vlans = { 10, 100 } } } } } }, + stats = {}, + } + service._test.rebuild_derived(snap) + local found = false + for _, v in ipairs(snap.violations or {}) do + if v.kind == 'missing_user_segment_carriage' and v.segment == 'guest' and v.vlan == 101 then found = true end + end + assert_true(found, 'expected missing guest user-segment carriage violation') +end + +function tests.test_observed_surface_capability_checks_report_unsupported_access_trunk_and_poe() + local intent, err = config.normalise({ + schema = config.SCHEMA, + surfaces = { + ['lan-1'] = { + capabilities = { poe = true }, + attachment = { mode = 'access', segment = 'lan' }, + }, + ['trunk-1'] = { + attachment = { mode = 'trunk', segments = { 'lan' } }, + }, + }, + }) + assert_not_nil(intent, err) + local snap = { + net = { segments = { lan = { vlan = { id = 100 } } } }, + config_intent = intent, + assembly = { surfaces = { ['lan-1'] = { component = 'switch-main', observed_surface = 'GE1' }, ['trunk-1'] = { component = 'switch-main', observed_surface = 'GE2' } } }, + observations = { + ['switch-main'] = { + status = { state = 'available', available = true }, + surfaces = { + ['GE1'] = { capabilities = { access = false, trunk = true, poe = false }, attachment = { mode = 'access', vlan = 100 } }, + ['GE2'] = { capabilities = { access = true, trunk = false, poe = false }, attachment = { mode = 'access', vlan = 100 } }, + }, + }, + }, + stats = {}, + } + service._test.rebuild_derived(snap) + local seen = {} + for _, v in ipairs(snap.violations or {}) do seen[v.kind] = true end + assert_true(seen.observed_surface_does_not_support_access, 'access capability violation expected') + assert_true(seen.observed_surface_does_not_support_trunk, 'trunk capability violation expected') + assert_true(seen.observed_surface_does_not_support_poe, 'poe capability violation expected') +end + + +function tests.test_counter_observations_project_to_metric_not_stable_surface() + local snap = { + net = { segments = { jan = { vlan = 100 } } }, + config_intent = { surfaces = { lan = { surface_id = 'lan', id = 'lan', kind = 'ethernet-port', attachment = { mode = 'access', segment = 'jan' } } } }, + assembly = { surfaces = { lan = { component = 'switch-main', observed_surface = 'GE1' } } }, + observations = { + ['switch-main'] = { + status = { state = 'available', available = true }, + surfaces = { GE1 = { link = { state = 'up' }, attachment = { mode = 'access', vlan = 100 } } }, + counters = { GE1 = { rx = { bytes = 1234 } } }, + }, + }, + } + snap = service._test.rebuild_derived(snap) + assert_eq(snap.surfaces.lan.counters, nil) + assert_eq(snap.counters.lan.counters.rx.bytes, 1234) +end + +function tests.test_rebuild_derived_does_not_stamp_surfaces_with_volatile_updated_at() + local snap = { + net = { segments = { lan = { vlan = { id = 100 } } } }, + config_intent = select(1, config.normalise({ schema = config.SCHEMA, surfaces = { lan = { attachment = { mode = 'access', segment = 'lan' } } } })), + assembly = { surfaces = { lan = { component = 'switch-main', observed_surface = 'GE1' } } }, + observations = { ['switch-main'] = { status = { state = 'available', available = true }, surfaces = { GE1 = { link = { state = 'up' }, attachment = { mode = 'access', vlan = 100 } } } } }, + stats = {}, + } + service._test.rebuild_derived(snap) + assert_eq(snap.surfaces.lan.updated_at, nil) +end + +function tests.test_identical_wired_observation_does_not_invalidate_model() + local snap = { observations = {} } + local ev = { topic = { 'raw', 'host', 'wired', 'provider', 'switch-main', 'state', 'surfaces' }, payload = { surfaces = { GE1 = { link = { state = 'up' } } } } } + local changed, id = service._test.update_observation_from_event(snap, ev) + assert_eq(changed, true) + assert_eq(id, 'switch-main') + changed = service._test.update_observation_from_event(snap, ev) + assert_eq(changed, false) +end + +return tests diff --git a/vendor/lua-bus b/vendor/lua-bus new file mode 160000 index 00000000..57aae1bd --- /dev/null +++ b/vendor/lua-bus @@ -0,0 +1 @@ +Subproject commit 57aae1bd5f27248b07f7787a681dc427522e640e diff --git a/vendor/lua-fibers b/vendor/lua-fibers new file mode 160000 index 00000000..30353346 --- /dev/null +++ b/vendor/lua-fibers @@ -0,0 +1 @@ +Subproject commit 30353346ca58c3662b9942f62ff4a9610f8ab8cb diff --git a/vendor/lua-trie b/vendor/lua-trie new file mode 160000 index 00000000..dc03706b --- /dev/null +++ b/vendor/lua-trie @@ -0,0 +1 @@ +Subproject commit dc03706bed16c7aef93ee348dde48fe86aa94281